jmp-games/lddc
5
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add onnx and metadata

Browse Source
This commit is contained in:
Overside srl committed 2026-11-04 18:10:32 +01:00
1 parent 98d9fa24cf
commit 5f3f915258
18 files changed
+2229 -78
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

.gitignore
+9
View File
@@ -4,3 +4,12 @@
# Build & temp logs # Build & temp logs
*.log *.log
# Artefatti di build .NET (il progetto vive in bin/)
bin/bin/
bin/obj/
bin/.vs/
# Python
__pycache__/
*.pyc
.vs/LoopbackCapture.slnx/v18/.suo
BIN
View File
Binary file not shown.
.vs/LoopbackCapture.slnx/v18/DocumentLayout.json
+66
View File
@@ -0,0 +1,66 @@
{
"Version": 1,
"WorkspaceRootPath": "C:\\Users\\FlavioFabbrizi\\Documents\\Projects\\ldcc\\",
"Documents": [
{
"AbsoluteMoniker": "D:0:0:{0E03305C-5196-D0FE-BE44-8B31A1411D66}|LoopbackCapture.csproj|c:\\users\\flaviofabbrizi\\documents\\projects\\ldcc\\bin\\loopbackcapture.cs||{A6C744A8-0E4A-4FC6-886A-064283054674}",
"RelativeMoniker": "D:0:0:{0E03305C-5196-D0FE-BE44-8B31A1411D66}|LoopbackCapture.csproj|solutionrelative:bin\\loopbackcapture.cs||{A6C744A8-0E4A-4FC6-886A-064283054674}"
},
{
"AbsoluteMoniker": "D:0:0:{0E03305C-5196-D0FE-BE44-8B31A1411D66}|LoopbackCapture.csproj|c:\\users\\flaviofabbrizi\\documents\\projects\\ldcc\\bin\\genremetadataprovider.cs||{A6C744A8-0E4A-4FC6-886A-064283054674}",
"RelativeMoniker": "D:0:0:{0E03305C-5196-D0FE-BE44-8B31A1411D66}|LoopbackCapture.csproj|solutionrelative:bin\\genremetadataprovider.cs||{A6C744A8-0E4A-4FC6-886A-064283054674}"
}
],
"DocumentGroupContainers": [
{
"Orientation": 0,
"VerticalTabListWidth": 256,
"DocumentGroups": [
{
"DockedWidth": 200,
"SelectedChildIndex": 4,
"Children": [
{
"$type": "Bookmark",
"Name": "ST:128:0:{116d2292-e37d-41cd-a077-ebacac4c8cc4}"
},
{
"$type": "Bookmark",
"Name": "ST:129:0:{116d2292-e37d-41cd-a077-ebacac4c8cc4}"
},
{
"$type": "Bookmark",
"Name": "ST:0:0:{aa2115a1-9712-457b-9047-dbb71ca2cdd2}"
},
{
"$type": "Document",
"DocumentIndex": 1,
"Title": "GenreMetadataProvider.cs",
"DocumentMoniker": "C:\\Users\\FlavioFabbrizi\\Documents\\Projects\\ldcc\\bin\\GenreMetadataProvider.cs",
"RelativeDocumentMoniker": "bin\\GenreMetadataProvider.cs",
"ToolTip": "C:\\Users\\FlavioFabbrizi\\Documents\\Projects\\ldcc\\bin\\GenreMetadataProvider.cs",
"RelativeToolTip": "bin\\GenreMetadataProvider.cs",
"ViewState": "AgIAAAAAAAAAAAAAAAAAABoAAABCAAAAAAAAAA==",
"Icon": "ae27a6b0-e345-4288-96df-5eaf394ee369.000738|",
"WhenOpened": "2026-11-04T14:35:12.6Z",
"EditorCaption": ""
},
{
"$type": "Document",
"DocumentIndex": 0,
"Title": "LoopbackCapture.cs",
"DocumentMoniker": "C:\\Users\\FlavioFabbrizi\\Documents\\Projects\\ldcc\\bin\\LoopbackCapture.cs",
"RelativeDocumentMoniker": "bin\\LoopbackCapture.cs",
"ToolTip": "C:\\Users\\FlavioFabbrizi\\Documents\\Projects\\ldcc\\bin\\LoopbackCapture.cs",
"RelativeToolTip": "bin\\LoopbackCapture.cs",
"ViewState": "AgIAAAAAAAAAAAAAAAAAABcAAAAzAAAAAAAAAA==",
"Icon": "ae27a6b0-e345-4288-96df-5eaf394ee369.000738|",
"WhenOpened": "2026-11-04T14:35:05.159Z",
"EditorCaption": ""
}
]
}
]
}
]
}
LoopbackCapture.slnx
+3
View File
@@ -0,0 +1,3 @@
<Solution>
<Project Path="bin/LoopbackCapture.csproj" />
</Solution>
bin/GenreMetadataProvider.cs
+425
View File
@@ -0,0 +1,425 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Net.Http;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using Windows.Media.Control;
namespace LoopbackCapture
{
// Only this background worker accesses GSMTC/network/cache. The audio thread reads
// one immutable snapshot, without locks, COM calls, network or waits.
sealed class GenreMetadataProvider
{
// Explicit mapping: unmapped genres (including Soundtrack) are not guessed.
static readonly Dictionary<string, string> GenreMap = new Dictionary<string, string>(StringComparer.OrdinalIgnoreCase)
{
{ "Latin", "latin" }, { "Reggaeton", "latin" }, { "Salsa", "latin" }, { "Bachata", "latin" },
{ "Electronic", "electronic" }, { "Dance", "electronic" }, { "House", "electronic" },
{ "Techno", "electronic" }, { "Trance", "electronic" },
{ "Hip-Hop/Rap", "hiphop" }, { "Rap", "hiphop" },
{ "Rock", "rock" }, { "Metal", "rock" }, { "Hard Rock", "rock" },
{ "Alternative", "rock" }, { "Punk", "rock" },
{ "Classical", "classical" }, { "Opera", "classical" }
};
const int PollMs = 5000, SnapshotTtlMs = 10000, WinRtTimeoutMs = 3000, CacheLimit = 512;
const long PositiveCacheMs = 86400000, NegativeCacheMs = 300000;
sealed class Snapshot
{
internal readonly string Label;
internal readonly string Detail;
internal readonly float Bpm;
internal readonly long Expires;
internal Snapshot(string label, string detail, float bpm, long expires) { Label = label; Detail = detail; Bpm = bpm; Expires = expires; }
}
sealed class CacheEntry
{
internal string Label;
internal string Detail;
internal float Bpm;
internal long Expires;
internal string ItunesStatus;
internal string DeezerStatus;
}
sealed class Track
{
internal GlobalSystemMediaTransportControlsSession Session;
internal string Artist, Title, Key;
}
// Lazy initialization stays inside the guarded worker, including HttpClient failures.
static class Http
{
internal static readonly HttpClient Client = Create();
static HttpClient Create()
{
var client = new HttpClient { Timeout = TimeSpan.FromSeconds(3) };
client.DefaultRequestHeaders.UserAgent.ParseAdd("ldcc/1.0");
return client;
}
}
readonly Dictionary<string, CacheEntry> cache = new Dictionary<string, CacheEntry>(StringComparer.Ordinal);
Snapshot current;
string currentKey;
// True when the genre from metadata is actually consumed (EnableMetadataGenre).
// When false the provider only resolves the BPM: iTunes is skipped, Deezer still runs.
readonly bool wantGenre;
// Diagnostic state is never sent over UDP; useful for read-only verification.
internal string LastStatus { get; private set; } = "starting";
// Per-source diagnostic state (concise strings, no spaces). Updated where the
// fact happens (Worker, Lookup), read by the DIAG: packet without locks: a
// reference assignment of an immutable string is atomic on .NET.
string metaStatus = "starting";
string itunesStatus = "skipped";
string deezerStatus = "skipped";
string trackText = "";
internal GenreMetadataProvider(bool wantGenre)
{
this.wantGenre = wantGenre;
if (!wantGenre)
{
metaStatus = "bpm-only";
itunesStatus = "off";
}
}
internal string MetaStatus { get { return Volatile.Read(ref metaStatus); } }
internal string ItunesStatus { get { return Volatile.Read(ref itunesStatus); } }
internal string DeezerStatus { get { return Volatile.Read(ref deezerStatus); } }
internal string TrackText { get { return Volatile.Read(ref trackText); } }
void SetMeta(string s) { Volatile.Write(ref metaStatus, s); }
void SetItunes(string s) { Volatile.Write(ref itunesStatus, s); }
void SetDeezer(string s) { Volatile.Write(ref deezerStatus, s); }
void SetTrack(string s) { Volatile.Write(ref trackText, s); }
internal void Start()
{
try
{
var thread = new Thread(Worker) { IsBackground = true, Name = "Genre metadata" };
thread.SetApartmentState(ApartmentState.MTA);
thread.Start();
}
catch { Clear("unavailable"); SetMeta("error"); }
}
internal string GetLabel()
{
try
{
Snapshot value = Volatile.Read(ref current);
return value != null && Environment.TickCount64 < value.Expires ? value.Label : null;
}
catch { return null; }
}
// Genere specifico del catalogo ("Hard Rock", "Hip-Hop/Rap"). Solo descrittivo:
// la classe grossolana resta il contratto per il gameplay.
internal string GetDetail()
{
try
{
Snapshot value = Volatile.Read(ref current);
return value != null && Environment.TickCount64 < value.Expires ? value.Detail : null;
}
catch { return null; }
}
internal bool TryGetBpm(out float bpm)
{
bpm = 0;
try
{
Snapshot value = Volatile.Read(ref current);
if (value == null || Environment.TickCount64 >= value.Expires) return false;
if (value.Bpm <= 0) return false;
bpm = value.Bpm;
return true;
}
catch { return false; }
}
void Clear(string status)
{
Volatile.Write(ref current, null);
currentKey = null;
LastStatus = status;
}
static T AwaitWinRt<T>(Task<T> task)
{
return task.WaitAsync(TimeSpan.FromMilliseconds(WinRtTimeoutMs)).GetAwaiter().GetResult();
}
static Track ReadPlayingTrack()
{
var manager = AwaitWinRt(GlobalSystemMediaTransportControlsSessionManager.RequestAsync().AsTask());
foreach (var session in manager.GetSessions())
{
try
{
if (session.GetPlaybackInfo()?.PlaybackStatus != GlobalSystemMediaTransportControlsSessionPlaybackStatus.Playing) continue;
var media = AwaitWinRt(session.TryGetMediaPropertiesAsync().AsTask());
string artist = Normalize(media.Artist), title = Normalize(media.Title);
if (artist.Length == 0 || title.Length == 0) continue;
return new Track { Session = session, Artist = media.Artist, Title = media.Title, Key = artist + "\n" + title };
}
catch { }
}
return null;
}
void Worker()
{
// No async-void callback: every failure, including WinRT startup, is contained here.
while (true)
{
long start = Environment.TickCount64;
try
{
Track track = ReadPlayingTrack();
if (track == null)
{
Clear("no-playing-track");
SetMeta(wantGenre ? "no-track" : "bpm-only");
SetTrack("");
SetItunes(wantGenre ? "skipped" : "off");
SetDeezer("skipped");
}
else
{
SetTrack(SanitizeTrack(track.Artist + " - " + track.Title));
if (track.Key != currentKey) Clear("lookup");
currentKey = track.Key;
if (wantGenre) SetMeta("starting");
(string label, string detail, float bpm) = Lookup(track);
// A track may change while HTTP is in flight. Never publish that stale result.
var media = AwaitWinRt(track.Session.TryGetMediaPropertiesAsync().AsTask());
string key = Normalize(media.Artist) + "\n" + Normalize(media.Title);
if (key != track.Key || track.Session.GetPlaybackInfo()?.PlaybackStatus != GlobalSystemMediaTransportControlsSessionPlaybackStatus.Playing)
{
Clear("track-changed");
if (wantGenre) SetMeta("starting");
}
else if (wantGenre && label == null)
{
Clear("no-match-or-lookup-unavailable");
SetMeta("no-match");
}
else if (!wantGenre && bpm <= 0)
{
// BPM-only with no BPM: nothing to publish.
Clear("no-bpm");
}
else
{
currentKey = track.Key;
Volatile.Write(ref current, new Snapshot(label, detail, bpm, Environment.TickCount64 + SnapshotTtlMs));
LastStatus = wantGenre ? "metadata" : "bpm";
SetMeta(wantGenre ? "ok" : "bpm-only");
}
}
}
catch { Clear("unavailable"); SetMeta("error"); }
try { Thread.Sleep((int)Math.Max(1, PollMs - (Environment.TickCount64 - start))); }
catch { return; }
}
}
(string Label, string Detail, float Bpm) Lookup(Track track)
{
long now = Environment.TickCount64;
if (cache.TryGetValue(track.Key, out CacheEntry value) && now < value.Expires)
{
// Replay the per-source status captured at lookup time, so a negative
// cache does not hide "no-match"/"error" behind "skipped" for 5 minutes.
SetItunes(value.ItunesStatus ?? (wantGenre ? "skipped" : "off"));
SetDeezer(value.DeezerStatus ?? "skipped");
return (value.Label, value.Detail, value.Bpm);
}
string label = null, detail = null, itunesState;
if (wantGenre)
{
(label, detail, itunesState, _) = LookupItunesDetailed(track.Artist, track.Title);
}
else
{
// BPM-only: iTunes is not needed, skip the extra network call.
itunesState = "off";
}
var (bpm, deezerState, _) = LookupDeezerDetailed(track.Artist, track.Title);
SetItunes(itunesState);
SetDeezer(deezerState);
if (cache.Count >= CacheLimit) cache.Clear();
cache[track.Key] = new CacheEntry
{
Label = label,
Detail = detail,
Bpm = bpm,
Expires = now + (label == null ? NegativeCacheMs : PositiveCacheMs),
ItunesStatus = itunesState,
DeezerStatus = deezerState
};
return (label, detail, bpm);
}
// One iTunes lookup (search + genre mapping) with a per-source status and a
// human-readable reason. Used by both the worker (status only) and --check-sources.
internal static (string Label, string Detail, string Status, string Reason) LookupItunesDetailed(string artist, string title)
{
try
{
string url = "https://itunes.apple.com/search?term=" + Uri.EscapeDataString(artist + " " + title) + "&entity=song&limit=5&country=IT";
string json = Http.Client.GetStringAsync(url).GetAwaiter().GetResult();
(string label, string detail) = MatchResponse(json, artist, title);
if (label == null) return (null, null, "no-match", "nessun match iTunes (artista/titolo non corrispondenti o genere non mappato)");
return (label, detail, "ok", "match");
}
catch (Exception ex) { return (null, null, "error", ClassifyHttp(ex)); }
}
// One Deezer lookup (search -> track id -> bpm) with status and reason.
internal static (float Bpm, string Status, string Reason) LookupDeezerDetailed(string artist, string title)
{
try
{
string searchUrl = "https://api.deezer.com/search?q=" + Uri.EscapeDataString(artist + " " + title) + "&limit=1";
string searchJson = Http.Client.GetStringAsync(searchUrl).GetAwaiter().GetResult();
long id = DeezerSearchId(searchJson);
if (id <= 0) return (0, "no-match", "nessun risultato di ricerca Deezer");
string trackUrl = "https://api.deezer.com/track/" + id;
string trackJson = Http.Client.GetStringAsync(trackUrl).GetAwaiter().GetResult();
(float bpm, string status) = DeezerTrackBpmStatus(trackJson, artist, title);
return (bpm, status, status == "ok" ? "match" : status == "no-bpm" ? "BPM assente" : "artista/titolo non corrispondenti");
}
catch (Exception ex) { return (0, "error", ClassifyHttp(ex)); }
}
// Exact reason for a failed lookup: HTTP status, timeout or invalid JSON.
static string ClassifyHttp(Exception ex)
{
if (ex is HttpRequestException hre && hre.StatusCode != null) return "HTTP " + (int)hre.StatusCode;
if (ex is TaskCanceledException) return "timeout (3s)";
if (ex is JsonException) return "JSON invalido";
return ex.GetType().Name + ": " + ex.Message;
}
// GSMTC track for --check-sources. Null when no session is playing or GSMTC fails.
internal static (string Artist, string Title) ReadCurrentTrack()
{
try
{
Track track = ReadPlayingTrack();
return track == null ? (null, null) : (track.Artist, track.Title);
}
catch { return (null, null); }
}
static string SanitizeTrack(string value)
{
if (string.IsNullOrEmpty(value)) return "";
StringBuilder text = new StringBuilder(value.Length);
foreach (char c in value)
{
if (c == ';' || c == '=' || c == '\r' || c == '\n') text.Append(' ');
else text.Append(c);
}
return text.ToString().Trim();
}
static long DeezerSearchId(string json)
{
using (JsonDocument document = JsonDocument.Parse(json))
{
if (!document.RootElement.TryGetProperty("data", out JsonElement data) || data.ValueKind != JsonValueKind.Array) return 0;
foreach (JsonElement item in data.EnumerateArray())
{
if (item.TryGetProperty("id", out JsonElement id) && id.ValueKind == JsonValueKind.Number)
return id.GetInt64();
}
}
return 0;
}
// Distinguishes "no match" (artist/title differ) from "no bpm" (matched but
// the field is absent or zero), which the DIAG states expose separately.
static (float Bpm, string Status) DeezerTrackBpmStatus(string json, string artist, string title)
{
using (JsonDocument document = JsonDocument.Parse(json))
{
JsonElement root = document.RootElement;
if (!root.TryGetProperty("title", out JsonElement t) || t.ValueKind != JsonValueKind.String) return (0, "no-match");
string foundTitle = t.GetString();
if (!root.TryGetProperty("artist", out JsonElement art) || art.ValueKind != JsonValueKind.Object) return (0, "no-match");
if (!art.TryGetProperty("name", out JsonElement an) || an.ValueKind != JsonValueKind.String) return (0, "no-match");
string foundArtist = an.GetString();
if (!MatchesTrack(foundArtist, foundTitle, artist, title)) return (0, "no-match");
if (!root.TryGetProperty("bpm", out JsonElement bpm) || bpm.ValueKind != JsonValueKind.Number) return (0, "no-bpm");
float v = bpm.GetSingle();
return (v > 0 ? v : 0, v > 0 ? "ok" : "no-bpm");
}
}
// Same normalization/word-boundary match used for iTunes: artist and title must
// both correspond, otherwise the Deezer result is discarded.
static bool MatchesTrack(string foundArtist, string foundTitle, string wantedArtist, string wantedTitle)
{
string fA = Normalize(foundArtist), fT = Normalize(foundTitle);
string wA = Normalize(wantedArtist), wT = Normalize(wantedTitle);
if (fA.Length == 0 || fT.Length == 0 || wA.Length == 0 || wT.Length == 0) return false;
if (fT != wT) return false;
if (!(" " + fA + " ").Contains(" " + wA + " ", StringComparison.Ordinal) &&
!(" " + wA + " ").Contains(" " + fA + " ", StringComparison.Ordinal)) return false;
return true;
}
internal static string Normalize(string value)
{
if (string.IsNullOrWhiteSpace(value)) return "";
// Remove parenthesized edition qualifiers; punctuation becomes whitespace.
value = Regex.Replace(value, @"\([^)]*\)|\[[^\]]*\]", " ").ToLowerInvariant();
var text = new StringBuilder();
bool space = false;
foreach (char c in value)
{
if (char.IsLetterOrDigit(c)) { if (space && text.Length > 0) text.Append(' '); text.Append(c); space = false; }
else space = true;
}
return text.ToString();
}
internal static (string Label, string Detail) MatchResponse(string json, string artist, string title)
{
string wantedArtist = Normalize(artist), wantedTitle = Normalize(title);
if (wantedArtist.Length == 0 || wantedTitle.Length == 0) return (null, null);
using (JsonDocument document = JsonDocument.Parse(json))
{
if (!document.RootElement.TryGetProperty("results", out JsonElement results) || results.ValueKind != JsonValueKind.Array) return (null, null);
string match = null, detail = null;
foreach (JsonElement item in results.EnumerateArray())
{
if (!item.TryGetProperty("artistName", out JsonElement a) || a.ValueKind != JsonValueKind.String ||
!item.TryGetProperty("trackName", out JsonElement t) || t.ValueKind != JsonValueKind.String ||
!item.TryGetProperty("primaryGenreName", out JsonElement g) || g.ValueKind != JsonValueKind.String) continue;
string foundArtist = Normalize(a.GetString()), foundTitle = Normalize(t.GetString());
// Word boundaries avoid e.g. "Cher" matching "Cherry". Title must also match.
if (foundArtist.Length == 0 || foundTitle != wantedTitle) continue;
if (!(" " + foundArtist + " ").Contains(" " + wantedArtist + " ", StringComparison.Ordinal) &&
!(" " + wantedArtist + " ").Contains(" " + foundArtist + " ", StringComparison.Ordinal)) continue;
string raw = g.GetString().Trim();
if (!GenreMap.TryGetValue(raw, out string label)) continue;
if (match != null && match != label) return (null, null); // Ambiguous editions: prefer rules.
match = label;
if (detail == null) detail = raw; // Genere specifico del catalogo, es. "Hard Rock".
}
return (match, detail);
}
}
}
}
bin/GenreOnnxProvider.cs
+346
View File
@@ -0,0 +1,346 @@
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Text;
using System.Text.Json;
using System.Threading;
using Microsoft.ML.OnnxRuntime;
using Microsoft.ML.OnnxRuntime.Tensors;
namespace LoopbackCapture
{
// Immutable publication; never expose worker-owned/reused inference buffers.
sealed class GenrePrediction
{
internal readonly string Label, Source;
// Genere specifico dal catalogo ("Hard Rock"), vuoto per regole/ONNX.
internal readonly string Detail;
internal readonly float[] Probabilities;
internal readonly long ExpiresMs;
internal float Confidence { get { return Probabilities.Max(); } }
internal GenrePrediction(float[] probabilities, string source, long expires, string detail = null)
{
Probabilities = probabilities; Source = source; ExpiresMs = expires; Detail = detail;
int best = 0;
for (int i = 1; i < probabilities.Length; i++) if (probabilities[i] > probabilities[best]) best = i;
Label = probabilities[best] < 0.30f ? "unknown" : GenreOnnxProvider.Labels[best];
}
}
sealed class GenreOnnxProvider
{
internal static readonly string[] Labels = { "latin", "electronic", "hiphop", "rock", "classical" };
internal const int FftSize = 2048, MelBands = 128, FrameCount = 300;
const int InferenceMs = 1000, ResultTtlMs = 3000;
readonly int rate;
readonly string dumpPath;
readonly ManualResetEventSlim initialized = new ManualResetEventSlim(false);
MelFrontend frontend;
GenrePrediction prediction;
int disabled, dumpWritten;
string status = "warming";
string failureReason = "";
internal bool Failed { get { return Volatile.Read(ref disabled) != 0; } }
internal bool DumpWritten { get { return Volatile.Read(ref dumpWritten) != 0; } }
// Diagnostic status: off | no-model | warming | ok | failed. Read by the DIAG:
// packet without locks (string reference assignment is atomic).
internal string Status { get { return Volatile.Read(ref status); } }
internal string FailureReason { get { return Volatile.Read(ref failureReason); } }
void SetStatus(string s) { Volatile.Write(ref status, s); }
void SetFailure(string s) { Volatile.Write(ref failureReason, s); }
internal GenreOnnxProvider(int sampleRate, string dump)
{ rate = sampleRate; dumpPath = dump; }
internal void Start()
{
try { new Thread(Worker) { IsBackground = true, Name = "Genre ONNX" }.Start(); }
catch (Exception ex) { Disable(ex); initialized.Set(); }
}
void Disable(Exception ex)
{
Volatile.Write(ref prediction, null);
if (Interlocked.Exchange(ref disabled, 1) == 0)
{
SetFailure(ex.Message);
SetStatus("failed");
Console.Error.WriteLine("ONNX disabled for this session: " + ex.Message +
(dumpPath != null && !DumpWritten ? " (tensor dump rejected: no valid tensor written)" : ""));
}
}
internal bool WaitForInitialization(int milliseconds)
{ return initialized.Wait(milliseconds) && !Failed && Volatile.Read(ref frontend) != null; }
// Called on audio thread: preallocated math only. No locks, I/O, or waits.
internal void AddSample(float sample)
{
if (Failed) return;
try { Volatile.Read(ref frontend)?.AddSample(sample); }
catch (Exception ex) { Disable(ex); }
}
internal GenrePrediction GetPrediction()
{
if (Failed) return null;
GenrePrediction value = Volatile.Read(ref prediction);
return value != null && Environment.TickCount64 < value.ExpiresMs ? value : null;
}
void Worker()
{
try
{
string modelPath = Path.Combine(AppContext.BaseDirectory, "genre_model.onnx");
string jsonPath = Path.Combine(AppContext.BaseDirectory, "genre_model.json");
if (!File.Exists(modelPath) || !File.Exists(jsonPath))
{
// Permanent: the model is absent, not a transient load failure.
SetFailure("genre_model.onnx / genre_model.json missing");
SetStatus("no-model");
Volatile.Write(ref disabled, 1);
return;
}
using var json = JsonDocument.Parse(File.ReadAllText(jsonPath));
var root = json.RootElement;
if (root.GetProperty("version").GetInt32() != 1 ||
!root.GetProperty("labels").EnumerateArray().Select(x => x.GetString()).SequenceEqual(Labels))
throw new InvalidDataException("Unsupported model version or label order");
double mean = root.GetProperty("mean").GetDouble(), std = root.GetProperty("std").GetDouble();
if (!double.IsFinite(mean) || !double.IsFinite(std) || std <= 0) throw new InvalidDataException("Invalid model mean/std");
using var options = new SessionOptions { IntraOpNumThreads = 1, InterOpNumThreads = 1, ExecutionMode = ExecutionMode.ORT_SEQUENTIAL };
using var session = new InferenceSession(modelPath, options);
if (session.InputMetadata.Count != 1 || session.OutputMetadata.Count != 1) throw new InvalidDataException("Expected one model input and output");
var input = session.InputMetadata.Single(); var output = session.OutputMetadata.Single();
if (input.Value.ElementType != typeof(float) || !input.Value.Dimensions.SequenceEqual(new[] { 1, 1, MelBands, FrameCount }) ||
output.Value.ElementType != typeof(float) || !output.Value.Dimensions.SequenceEqual(new[] { 1, 5 }))
throw new InvalidDataException("Expected float32 [1,1,128,300] -> [1,5]");
var localFrontend = new MelFrontend(rate, mean, std);
var snapshot = new float[MelBands * FrameCount];
var tensor = new DenseTensor<float>(snapshot, new[] { 1, 1, MelBands, FrameCount });
var inputs = new[] { NamedOnnxValue.CreateFromTensor(input.Key, tensor) };
Volatile.Write(ref frontend, localFrontend);
initialized.Set();
SetStatus("warming");
long next = Environment.TickCount64, lastFrame = -1;
while (!Failed)
{
long wait = next - Environment.TickCount64;
if (wait > 0) Thread.Sleep((int)Math.Min(wait, InferenceMs)); // Worker only.
if (Failed) break;
next += InferenceMs;
if (localFrontend.TrySnapshot(snapshot, out long frame) && frame != lastFrame)
{
lastFrame = frame;
if (dumpPath != null && !DumpWritten)
{
// BinaryWriter writes IEEE float32 little-endian; exactly 153600 bytes.
// Write atomically: no empty/partial target if initialization or I/O fails.
string temporary = dumpPath + "." + Guid.NewGuid().ToString("N") + ".tmp";
try
{
using (var writer = new BinaryWriter(File.Open(temporary, FileMode.CreateNew)))
foreach (float value in snapshot) writer.Write(value);
File.Move(temporary, dumpPath, true);
Volatile.Write(ref dumpWritten, 1);
}
finally { if (File.Exists(temporary)) File.Delete(temporary); }
}
using var result = session.Run(inputs);
float[] probabilities = result.Single().AsTensor<float>().ToArray();
double sum = probabilities.Sum(x => (double)x);
if (probabilities.Length != 5 || probabilities.Any(x => !float.IsFinite(x) || x < 0 || x > 1) || Math.Abs(sum - 1) > 0.001)
throw new InvalidDataException("Model output is not five probabilities summing to one");
if (!Failed) Volatile.Write(ref prediction, new GenrePrediction(probabilities, "onnx", Environment.TickCount64 + ResultTtlMs));
if (!Failed) SetStatus("ok");
}
// Skip missed slots. Never accumulate work when Run exceeds one second.
long now = Environment.TickCount64;
if (next <= now) next += ((now - next) / InferenceMs + 1) * InferenceMs;
}
}
catch (Exception ex) { Disable(ex); }
finally { initialized.Set(); }
}
sealed class MelFrontend
{
readonly int hop;
readonly double mean, std;
readonly double[] pcm = new double[FftSize], window = new double[FftSize];
readonly double[] real = new double[FftSize], imag = new double[FftSize], power = new double[FftSize / 2 + 1];
readonly double[] cos = new double[FftSize / 2], sin = new double[FftSize / 2];
readonly int[] reverse = new int[FftSize], starts = new int[MelBands];
readonly double[][] weights = new double[MelBands][];
readonly float[] ring = new float[MelBands * FrameCount];
int write, pending, sequence;
long frames;
internal MelFrontend(int rate, double mean, double std)
{
if (rate != 44100 && rate != 48000) throw new NotSupportedException("ONNX front-end supports 44100/48000 Hz");
hop = rate / 100; this.mean = mean; this.std = std;
for (int i = 0; i < FftSize; i++)
{
window[i] = 0.5 - 0.5 * Math.Cos(2 * Math.PI * i / FftSize); // PERIODIC, independent of rules.
int x = i, r = 0;
for (int b = 0; b < 11; b++) { r = (r << 1) | (x & 1); x >>= 1; }
reverse[i] = r;
}
for (int i = 0; i < cos.Length; i++) { cos[i] = Math.Cos(-2 * Math.PI * i / FftSize); sin[i] = Math.Sin(-2 * Math.PI * i / FftSize); }
double low = 2595 * Math.Log10(1 + 30.0 / 700), high = 2595 * Math.Log10(1 + 11000.0 / 700);
var edges = new double[MelBands + 2];
for (int i = 0; i < edges.Length; i++) edges[i] = 700 * (Math.Pow(10, (low + (high - low) * i / (MelBands + 1)) / 2595) - 1) * FftSize / rate;
for (int m = 0; m < MelBands; m++)
{
double left = edges[m], center = edges[m + 1], right = edges[m + 2];
starts[m] = Math.Max(0, (int)Math.Ceiling(left));
int end = Math.Min(FftSize / 2, (int)Math.Floor(right));
weights[m] = new double[Math.Max(0, end - starts[m] + 1)];
for (int j = 0; j < weights[m].Length; j++)
{
int k = starts[m] + j;
// Fractional edges; divide by FULL support width in bins (not half-area).
weights[m][j] = Math.Max(0, Math.Min((k - left) / (center - left), (right - k) / (right - center))) / (right - left);
}
}
}
internal void AddSample(float sample)
{
if (!float.IsFinite(sample)) throw new InvalidDataException("Non-finite PCM");
pcm[write] = sample; write = (write + 1) % FftSize;
if (++pending < hop) return;
pending = 0;
// The initial zeroed ring supplies exactly N-hop left padding.
for (int i = 0; i < FftSize; i++) { real[reverse[i]] = pcm[(write + i) % FftSize] * window[i]; imag[reverse[i]] = 0; }
for (int n = 2; n <= FftSize; n <<= 1)
for (int start = 0; start < FftSize; start += n)
for (int j = 0; j < n / 2; j++)
{
int a = start + j, b = a + n / 2, t = j * (FftSize / n);
double tr = cos[t] * real[b] - sin[t] * imag[b], ti = sin[t] * real[b] + cos[t] * imag[b];
real[b] = real[a] - tr; imag[b] = imag[a] - ti; real[a] += tr; imag[a] += ti;
}
for (int k = 0; k < power.Length; k++) power[k] = real[k] * real[k] + imag[k] * imag[k];
Interlocked.Increment(ref sequence); // odd: producer updating frame. Never waits.
int column = (int)(frames % FrameCount);
for (int m = 0; m < MelBands; m++)
{
double energy = 0;
for (int j = 0; j < weights[m].Length; j++) energy += weights[m][j] * power[starts[m] + j];
ring[m * FrameCount + column] = (float)((Math.Log10(Math.Max(energy, 1e-10)) - mean) / std);
}
frames++;
Interlocked.Increment(ref sequence); // even: snapshot available.
}
internal bool TrySnapshot(float[] target, out long frame)
{
// Seqlock: only the worker retries; audio producer is never locked or blocked.
frame = 0;
for (int attempt = 0; attempt < 3; attempt++)
{
int before = Volatile.Read(ref sequence);
if ((before & 1) != 0) continue;
long count = Interlocked.Read(ref frames);
if (count < FrameCount) return false;
int start = (int)(count % FrameCount);
for (int m = 0; m < MelBands; m++)
{
Array.Copy(ring, m * FrameCount + start, target, m * FrameCount, FrameCount - start);
Array.Copy(ring, m * FrameCount, target, m * FrameCount + FrameCount - start, start);
}
Thread.MemoryBarrier();
if (before == Volatile.Read(ref sequence)) { frame = count; return true; }
}
return false;
}
}
// Diagnostic input bypasses WASAPI, but uses the SAME frontend/session worker.
// --rate reinterprets samples (no resampling), taking the first 300*hop samples.
internal static int VerifyWav(string path, int forcedRate, string dump, int port)
{
try
{
float[] pcm = ReadWav(path, out int rate);
if (forcedRate != 0) rate = forcedRate;
int count = (rate / 100) * FrameCount;
if (pcm.Length < count) throw new InvalidDataException("Fixture too short for 300 frames at requested rate");
var provider = new GenreOnnxProvider(rate, dump);
provider.Start();
if (!provider.WaitForInitialization(30000))
{
if (!provider.Failed) provider.Disable(new TimeoutException("ONNX initialization timed out"));
return 5;
}
for (int i = 0; i < count; i++) provider.AddSample(pcm[i]);
var clock = Stopwatch.StartNew();
while (!provider.Failed && (provider.GetPrediction() == null || (dump != null && !provider.DumpWritten)) && clock.ElapsedMilliseconds < 30000) Thread.Sleep(10);
if (provider.Failed) return 5;
var result = provider.GetPrediction();
if (result == null) throw new TimeoutException("ONNX verification inference timed out");
var message = new StringBuilder();
message.AppendFormat(System.Globalization.CultureInfo.InvariantCulture, "GENR:2;{0};{1:F3};onnx", result.Label, result.Confidence);
for (int i = 0; i < 5; i++) message.AppendFormat(System.Globalization.CultureInfo.InvariantCulture, ";{0}={1:F3}", Labels[i], result.Probabilities[i]);
using var udp = new UdpClient();
byte[] bytes = Encoding.UTF8.GetBytes(message.ToString());
udp.Send(bytes, bytes.Length, new IPEndPoint(IPAddress.Loopback, port));
return 0;
}
catch (Exception ex) { Console.Error.WriteLine("ONNX verification rejected: " + ex.Message); return 5; }
}
static float[] ReadWav(string path, out int rate)
{
using var reader = new BinaryReader(File.OpenRead(path));
if (new string(reader.ReadChars(4)) != "RIFF") throw new InvalidDataException("Expected RIFF WAV");
reader.ReadUInt32();
if (new string(reader.ReadChars(4)) != "WAVE") throw new InvalidDataException("Expected WAVE");
int format = 0, channels = 0, bits = 0, align = 0; rate = 0; byte[] data = null;
while (reader.BaseStream.Position + 8 <= reader.BaseStream.Length)
{
string id = new string(reader.ReadChars(4)); uint size = reader.ReadUInt32();
long next = reader.BaseStream.Position + size + (size & 1);
if (next > reader.BaseStream.Length + 1) throw new InvalidDataException("Truncated WAV");
if (id == "fmt ")
{
if (size < 16) throw new InvalidDataException("Invalid WAV fmt");
format = reader.ReadUInt16(); channels = reader.ReadUInt16(); rate = reader.ReadInt32();
reader.ReadUInt32(); align = reader.ReadUInt16(); bits = reader.ReadUInt16();
if (format == 65534 && size >= 40) { reader.ReadUInt16(); reader.ReadUInt16(); reader.ReadUInt32(); format = reader.ReadUInt16(); }
}
else if (id == "data") data = reader.ReadBytes(checked((int)size));
reader.BaseStream.Position = next;
}
if (data == null || channels < 1 || (format != 1 && format != 3) || align != channels * (bits / 8) || align == 0 || data.Length % align != 0)
throw new InvalidDataException("Unsupported WAV format");
var samples = new float[data.Length / align];
for (int i = 0; i < samples.Length; i++)
{
double sum = 0;
for (int c = 0; c < channels; c++)
{
int offset = i * align + c * (bits / 8);
if (format == 3 && bits == 32) sum += BitConverter.ToSingle(data, offset);
else if (format == 3 && bits == 64) sum += BitConverter.ToDouble(data, offset);
else if (format == 1 && bits == 8) sum += (data[offset] - 128) / 128.0;
else if (format == 1 && bits == 16) sum += BitConverter.ToInt16(data, offset) / 32768.0;
else if (format == 1 && bits == 24)
{ int v = data[offset] | data[offset + 1] << 8 | data[offset + 2] << 16; if ((v & 0x800000) != 0) v |= unchecked((int)0xff000000); sum += v / 8388608.0; }
else if (format == 1 && bits == 32) sum += BitConverter.ToInt32(data, offset) / 2147483648.0;
else throw new InvalidDataException("Unsupported WAV sample encoding");
}
samples[i] = (float)(sum / channels);
}
return samples;
}
}
}
bin/LoopbackCapture.cs
+484 -57
View File
@@ -15,6 +15,145 @@ using System.Threading;
namespace LoopbackCapture namespace LoopbackCapture
{ {
static class GenreSources
{
// Sorgente della classificazione di genere.
// Soluzione 1: metadati del player (GSMTC) + lookup online.
// Soluzione 2: modello ONNX locale (fase 2; se il modello manca si ricade sulle regole).
internal const bool EnableMetadataGenre = true;
internal const bool EnableOnnxGenre = false;
// Ancoraggio del BPM rilevato al tempo fornito dai metadati del player.
internal const bool EnableMetadataBpm = true;
static GenreMetadataProvider metadata;
static GenreOnnxProvider onnx;
internal static void InitializeOnnx(int rate, string dump)
{ InitializeOnnxIfEnabled(EnableOnnxGenre, rate, dump); }
static void InitializeOnnxIfEnabled(bool enabled, int rate, string dump)
{
if (!enabled) return;
onnx = new GenreOnnxProvider(rate, dump);
onnx.Start();
}
internal static void AddOnnxSample(float sample) { onnx?.AddSample(sample); }
internal static void Start() { StartMetadata(EnableMetadataGenre || EnableMetadataBpm); }
static void StartMetadata(bool enabled)
{
try
{
if (!enabled) return;
// wantGenre tells the provider whether iTunes is needed at all; with
// EnableMetadataGenre=false it only resolves the BPM (Deezer), no iTunes.
metadata = new GenreMetadataProvider(EnableMetadataGenre);
metadata.Start();
}
catch { metadata = null; }
}
internal static GenrePrediction GetPrediction(out string source)
{ return Select(EnableMetadataGenre, EnableOnnxGenre, out source); }
static GenrePrediction Select(bool useMetadata, bool useOnnx, out string source)
{
source = "rules";
try
{
string label = useMetadata ? metadata?.GetLabel() : null;
if (label != null)
{
int index = Array.IndexOf(GenreOnnxProvider.Labels, label);
if (index >= 0)
{
var probabilities = new float[5]; probabilities[index] = 1;
source = "metadata";
return new GenrePrediction(probabilities, source, Environment.TickCount64 + 10000, metadata?.GetDetail());
}
}
if (useOnnx)
{
GenrePrediction result = onnx?.GetPrediction();
if (result != null) { source = "onnx"; return result; }
}
}
catch { }
source = "rules";
return null;
}
// Ancoraggio BPM: inoltra al provider di metadati. Restituisce false se il flag
// e' disabilitato, se manca uno snapshot valido o se il bpm non e' disponibile (0).
internal static bool TryGetMetadataBpm(out float bpm)
{ return TryGetMetadataBpm(EnableMetadataBpm, out bpm); }
static bool TryGetMetadataBpm(bool enabled, out float bpm)
{
bpm = 0;
if (!enabled) return false;
try { return metadata?.TryGetBpm(out bpm) ?? false; }
catch { return false; }
}
// Raw metadata BPM for the DIAG "anchor" field, independent of EnableMetadataBpm:
// it reports what the metadata source would provide even when anchoring is off.
static float GetMetadataBpmForDiag()
{
try
{
float bpm;
return metadata != null && metadata.TryGetBpm(out bpm) ? bpm : 0;
}
catch { return 0; }
}
static string SanitizeDiagTrack(string value, int maxLength)
{
if (string.IsNullOrEmpty(value)) return "";
StringBuilder text = new StringBuilder(Math.Min(value.Length, maxLength));
foreach (char c in value)
{
if (text.Length >= maxLength) break;
if (c == ';' || c == '=' || c == '\r' || c == '\n') text.Append(' ');
else text.Append(c);
}
return text.ToString().Trim();
}
// Assemble the DIAG: packet from per-source state. Called on the capture thread
// every 5 s; reads only atomic string/number fields (no locks in the hot path).
internal static string BuildDiagPacket(float detectedBpm)
{
string meta, itunes, deezer, track;
if (metadata == null)
{
// Provider not started (both EnableMetadataGenre and EnableMetadataBpm off).
meta = "off"; itunes = "off"; deezer = "off"; track = "";
}
else
{
meta = metadata.MetaStatus ?? "error";
itunes = metadata.ItunesStatus ?? "off";
deezer = metadata.DeezerStatus ?? "off";
track = metadata.TrackText ?? "";
}
string onnxStatus = onnx == null ? "off" : (onnx.Status ?? "failed");
track = SanitizeDiagTrack(track, 80);
float anchor = GetMetadataBpmForDiag();
return "DIAG:1" +
";meta=" + meta +
";track=" + track +
";itunes=" + itunes +
";deezer=" + deezer +
";onnx=" + onnxStatus +
";bpm=" + detectedBpm.ToString("F1", CultureInfo.InvariantCulture) +
";anchor=" + anchor.ToString("F1", CultureInfo.InvariantCulture);
}
}
// Independent analysis branch: never writes to RMS/onset/BPM detector state. // Independent analysis branch: never writes to RMS/onset/BPM detector state.
sealed class RuleGenreAnalyzer sealed class RuleGenreAnalyzer
{ {
@@ -227,8 +366,18 @@ namespace LoopbackCapture
static double Tempo(double bpm, double min, double max) static double Tempo(double bpm, double min, double max)
{ return bpm <= 0 ? 0 : Clamp(1 - Math.Max(min - bpm, Math.Max(0, bpm - max)) / TempoShoulderBpm); } { return bpm <= 0 ? 0 : Clamp(1 - Math.Max(min - bpm, Math.Max(0, bpm - max)) / TempoShoulderBpm); }
void RhythmFeatures(double bpm) static string SanitizeDetail(string value)
{ {
StringBuilder text = new StringBuilder(value.Length);
foreach (char c in value)
{
if (c == ';' || c == '=' || c == '\r' || c == '\n') text.Append(' ');
else text.Append(c);
}
return text.ToString().Trim();
}
void RhythmFeatures(double bpm) {
double maximum = PowerEpsilon; double maximum = PowerEpsilon;
for (int i = 0; i < shortCount; i++) for (int i = 0; i < shortCount; i++)
{ {
@@ -367,9 +516,18 @@ namespace LoopbackCapture
int best = 0; int best = 0;
for (int c = 1; c < Labels.Length; c++) if (probabilities[c] > probabilities[best]) best = c; for (int c = 1; c < Labels.Length; c++) if (probabilities[c] > probabilities[best]) best = c;
string label = probabilities[best] < ConfidenceThreshold ? "unknown" : Labels[best]; string label = probabilities[best] < ConfidenceThreshold ? "unknown" : Labels[best];
// Rules above ALWAYS run. Override only the wire output, not their history/dump.
string source = "rules";
GenrePrediction preferred = null;
try { preferred = GenreSources.GetPrediction(out source); }
catch { source = "rules"; }
StringBuilder packet = new StringBuilder(); StringBuilder packet = new StringBuilder();
packet.AppendFormat(CultureInfo.InvariantCulture, "GENR:1;{0};{1:F3}", label, probabilities[best]); packet.AppendFormat(CultureInfo.InvariantCulture, "GENR:2;{0};{1:F3};{2}", preferred?.Label ?? label, preferred != null ? preferred.Confidence : probabilities[best], source);
for (int c = 0; c < Labels.Length; c++) packet.AppendFormat(CultureInfo.InvariantCulture, ";{0}={1:F3}", Labels[c], probabilities[c]); // Genere specifico del catalogo: descrittivo, non partecipa alle probabilita'.
// ';' e '=' sono delimitatori del protocollo: sostituiti prima dell'invio.
if (preferred != null && !string.IsNullOrEmpty(preferred.Detail))
packet.Append(";detail=").Append(SanitizeDetail(preferred.Detail));
for (int c = 0; c < Labels.Length; c++) packet.AppendFormat(CultureInfo.InvariantCulture, ";{0}={1:F3}", Labels[c], preferred == null ? probabilities[c] : preferred.Probabilities[c]);
try { byte[] bytes = Encoding.UTF8.GetBytes(packet.ToString()); udp.Send(bytes, bytes.Length, destination); } catch { } try { byte[] bytes = Encoding.UTF8.GetBytes(packet.ToString()); udp.Send(bytes, bytes.Length, destination); } catch { }
if (dump != null) if (dump != null)
{ {
@@ -402,6 +560,202 @@ namespace LoopbackCapture
static ManualResetEvent cmdReady = new ManualResetEvent(false); static ManualResetEvent cmdReady = new ManualResetEvent(false);
static long leaseDeadlineMs; static long leaseDeadlineMs;
// Ancoraggio del BPM rilevato al tempo fornito dai metadati. Metodo puro e
// riusabile: usato dal loop di cattura e da --check-anchor.
// Il rilevatore puo' leggere 4/3 o 4/5 del tempo reale su pattern 3+3+2, e puo'
// sbagliare l'ottava (2x / 1/2x). L'ottava e' decisa dall'energia percussiva e
// quindi e' approssimativa (tolleranza 4%), mentre i rapporti 3+3+2 sono
// deterministici (tolleranza 2%): cosi' un tempo vicino ma non derivato dai
// metadati (es. 130 vs 4/3 * 95 = 126.7) non viene toccato.
static int AnchorBpm(double detected, float metadataBpm)
{
if (metadataBpm <= 0) return (int)Math.Round(detected);
double a = metadataBpm;
double best = detected, bestTarget = best, bestDist = double.MaxValue;
// (valore che il rilevatore puo' leggere, valore corretto, tolleranza)
var relations = new (double value, double target, double tol)[]
{
(a, a, 0.04),
(2 * a, 2 * a, 0.04),
(a / 2, a / 2, 0.04),
(a * 4.0 / 3.0, a, 0.02),
(a * 4.0 / 5.0, a, 0.02),
};
foreach (var r in relations)
{
double rel = Math.Abs(best - r.value) / r.value;
if (rel <= r.tol && rel < bestDist)
{
bestDist = rel;
bestTarget = r.target;
}
}
return (int)Math.Round(bestTarget);
}
static void CheckAnchor(string spec)
{
string[] parts = spec.Split(',');
if (parts.Length < 2 ||
!float.TryParse(parts[0], NumberStyles.Float, CultureInfo.InvariantCulture, out float anchor))
{
Console.Error.WriteLine("--check-anchor richiede <anchor>,<detected1>,... (numeri)");
return;
}
for (int i = 1; i < parts.Length; i++)
{
if (!float.TryParse(parts[i], NumberStyles.Float, CultureInfo.InvariantCulture, out float detected))
{
Console.Error.WriteLine("--check-anchor: valore non valido: " + parts[i]);
continue;
}
int anchored = AnchorBpm(detected, anchor);
Console.WriteLine(string.Format(CultureInfo.InvariantCulture, "{0:F1} -> {1}", detected, anchored));
}
}
// Diagnostics for --check-sources: one attempt per source, readable report, exit 0.
// Works regardless of EnableMetadataGenre/EnableOnnxGenre, like --check-anchor for
// the pure function; the verdict reports what would win with the current config.
static void CheckSources(string spec)
{
try
{
// 1. Model file presence (same path the ONNX worker resolves).
string modelPath = Path.Combine(AppContext.BaseDirectory, "genre_model.onnx");
string jsonPath = Path.Combine(AppContext.BaseDirectory, "genre_model.json");
bool modelExists = File.Exists(modelPath) && File.Exists(jsonPath);
Console.WriteLine("[MODEL] genre_model.onnx trovato=" + (modelExists ? "si" : "no") +
(modelExists ? "" : " -> ONNX non disponibile"));
// 2. ONNX load attempt (only if the files are present).
bool onnxLoaded = false;
if (!modelExists)
{
Console.WriteLine("[ONNX] caricamento: saltato (modello assente)");
}
else
{
try
{
var provider = new GenreOnnxProvider(44100, null);
provider.Start();
if (provider.WaitForInitialization(30000))
{
onnxLoaded = true;
Console.WriteLine("[ONNX] caricamento: ok");
}
else
{
string reason = provider.FailureReason;
Console.WriteLine("[ONNX] caricamento: fallito (" + (string.IsNullOrEmpty(reason) ? provider.Status : reason) + ")");
}
}
catch (Exception ex)
{
Console.WriteLine("[ONNX] caricamento: fallito (" + ex.Message + ")");
}
}
// 3. GSMTC / test track.
string artist, title;
bool gsmPlaying = false;
if (!string.IsNullOrEmpty(spec))
{
int sep = spec.IndexOf('|');
if (sep < 0) { artist = spec.Trim(); title = ""; }
else { artist = spec.Substring(0, sep).Trim(); title = spec.Substring(sep + 1).Trim(); }
gsmPlaying = ReadGsmcForReport(out string gArtist, out string gTitle);
string gsmText = gsmPlaying ? "si (\"" + gArtist + " - " + gTitle + "\")" : "no";
Console.WriteLine("[GSMTC] sessione in riproduzione: " + gsmText + " [test: \"" + artist + " - " + title + "\"]");
}
else
{
gsmPlaying = ReadGsmcForReport(out artist, out title);
if (gsmPlaying)
Console.WriteLine("[GSMTC] sessione in riproduzione: si (\"" + artist + " - " + title + "\")");
else
{
artist = "Daft Punk"; title = "Get Lucky";
Console.WriteLine("[GSMTC] sessione in riproduzione: no -> uso test \"Daft Punk - Get Lucky\"");
}
}
if (artist.Length == 0 && title.Length == 0)
{
Console.WriteLine("[ITUNES] saltato: nessun artista/titolo");
Console.WriteLine("[DEEZER] saltato: nessun artista/titolo");
Console.WriteLine("[VERDETTO] sorgente che vincerebbe: rules (nessun brano)");
return;
}
// 4. iTunes.
var (label, detail, itunesState, itunesReason) = GenreMetadataProvider.LookupItunesDetailed(artist, title);
if (itunesState == "ok")
Console.WriteLine("[ITUNES] risposta ok, match=si, genere=\"" + (detail ?? "") + "\" -> " + label);
else if (itunesState == "no-match")
Console.WriteLine("[ITUNES] risposta ok, match=no -> " + itunesReason);
else
Console.WriteLine("[ITUNES] errore: " + itunesReason);
// 5. Deezer.
var (bpm, deezerState, deezerReason) = GenreMetadataProvider.LookupDeezerDetailed(artist, title);
if (deezerState == "ok")
Console.WriteLine("[DEEZER] risposta ok, match=si, bpm=" + bpm.ToString("F2", CultureInfo.InvariantCulture));
else if (deezerState == "no-bpm")
Console.WriteLine("[DEEZER] risposta ok, match=si, bpm assente (" + deezerReason + ")");
else if (deezerState == "no-match")
Console.WriteLine("[DEEZER] risposta ok, match=no -> " + deezerReason);
else
Console.WriteLine("[DEEZER] errore: " + deezerReason);
// 6. Provider startup mode under the current configuration.
bool metadataEnabled = GenreSources.EnableMetadataGenre;
bool metadataBpmEnabled = GenreSources.EnableMetadataBpm;
bool onnxEnabled = GenreSources.EnableOnnxGenre;
string metaMode;
if (!metadataEnabled && !metadataBpmEnabled)
metaMode = "off (provider non avviato: genere e BPM disabilitati)";
else if (!metadataEnabled)
metaMode = "bpm-only (provider avviato per solo BPM)";
else
metaMode = "on (genere" + (metadataBpmEnabled ? " + BPM" : "") + ")";
Console.WriteLine("[META] provider: " + metaMode);
// 7. Verdict under the current configuration.
bool metadataWins = metadataEnabled && itunesState == "ok";
bool onnxWins = onnxEnabled && onnxLoaded;
string winner = metadataWins ? "metadata" : onnxWins ? "onnx" : "rules";
var reasons = new List<string>();
if (!metadataWins)
{
if (!metadataEnabled)
reasons.Add(metadataBpmEnabled ? "metadati solo BPM (genere disabilitato)" : "metadati disabilitati");
else if (itunesState == "no-match") reasons.Add("nessun match iTunes");
else reasons.Add("iTunes non disponibile");
}
if (!onnxWins)
{
if (!onnxEnabled) reasons.Add("ONNX disabilitato");
else if (!modelExists) reasons.Add("ONNX assente");
else reasons.Add("ONNX non caricabile");
}
string reasonText = reasons.Count > 0 ? string.Join(", ", reasons) : "";
Console.WriteLine("[VERDETTO] sorgente che vincerebbe: " + winner + (reasonText.Length > 0 ? " (" + reasonText + ")" : ""));
}
catch (Exception ex)
{
Console.WriteLine("[VERDETTO] errore imprevisto nel check: " + ex.Message);
}
}
static bool ReadGsmcForReport(out string artist, out string title)
{
var (a, t) = GenreMetadataProvider.ReadCurrentTrack();
artist = a; title = t;
return a != null && t != null;
}
static void Main(string[] args) static void Main(string[] args)
{ {
try try
@@ -420,6 +774,44 @@ namespace LoopbackCapture
Stopwatch lifecycleClock = Stopwatch.StartNew(); Stopwatch lifecycleClock = Stopwatch.StartNew();
int leaseMs = 0; int leaseMs = 0;
string dumpPath = null; string dumpPath = null;
string tensorDump = null, verifyWav = null;
int verifyRate = 0;
for (int i = 0; i < args.Length; i++)
{
if (args[i].StartsWith("--dump-tensor=", StringComparison.Ordinal)) tensorDump = args[i].Substring(14);
if (args[i].StartsWith("--verify-wav=", StringComparison.Ordinal)) verifyWav = args[i].Substring(13);
if (args[i] == "--rate")
{
if (++i >= args.Length || !int.TryParse(args[i], out verifyRate) || (verifyRate != 44100 && verifyRate != 48000))
throw new ArgumentException("--rate requires 44100 or 48000 (sample reinterpretation, no resampling)");
}
}
for (int i = 0; i < args.Length; i++)
{
if (args[i].StartsWith("--check-anchor=", StringComparison.Ordinal))
{
CheckAnchor(args[i].Substring(15));
Environment.Exit(0);
}
}
for (int i = 0; i < args.Length; i++)
{
if (args[i] == "--check-sources")
{
CheckSources(null);
Environment.Exit(0);
}
if (args[i].StartsWith("--check-sources=", StringComparison.Ordinal))
{
CheckSources(args[i].Substring(16));
Environment.Exit(0);
}
}
bool onnxEnabled = GenreSources.EnableOnnxGenre;
if ((tensorDump != null || verifyWav != null) && !onnxEnabled)
throw new InvalidOperationException("ONNX tensor dump/verification rejected: EnableOnnxGenre=false; frontend not initialized");
if (tensorDump != null && string.IsNullOrWhiteSpace(tensorDump)) throw new ArgumentException("--dump-tensor requires a file path");
if (verifyRate != 0 && verifyWav == null) throw new ArgumentException("--rate is only valid with --verify-wav");
foreach (string arg in args) foreach (string arg in args)
{ {
if (arg.StartsWith("--dump-features=", StringComparison.Ordinal)) if (arg.StartsWith("--dump-features=", StringComparison.Ordinal))
@@ -434,12 +826,18 @@ namespace LoopbackCapture
Interlocked.Exchange(ref leaseDeadlineMs, lifecycleClock.ElapsedMilliseconds + leaseMs); Interlocked.Exchange(ref leaseDeadlineMs, lifecycleClock.ElapsedMilliseconds + leaseMs);
int port = 4242; int port = 4242;
int cmdPort = 4243; int cmdPort = 4243;
if (args.Length > 0) if (args.Length > 0 && (verifyWav == null || !args[0].StartsWith("--", StringComparison.Ordinal)))
{ {
int.TryParse(args[0], out port); int.TryParse(args[0], out port);
cmdPort = port + 1; cmdPort = port + 1;
} }
if (verifyWav != null)
{
Environment.Exit(GenreOnnxProvider.VerifyWav(verifyWav, verifyRate, tensorDump, port));
return;
}
int parentPid = 0; int parentPid = 0;
if (args.Length > 1) int.TryParse(args[1], out parentPid); if (args.Length > 1) int.TryParse(args[1], out parentPid);
if (parentPid > 0) if (parentPid > 0)
@@ -468,6 +866,7 @@ namespace LoopbackCapture
Environment.Exit(3); Environment.Exit(3);
} }
GC.KeepAlive(instanceMutex); GC.KeepAlive(instanceMutex);
try { GenreSources.Start(); } catch { }
UdpClient udpSend = null; UdpClient udpSend = null;
try try
@@ -599,6 +998,7 @@ namespace LoopbackCapture
int hopSize = sampleRate / 100; int hopSize = sampleRate / 100;
if (hopSize < 1) hopSize = 480; if (hopSize < 1) hopSize = 480;
RuleGenreAnalyzer genreAnalyzer = new RuleGenreAnalyzer(sampleRate, hopSize, dumpPath); RuleGenreAnalyzer genreAnalyzer = new RuleGenreAnalyzer(sampleRate, hopSize, dumpPath);
GenreSources.InitializeOnnx(sampleRate, tensorDump);
const int BUFFER_HOPS = 400; // 4.0 secondi di finestra temporale const int BUFFER_HOPS = 400; // 4.0 secondi di finestra temporale
float[] onsetBuffer = new float[BUFFER_HOPS]; float[] onsetBuffer = new float[BUFFER_HOPS];
int onsetHead = 0; int onsetHead = 0;
@@ -621,6 +1021,7 @@ namespace LoopbackCapture
long lastHeartbeat = -250; long lastHeartbeat = -250;
long heartbeatCounter = 0; long heartbeatCounter = 0;
long captureErrorSince = -1; long captureErrorSince = -1;
long lastDiagBroadcast = -5000;
while (true) while (true)
{ {
@@ -655,6 +1056,17 @@ namespace LoopbackCapture
} }
catch { } catch { }
} }
// Per-source diagnostic packet, every 5 s, same socket/destination as the rest.
if (lifecycleClock.ElapsedMilliseconds - lastDiagBroadcast >= 5000)
{
lastDiagBroadcast = lifecycleClock.ElapsedMilliseconds;
try
{
byte[] diag = Encoding.UTF8.GetBytes(GenreSources.BuildDiagPacket(detectedBpm));
udpSend.Send(diag, diag.Length, remoteEp);
}
catch { }
}
// 1. Lettura campioni PCM WASAPI dal loopback di sistema // 1. Lettura campioni PCM WASAPI dal loopback di sistema
uint packetLength = 0; uint packetLength = 0;
hr = captureClient.GetNextPacketSize(out packetLength); hr = captureClient.GetNextPacketSize(out packetLength);
@@ -702,6 +1114,7 @@ namespace LoopbackCapture
} }
genreAnalyzer.AddSample((flags & 2) != 0 ? 0f : monoSample); genreAnalyzer.AddSample((flags & 2) != 0 ? 0f : monoSample);
GenreSources.AddOnnxSample((flags & 2) != 0 ? 0f : monoSample);
totalSumSq += (monoSample * monoSample); totalSumSq += (monoSample * monoSample);
// Filtro IIR passa-basso a doppio polo per estrarre la cassa // Filtro IIR passa-basso a doppio polo per estrarre la cassa
@@ -745,7 +1158,11 @@ namespace LoopbackCapture
} }
else else
{ {
for (uint f = 0; f < numFramesRead; f++) genreAnalyzer.AddSample(0f); for (uint f = 0; f < numFramesRead; f++)
{
genreAnalyzer.AddSample(0f);
GenreSources.AddOnnxSample(0f);
}
currentTotalRms *= 0.7f; currentTotalRms *= 0.7f;
} }
@@ -895,6 +1312,16 @@ namespace LoopbackCapture
} }
} }
// Ancoraggio ai metadati: corregge gli errori 3+3+2 (4/3 e 4/5)
// e le ottave usando il tempo reale del player. Se nessun candidato
// rientra in tolleranza, bestCandidateBpm resta invariato (il metadato
// puo' essere sbagliato: remix, edit) e l'audio resta la sorgente primaria.
float anchorBpm;
if (GenreSources.TryGetMetadataBpm(out anchorBpm))
{
bestCandidateBpm = AnchorBpm(bestCandidateBpm, anchorBpm);
}
if (bestCandidateScore > 0.0005) if (bestCandidateScore > 0.0005)
{ {
if (detectedBpm <= 10.0f) if (detectedBpm <= 10.0f)
@@ -1030,63 +1457,63 @@ namespace LoopbackCapture
public enum EDataFlow { eRender, eCapture, eAll } public enum EDataFlow { eRender, eCapture, eAll }
public enum ERole { eConsole, eMultimedia, eCommunications } public enum ERole { eConsole, eMultimedia, eCommunications }
[Guid("A95664D2-9614-4F35-A746-DE8DB63617E6"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] [ComImport, Guid("A95664D2-9614-4F35-A746-DE8DB63617E6"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
public interface IMMDeviceEnumerator public interface IMMDeviceEnumerator
{ {
int EnumAudioEndpoints(EDataFlow dataFlow, int stateMask, out IntPtr devices); [PreserveSig] int EnumAudioEndpoints(EDataFlow dataFlow, int stateMask, out IntPtr devices);
int GetDefaultAudioEndpoint(EDataFlow dataFlow, ERole role, out IMMDevice endpoint); [PreserveSig] int GetDefaultAudioEndpoint(EDataFlow dataFlow, ERole role, out IMMDevice endpoint);
int GetDevice(string id, out IMMDevice device); [PreserveSig] int GetDevice([MarshalAs(UnmanagedType.LPWStr)] string id, out IMMDevice device);
int RegisterEndpointNotificationCallback(IntPtr client); [PreserveSig] int RegisterEndpointNotificationCallback(IntPtr client);
int UnregisterEndpointNotificationCallback(IntPtr client); [PreserveSig] int UnregisterEndpointNotificationCallback(IntPtr client);
} }
[Guid("D666063F-1587-4E43-81F1-B948E807363F"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] [ComImport, Guid("D666063F-1587-4E43-81F1-B948E807363F"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
public interface IMMDevice public interface IMMDevice
{ {
int Activate(ref Guid id, int clsCtx, IntPtr activationParams, [MarshalAs(UnmanagedType.IUnknown)] out object interfacePointer); [PreserveSig] int Activate(ref Guid id, int clsCtx, IntPtr activationParams, [MarshalAs(UnmanagedType.IUnknown)] out object interfacePointer);
int OpenPropertyStore(int stgmAccess, out IntPtr properties); [PreserveSig] int OpenPropertyStore(int stgmAccess, out IntPtr properties);
int GetId([MarshalAs(UnmanagedType.LPWStr)] out string id); [PreserveSig] int GetId([MarshalAs(UnmanagedType.LPWStr)] out string id);
int GetState(out int state); [PreserveSig] int GetState(out int state);
} }
[Guid("77AA99A0-1BD6-484F-8BC7-2C654C9A9B6F"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] [ComImport, Guid("77AA99A0-1BD6-484F-8BC7-2C654C9A9B6F"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
public interface IAudioSessionManager2 public interface IAudioSessionManager2
{ {
int GetAudioSessionControl(IntPtr sessionId, int priority, out IntPtr control); [PreserveSig] int GetAudioSessionControl(IntPtr sessionId, int priority, out IntPtr control);
int GetSimpleAudioVolume(IntPtr sessionId, int priority, out IntPtr volume); [PreserveSig] int GetSimpleAudioVolume(IntPtr sessionId, int priority, out IntPtr volume);
int GetSessionEnumerator(out IAudioSessionEnumerator sessionEnum); [PreserveSig] int GetSessionEnumerator(out IAudioSessionEnumerator sessionEnum);
} }
[Guid("E2F5BB11-0570-40CA-ACDD-3AA01277DEE8"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] [ComImport, Guid("E2F5BB11-0570-40CA-ACDD-3AA01277DEE8"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
public interface IAudioSessionEnumerator public interface IAudioSessionEnumerator
{ {
int GetCount(out int sessionCount); [PreserveSig] int GetCount(out int sessionCount);
int GetSession(int sessionIndex, [MarshalAs(UnmanagedType.IUnknown)] out object session); [PreserveSig] int GetSession(int sessionIndex, [MarshalAs(UnmanagedType.IUnknown)] out object session);
} }
[Guid("bfb7ff88-7239-4fc9-8fa2-07c950be9c6d"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] [ComImport, Guid("bfb7ff88-7239-4fc9-8fa2-07c950be9c6d"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
public interface IAudioSessionControl2 public interface IAudioSessionControl2
{ {
int GetState(out int state); [PreserveSig] int GetState(out int state);
int GetDisplayName(out IntPtr name); [PreserveSig] int GetDisplayName(out IntPtr name);
int SetDisplayName(string value, IntPtr eventContext); [PreserveSig] int SetDisplayName([MarshalAs(UnmanagedType.LPWStr)] string value, IntPtr eventContext);
int GetIconPath(out IntPtr path); [PreserveSig] int GetIconPath(out IntPtr path);
int SetIconPath(string value, IntPtr eventContext); [PreserveSig] int SetIconPath([MarshalAs(UnmanagedType.LPWStr)] string value, IntPtr eventContext);
int GetGroupingParam(out Guid grouping); [PreserveSig] int GetGroupingParam(out Guid grouping);
int SetGroupingParam(ref Guid grouping, IntPtr eventContext); [PreserveSig] int SetGroupingParam(ref Guid grouping, IntPtr eventContext);
int RegisterAudioSessionNotification(IntPtr client); [PreserveSig] int RegisterAudioSessionNotification(IntPtr client);
int UnregisterAudioSessionNotification(IntPtr client); [PreserveSig] int UnregisterAudioSessionNotification(IntPtr client);
int GetSessionIdentifier(out IntPtr id); [PreserveSig] int GetSessionIdentifier(out IntPtr id);
int GetSessionInstanceIdentifier(out IntPtr id); [PreserveSig] int GetSessionInstanceIdentifier(out IntPtr id);
int GetProcessId(out uint processId); [PreserveSig] int GetProcessId(out uint processId);
int IsSystemSoundsSession(); [PreserveSig] int IsSystemSoundsSession();
int SetDuckingPreference(bool optOut); [PreserveSig] int SetDuckingPreference([MarshalAs(UnmanagedType.Bool)] bool optOut);
} }
[Guid("C02216F6-8C67-4B5B-9D00-D008E73E0064"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] [ComImport, Guid("C02216F6-8C67-4B5B-9D00-D008E73E0064"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
public interface IAudioMeterInformation public interface IAudioMeterInformation
{ {
int GetPeakValue(out float peak); [PreserveSig] int GetPeakValue(out float peak);
} }
[StructLayout(LayoutKind.Sequential, Pack = 1)] [StructLayout(LayoutKind.Sequential, Pack = 1)]
@@ -1110,28 +1537,28 @@ namespace LoopbackCapture
public Guid SubFormat; public Guid SubFormat;
} }
[Guid("1CB9AD4C-DBFA-4c32-B178-C2F568A703B2"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] [ComImport, Guid("1CB9AD4C-DBFA-4c32-B178-C2F568A703B2"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
public interface IAudioClient public interface IAudioClient
{ {
int Initialize(int shareMode, int streamFlags, long hnsBufferDuration, long hnsPeriodicity, IntPtr pFormat, ref Guid audioSessionGuid); [PreserveSig] int Initialize(int shareMode, int streamFlags, long hnsBufferDuration, long hnsPeriodicity, IntPtr pFormat, ref Guid audioSessionGuid);
int GetBufferSize(out uint bufferFrameCount); [PreserveSig] int GetBufferSize(out uint bufferFrameCount);
int GetStreamLatency(out long hnsLatency); [PreserveSig] int GetStreamLatency(out long hnsLatency);
int GetCurrentPadding(out uint numPaddingFrames); [PreserveSig] int GetCurrentPadding(out uint numPaddingFrames);
int IsFormatSupported(int shareMode, IntPtr pFormat, out IntPtr ppClosestMatch); [PreserveSig] int IsFormatSupported(int shareMode, IntPtr pFormat, out IntPtr ppClosestMatch);
int GetMixFormat(out IntPtr ppDeviceFormat); [PreserveSig] int GetMixFormat(out IntPtr ppDeviceFormat);
int GetDevicePeriod(out long hnsDefaultDevicePeriod, out long hnsMinimumDevicePeriod); [PreserveSig] int GetDevicePeriod(out long hnsDefaultDevicePeriod, out long hnsMinimumDevicePeriod);
int Start(); [PreserveSig] int Start();
int Stop(); [PreserveSig] int Stop();
int Reset(); [PreserveSig] int Reset();
int SetEventHandle(IntPtr eventHandle); [PreserveSig] int SetEventHandle(IntPtr eventHandle);
int GetService(ref Guid interfaceId, [MarshalAs(UnmanagedType.IUnknown)] out object interfacePointer); [PreserveSig] int GetService(ref Guid interfaceId, [MarshalAs(UnmanagedType.IUnknown)] out object interfacePointer);
} }
[Guid("C8ADBD64-E71E-48a0-A4DE-185C395CD317"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)] [ComImport, Guid("C8ADBD64-E71E-48a0-A4DE-185C395CD317"), InterfaceType(ComInterfaceType.InterfaceIsIUnknown)]
public interface IAudioCaptureClient public interface IAudioCaptureClient
{ {
int GetBuffer(out IntPtr dataPointer, out uint numFramesToRead, out int bufferFlags, out ulong devicePosition, out ulong qpcPosition); [PreserveSig] int GetBuffer(out IntPtr dataPointer, out uint numFramesToRead, out int bufferFlags, out ulong devicePosition, out ulong qpcPosition);
int ReleaseBuffer(uint numFramesRead); [PreserveSig] int ReleaseBuffer(uint numFramesRead);
int GetNextPacketSize(out uint numFramesInNextPacket); [PreserveSig] int GetNextPacketSize(out uint numFramesInNextPacket);
} }
} }
bin/LoopbackCapture.csproj
+23
View File
@@ -0,0 +1,23 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<TargetFramework>net8.0-windows10.0.19041.0</TargetFramework>
<OutputType>WinExe</OutputType>
<AllowUnsafeBlocks>true</AllowUnsafeBlocks>
<Nullable>disable</Nullable>
<ImplicitUsings>disable</ImplicitUsings>
<LangVersion>latest</LangVersion>
<RuntimeIdentifier>win-x64</RuntimeIdentifier>
<SelfContained>true</SelfContained>
<PublishSingleFile>true</PublishSingleFile>
<IncludeNativeLibrariesForSelfExtract>true</IncludeNativeLibrariesForSelfExtract>
<InvariantGlobalization>true</InvariantGlobalization>
<EnableDefaultCompileItems>false</EnableDefaultCompileItems>
<DebugType>none</DebugType>
</PropertyGroup>
<ItemGroup>
<Compile Include="LoopbackCapture.cs" />
<Compile Include="GenreMetadataProvider.cs" />
<Compile Include="GenreOnnxProvider.cs" />
<PackageReference Include="Microsoft.ML.OnnxRuntime" Version="1.20.1" />
</ItemGroup>
</Project>
bin/LoopbackCapture.exe
BIN
View File
Binary file not shown.
bin/build.ps1
+34 -9
View File
@@ -1,10 +1,35 @@
$csc = "C:\Windows\Microsoft.NET\Framework64\v4.0.30319\csc.exe" $ErrorActionPreference = 'Stop'
if (-not (Test-Path $csc)) { $work = Join-Path ([System.IO.Path]::GetTempPath()) ('ldcc-publish-' + [Guid]::NewGuid().ToString('N'))
$csc = "C:\Windows\Microsoft.NET\Framework\v4.0.30319\csc.exe" $exitCode = 1
try {
if (-not (Get-Command dotnet -ErrorAction SilentlyContinue)) {
throw 'dotnet SDK non trovato. Installare un SDK compatibile con .NET 8.'
}
$publish = Join-Path $work 'publish'
$obj = (Join-Path $work 'obj') + '\'
$output = (Join-Path $work 'build') + '\'
& dotnet publish (Join-Path $PSScriptRoot 'LoopbackCapture.csproj') -c Release -o $publish "-p:BaseIntermediateOutputPath=$obj" "-p:BaseOutputPath=$output"
$exitCode = $LASTEXITCODE
if ($exitCode -eq 0) {
try {
$built = Get-Item (Join-Path $publish 'LoopbackCapture.exe')
# Guardia: un self-contained single-file pesa decine di MB. Poche centinaia di
# KB significano publish framework-dependent, che su un PC senza runtime .NET
# non parte. Meglio fallire qui che distribuire un exe che non si avvia.
if ($built.Length -lt 20MB) {
throw "publish inatteso: $($built.Length) byte. Atteso self-contained (>20 MB): controllare SelfContained/RuntimeIdentifier/PublishSingleFile in LoopbackCapture.csproj."
}
Copy-Item $built.FullName (Join-Path $PSScriptRoot 'LoopbackCapture.exe') -Force
Write-Output ("Build completata: bin\LoopbackCapture.exe ({0:N0} byte, self-contained .NET 8)." -f $built.Length)
} catch {
$exitCode = 1
Write-Error "Impossibile aggiornare bin\LoopbackCapture.exe: file in uso o accesso negato. Chiudere il bridge e ripetere la build. $($_.Exception.Message)" -ErrorAction Continue
}
}
} catch {
$exitCode = 1
Write-Error $_ -ErrorAction Continue
} finally {
if (Test-Path $work) { Remove-Item $work -Recurse -Force -ErrorAction SilentlyContinue }
} }
if (-not (Test-Path $csc)) { exit $exitCode
Write-Error "Compilatore .NET Framework csc.exe non trovato (Framework64 / Framework)."
exit 1
}
& $csc /nologo /unsafe /optimize+ /target:exe /out:bin\LoopbackCapture.exe bin\LoopbackCapture.cs
exit $LASTEXITCODE
bin/genre_model.json
+26
View File
@@ -0,0 +1,26 @@
{
"labels": [
"latin",
"electronic",
"hiphop",
"rock",
"classical"
],
"mean": -4.0,
"std": 4.0,
"version": 1,
"source": "designed",
"frontend": {
"fft": 2048,
"hann": "periodic",
"hop": "sr//100",
"padding": "causal-left-N-hop",
"power": "abs(rfft)^2",
"mel": "HTK",
"bands": 128,
"fmin": 30,
"fmax": 11000,
"area": "divide by full support width in fractional bins",
"log_floor": 1e-10
}
}
bin/genre_model.onnx
BIN
View File
Binary file not shown.
models/designed/genre_model.json
+26
View File
@@ -0,0 +1,26 @@
{
"labels": [
"latin",
"electronic",
"hiphop",
"rock",
"classical"
],
"mean": -4.0,
"std": 4.0,
"version": 1,
"source": "designed",
"frontend": {
"fft": 2048,
"hann": "periodic",
"hop": "sr//100",
"padding": "causal-left-N-hop",
"power": "abs(rfft)^2",
"mel": "HTK",
"bands": 128,
"fmin": 30,
"fmax": 11000,
"area": "divide by full support width in fractional bins",
"log_floor": 1e-10
}
}
models/designed/genre_model.onnx
BIN
View File
Binary file not shown.
scripts/CompanionController.gd
+10 -2
View File
@@ -29,6 +29,8 @@ var _silence_timer: float = 0.0
var _current_rms: float = 0.0 var _current_rms: float = 0.0
var _current_genre: String = "" var _current_genre: String = ""
var _current_genre_conf: float = 0.0 var _current_genre_conf: float = 0.0
var _current_genre_source: String = "rules"
var _current_genre_detail: String = ""
# BPM dinamico: parte da 0 finché la canzone non viene agganciata # BPM dinamico: parte da 0 finché la canzone non viene agganciata
var _current_bpm: float = 0.0 var _current_bpm: float = 0.0
@@ -59,9 +61,11 @@ func _ready() -> void:
_set_state(State.IDLE) _set_state(State.IDLE)
func _on_genre_updated(genre: String, confidence: float, _probs: Dictionary) -> void: func _on_genre_updated(genre: String, confidence: float, _probs: Dictionary, source: String, detail: String) -> void:
_current_genre = genre _current_genre = genre
_current_genre_conf = confidence _current_genre_conf = confidence
_current_genre_source = source
_current_genre_detail = detail
func _on_applications_updated(apps: Array[Dictionary]) -> void: func _on_applications_updated(apps: Array[Dictionary]) -> void:
if not app_selector: if not app_selector:
@@ -216,7 +220,11 @@ func _update_debug_ui() -> void:
var genre_str: String = "Genere: --" var genre_str: String = "Genere: --"
if not _current_genre.is_empty() and _current_genre != "unknown": if not _current_genre.is_empty() and _current_genre != "unknown":
genre_str = "%s (%d%%)" % [_current_genre, roundi(_current_genre_conf * 100.0)] # Con i metadati mostriamo il genere specifico del catalogo ("Hard Rock"),
# che e' piu' informativo della classe grossolana usata dal gameplay.
var shown: String = _current_genre_detail if not _current_genre_detail.is_empty() else _current_genre
var source_label: String = "meta" if _current_genre_source == "metadata" else _current_genre_source
genre_str = "%s (%d%%) [%s]" % [shown, roundi(_current_genre_conf * 100.0), source_label]
debug_label.text = "Stato: %s | Volume: %.2f | %s | %s" % [ debug_label.text = "Stato: %s | Volume: %.2f | %s | %s" % [
State.keys()[current_state], State.keys()[current_state],
_current_rms, _current_rms,
scripts/SystemAudioAnalyzer.gd
+86 -10
View File
@@ -7,13 +7,14 @@ signal volume_updated(rms_value: float)
signal bpm_updated(bpm: float) signal bpm_updated(bpm: float)
## Emette la lista delle applicazioni audio attive (es. Spotify, Chrome, Discord) ## Emette la lista delle applicazioni audio attive (es. Spotify, Chrome, Discord)
signal applications_updated(apps: Array[Dictionary]) signal applications_updated(apps: Array[Dictionary])
signal genre_updated(genre: String, confidence: float, probs: Dictionary) signal genre_updated(genre: String, confidence: float, probs: Dictionary, source: String, detail: String)
@export var udp_port: int = 4242 @export var udp_port: int = 4242
@export var cmd_port: int = 4243 @export var cmd_port: int = 4243
@export var heartbeat_timeout_ms: int = 1500 @export var heartbeat_timeout_ms: int = 1500
@export var lease_enabled: bool = true @export var lease_enabled: bool = true
@export var genre_enabled: bool = true @export var genre_enabled: bool = true
@export var diag_enabled: bool = true
const LEASE_MS: int = 10000 const LEASE_MS: int = 10000
const GENRE_LABELS = ["latin", "electronic", "hiphop", "rock", "classical"] const GENRE_LABELS = ["latin", "electronic", "hiphop", "rock", "classical"]
@@ -44,6 +45,9 @@ var _last_ping_msec: int = 0
var _genre_probs: Dictionary = {} var _genre_probs: Dictionary = {}
var _genre_argmax: String = "" var _genre_argmax: String = ""
var _genre_last_emit_msec: int = -5000 var _genre_last_emit_msec: int = -5000
var _genre_source: String = ""
var _genre_last_packet_msec: int = -1
var _diag_last_status: String = ""
func _ready() -> void: func _ready() -> void:
process_mode = Node.PROCESS_MODE_ALWAYS process_mode = Node.PROCESS_MODE_ALWAYS
@@ -70,6 +74,10 @@ func _spawn_bridge() -> void:
while _udp_peer.get_available_packet_count() > 0: while _udp_peer.get_available_packet_count() > 0:
_udp_peer.get_packet() _udp_peer.get_packet()
_bridge_protocol = 0 _bridge_protocol = 0
if _genre_source == "metadata":
genre_updated.emit("unknown", 0.0, {}, "rules", "")
_genre_source = ""
_genre_last_packet_msec = -1
_genre_probs.clear() _genre_probs.clear()
_genre_argmax = "" _genre_argmax = ""
_genre_last_emit_msec = -5000 _genre_last_emit_msec = -5000
@@ -180,6 +188,12 @@ func _process(delta: float) -> void:
_parse_genre(packet.get_string_from_utf8()) _parse_genre(packet.get_string_from_utf8())
continue continue
# Pacchetto diagnostico per-sorgente (DIAG:...). Riconosciuto PRIMA del ramo
# binario e ignorato ai fini del genere.
if packet.size() >= 5 and packet[0] == 68 and packet[1] == 73 and packet[2] == 65 and packet[3] == 71 and packet[4] == 58:
_parse_diag(packet.get_string_from_utf8())
continue
# Convenzione: le future estensioni testuali non devono MAI essere lunghe # Convenzione: le future estensioni testuali non devono MAI essere lunghe
# esattamente 8 byte (messaggio completo, non solo prefisso). # esattamente 8 byte (messaggio completo, non solo prefisso).
# HELLO:/HB:/APPS:/GENR: sono riconosciuti sopra; PING: e i futuri STATUS:/SILENT: # HELLO:/HB:/APPS:/GENR: sono riconosciuti sopra; PING: e i futuri STATUS:/SILENT:
@@ -202,6 +216,14 @@ func _process(delta: float) -> void:
if not packet_received: if not packet_received:
_loopback_rms = move_toward(_loopback_rms, 0.0, delta * 3.0) _loopback_rms = move_toward(_loopback_rms, 0.0, delta * 3.0)
# A stalled metadata stream must not leave a permanent 100% label on screen.
if _genre_source == "metadata" and _genre_last_packet_msec >= 0 and Time.get_ticks_msec() - _genre_last_packet_msec > 10000:
_genre_probs.clear()
_genre_argmax = ""
_genre_source = "rules"
_genre_last_packet_msec = -1
genre_updated.emit("unknown", 0.0, {}, "rules", "")
# Read queued heartbeats before checking the watchdog to avoid false timeouts. # Read queued heartbeats before checking the watchdog to avoid false timeouts.
_watchdog_timer += delta _watchdog_timer += delta
if _watchdog_timer >= 0.5: if _watchdog_timer >= 0.5:
@@ -223,13 +245,31 @@ func _parse_genre(raw_text: String) -> void:
var fields: PackedStringArray = raw_text.substr(5).split(";") var fields: PackedStringArray = raw_text.substr(5).split(";")
if fields.size() < 8: if fields.size() < 8:
return return
# Version, raw label and raw confidence precede the five probabilities. var source: String = "rules"
# The displayed label/confidence are recomputed from the smoothed distribution. var first_probability: int = 3
if fields[0] == "2":
if fields.size() < 9 or not ["metadata", "onnx", "rules"].has(fields[3]):
return
source = fields[3]
first_probability = 4
elif fields[0] != "1":
return
if source == "metadata" and (not GENRE_LABELS.has(fields[1]) or not fields[2].is_valid_float() or fields[2].to_float() != 1.0):
return
var incoming: Dictionary = {} var incoming: Dictionary = {}
var total: float = 0.0 var total: float = 0.0
for i in range(3, fields.size()): var detail: String = ""
var pair: PackedStringArray = fields[i].split("=") for i in range(first_probability, fields.size()):
if pair.size() != 2 or not GENRE_LABELS.has(pair[0]): # maxsplit=1: un valore puo' contenere '=' senza rompere il parsing.
var pair: PackedStringArray = fields[i].split("=", true, 1)
if pair.size() != 2:
continue
# Genere specifico dal catalogo ("Hard Rock", "Hip-Hop/Rap"): solo descrittivo,
# non partecipa alle probabilita' e non entra nella media esponenziale.
if pair[0] == "detail":
detail = pair[1].strip_edges()
continue
if not GENRE_LABELS.has(pair[0]):
continue continue
if incoming.has(pair[0]) or not pair[1].is_valid_float(): if incoming.has(pair[0]) or not pair[1].is_valid_float():
return return
@@ -240,7 +280,13 @@ func _parse_genre(raw_text: String) -> void:
total += probability total += probability
if incoming.size() != GENRE_LABELS.size() or total <= 0.0: if incoming.size() != GENRE_LABELS.size() or total <= 0.0:
return return
var first: bool = _genre_probs.is_empty() if source == "metadata":
# Discrete lookup: require a coherent one-hot distribution, never EMA-blend it.
for genre in GENRE_LABELS:
if float(incoming[genre]) != (1.0 if genre == fields[1] else 0.0):
return
var source_changed: bool = source != _genre_source
var first: bool = _genre_probs.is_empty() or source_changed or source == "metadata"
var best: String = "" var best: String = ""
var confidence: float = -1.0 var confidence: float = -1.0
for genre in GENRE_LABELS: for genre in GENRE_LABELS:
@@ -253,12 +299,42 @@ func _parse_genre(raw_text: String) -> void:
confidence = float(_genre_probs[genre]) confidence = float(_genre_probs[genre])
best = genre best = genre
var label: String = best if confidence >= GENRE_MIN_CONFIDENCE else "unknown" var label: String = best if confidence >= GENRE_MIN_CONFIDENCE else "unknown"
if source != "metadata":
detail = ""
var now: int = Time.get_ticks_msec() var now: int = Time.get_ticks_msec()
if best != _genre_argmax or now - _genre_last_emit_msec >= 5000: _genre_last_packet_msec = now
_genre_source = source
if source_changed or best != _genre_argmax or now - _genre_last_emit_msec >= 5000:
_genre_argmax = best _genre_argmax = best
_genre_last_emit_msec = now _genre_last_emit_msec = now
print("[SystemAudioAnalyzer] Genere: %s (%.0f%%)" % [label, confidence * 100.0]) var shown: String = detail if not detail.is_empty() else label
genre_updated.emit(label, confidence, _genre_probs.duplicate()) print("[SystemAudioAnalyzer] Genere: %s (%.0f%%) [%s]" % [shown, confidence * 100.0, source])
genre_updated.emit(label, confidence, _genre_probs.duplicate(), source, detail)
## Stampa una riga compatta dello stato per-sorgente solo quando uno dei campi di
## stato cambia (non a ogni pacchetto). Il contenuto DIAG non influenza il genere.
func _parse_diag(raw_text: String) -> void:
if not diag_enabled:
return
var fields: PackedStringArray = raw_text.substr(5).split(";")
if fields.size() < 2:
return
var diag: Dictionary = {}
for i in range(1, fields.size()):
var pair: PackedStringArray = fields[i].split("=", true, 1)
if pair.size() != 2:
continue
diag[pair[0]] = pair[1]
var meta: String = diag.get("meta", "")
var onnx: String = diag.get("onnx", "")
var itunes: String = diag.get("itunes", "")
var deezer: String = diag.get("deezer", "")
var bpm: String = diag.get("bpm", "0.0")
var anchor: String = diag.get("anchor", "0.0")
var status_key: String = "%s|%s|%s|%s" % [meta, onnx, itunes, deezer]
if status_key != _diag_last_status:
_diag_last_status = status_key
print("[DIAG] meta=%s onnx=%s itunes=%s deezer=%s bpm=%s anchor=%s" % [meta, onnx, itunes, deezer, bpm, anchor])
func _parse_applications_list(raw_text: String) -> void: func _parse_applications_list(raw_text: String) -> void:
var payload = raw_text.substr(5) var payload = raw_text.substr(5)
tools/design_cnn.py
+330
View File
@@ -0,0 +1,330 @@
"""
Designed (non-trained) genre CNN for the ldcc bridge.
The weights are WRITTEN, not learned: there is no dataset and no gradient descent.
Everything is deterministic and reproducible.
Design
------
1. Conv layer with fixed, interpretable filters: eight uniform averages over
contiguous 16-band mel groups. No randomness, no training.
2. Temporal statistics per band group: mean, standard deviation and mean absolute
frame-to-frame difference (a cheap onset/flux proxy). 8 x 3 = 24 features.
3. Linear head obtained in CLOSED FORM (ridge) so that the 24 features of five
reference sounds map to five one-hot vectors. The references are synthesized
right here, not recorded, so no external data and no licensing issues.
What this is and is not
-----------------------
- It is: a complete, valid ONNX model that satisfies the bridge contract, with
weights derived from an explicit procedure and inspectable in the source.
- It is not: a trained classifier. Accuracy on real music will be roughly the
accuracy of the five reference prototypes. Train with tools/train_genre_model.py
on real labelled audio to beat it.
Usage
-----
python tools/design_cnn.py --output-dir bin
python tools/design_cnn.py --validate # checks unseen renders
Contract (identical to the bridge):
input [1, 1, 128, 300] float32 = (batch, channel, mel bands, frames)
output [1, 5] float32 = probabilities, label order LABELS
"""
from __future__ import annotations
import argparse
import json
import os
import sys
import tempfile
from pathlib import Path
import numpy as np
import torch
import torch.nn as nn
sys.path.insert(0, str(Path(__file__).resolve().parent))
import train_genre_model as tgm # noqa: E402 (front-end + contract, single source)
SR = 48000
CROP_SECONDS = 3.0
# Normalization constants are a design choice: they must match the json shipped
# with the model, because the bridge applies them before the tensor.
MEAN = -4.0
STD = 4.0
N_GROUPS = tgm.N_MELS // 16 # 128 / 16 = 8 mel groups
N_FEATURES = N_GROUPS * 3 # mean, std, flux
BPM_RANGE = {"latin": (85, 105), "electronic": (115, 135), "hiphop": (80, 95),
"rock": (125, 150), "classical": (70, 80)}
# --------------------------------------------------------------------------- #
# Synthesis of reference sounds (deterministic, no external data)
# --------------------------------------------------------------------------- #
def _add(buf, i0, wave):
"""Somma un burst nel buffer, troncando oltre la fine (niente broadcast negativo)."""
if i0 >= buf.size:
return
m = min(wave.size, buf.size - i0)
buf[i0:i0 + m] += wave[:m]
def _kick(buf, sr, t0, amp, rng):
n = int(0.35 * sr)
i0 = int(t0 * sr)
t = np.arange(n) / sr
wave = np.sin(2 * np.pi * (55 * t + (1 - np.exp(-t * 40)) / 40)) * np.exp(-t * 16) * amp
_add(buf, i0, wave)
def _snare(buf, sr, t0, amp, rng):
n = int(0.15 * sr)
i0 = int(t0 * sr)
t = np.arange(n) / sr
noise = (rng.standard_normal(n) * 2 - 1) * np.exp(-t * 28)
body = np.sin(2 * np.pi * 190 * t) * np.exp(-t * 30) * 0.4
wave = (noise + body) * amp
_add(buf, i0, wave)
def _hat(buf, sr, t0, amp, rng):
n = int(0.04 * sr)
i0 = int(t0 * sr)
t = np.arange(n) / sr
noise = rng.standard_normal(n)
hp = np.diff(noise, prepend=0.0)
wave = hp * np.exp(-t * 90) * amp
_add(buf, i0, wave)
def _chord(buf, sr, t0, dur, amp):
n = int(dur * sr)
i0 = int(t0 * sr)
t = np.arange(n) / sr
env = np.exp(-t * 3.0)
v = np.zeros(n)
for root in (110.0, 146.83, 164.81):
for h in range(1, 21):
v += np.sin(2 * np.pi * root * h * t + h) / h
wave = np.tanh(v * 6.0) * env * amp / 10.0
_add(buf, i0, wave)
def _tones(buf, sr, seconds, rng):
notes = (220.0, 261.63, 329.63, 392.0, 329.63, 261.63)
t = 0.0
i = 0
while t < seconds:
f = notes[i % len(notes)]
dur = 1.6 + rng.random() * 1.2
amp = 0.12 + rng.random() * 0.5
n = int(dur * sr)
i0 = int(t * sr)
tt = np.arange(n) / sr
env = np.exp(-tt * 1.2) * (1 - np.exp(-tt * 60))
wave = (np.sin(2 * np.pi * f * tt) + 0.3 * np.sin(4 * np.pi * f * tt)) * env * amp
_add(buf, i0, wave)
t += dur * 0.85
i += 1
def synth(kind: str, bpm: float, seconds: float, seed: int) -> np.ndarray:
rng = np.random.default_rng(seed)
sr = SR
buf = np.zeros(int(sr * seconds), dtype=np.float64)
sixteenth = 60.0 / bpm / 4.0
kick, snare, hat, chord = [], [], [], False
if kind == "latin":
kick, snare = [0, 8], [0, 3, 6, 8, 11, 14]
elif kind == "electronic":
kick, hat = [0, 4, 8, 12], [2, 6, 10, 14]
elif kind == "hiphop":
kick, snare, hat = [0, 6], [4, 12], [2, 6, 10, 14]
elif kind == "rock":
kick, snare, hat = [0, 6, 8], [4, 12], [0, 2, 4, 6, 8, 10, 12, 14]
chord = True
elif kind == "classical":
_tones(buf, sr, seconds, rng)
return np.clip(buf, -1, 1).astype(np.float32)
else:
raise ValueError(kind)
bar = 0
while bar * 16 * sixteenth < seconds:
base = bar * 16 * sixteenth
for s in kick:
_kick(buf, sr, base + s * sixteenth, 0.85, rng)
for s in snare:
_snare(buf, sr, base + s * sixteenth, 0.45, rng)
for s in hat:
_hat(buf, sr, base + s * sixteenth, 0.18, rng)
if chord:
_chord(buf, sr, base, sixteenth * 8, 0.30)
bar += 1
return np.clip(buf, -1, 1).astype(np.float32)
# --------------------------------------------------------------------------- #
# Fixed convolution + temporal statistics
# --------------------------------------------------------------------------- #
def band_group_weights() -> torch.Tensor:
"""Uno filtro per gruppo di bande: kernel [8,1,128,1], canale g attivo (1/16) sulle
righe mel 16g..16g+15. Deterministico e leggibile, nessuna inizializzazione casuale."""
w = torch.zeros(N_GROUPS, 1, tgm.N_MELS, 1, dtype=torch.float32)
for g in range(N_GROUPS):
w[g, 0, g * 16:(g + 1) * 16, 0] = 1.0 / 16.0
return w
class DesignedGenreCNN(nn.Module):
"""Conv(fixed) -> (mean, std, flux) -> Linear(fitted)."""
def __init__(self) -> None:
super().__init__()
self.bands = nn.Conv2d(1, N_GROUPS, (tgm.N_MELS, 1), stride=(1, 1), bias=False)
self.bands.weight = nn.Parameter(band_group_weights(), requires_grad=False)
self.head = nn.Linear(N_FEATURES, len(tgm.LABELS))
def pool(self, x: torch.Tensor) -> torch.Tensor:
# x: [N, G, 1, T]
x = x.squeeze(2) # [N, G, T]
mean = x.mean(dim=2) # [N, G]
var = torch.clamp((x * x).mean(dim=2) - mean * mean, min=0.0)
std = torch.sqrt(var + 1e-8) # [N, G]
flux = torch.abs(x[:, :, 1:] - x[:, :, :-1]).mean(dim=2)
return torch.cat([mean, std, flux], dim=1) # [N, 24]
def embed(self, x: torch.Tensor) -> torch.Tensor:
# Invarianza al livello: il bridge normalizza con una costante globale, ma il
# volume di riproduzione sposta tutto lo spettro. Sottrarre la media della
# finestra rende il modello insensibile a un guadagno costante.
return self.pool(self.bands(x - x.mean()))
def forward(self, x: torch.Tensor) -> torch.Tensor:
return torch.softmax(self.head(self.embed(x)), dim=1)
def features_of(audio: np.ndarray) -> np.ndarray:
"""Run the bridge front-end on a raw clip and return the 24 pooled features."""
model = DesignedGenreCNN()
ten = torch.from_numpy(tgm.tensor_frontend(audio, SR, MEAN, STD))
with torch.no_grad():
return model.embed(ten).numpy()[0]
# --------------------------------------------------------------------------- #
# Closed-form head
# --------------------------------------------------------------------------- #
def fit_head(features: np.ndarray, gain: float = 2.5) -> tuple[np.ndarray, np.ndarray]:
"""Minimum-norm least-squares fit of the 5 reference rows to one-hot targets.
Closed form, deterministic, no iterative training. `gain` sharpens the logits:
the minimum-norm solution is deliberately small, and without a gain the softmax
would stay close to uniform.
"""
x = np.concatenate([features, np.ones((features.shape[0], 1), dtype=np.float64)], axis=1)
y = np.eye(len(tgm.LABELS))
solution, *_ = np.linalg.lstsq(x, y, rcond=None) # [features+1, classes]
w = solution[:-1].T.astype(np.float32) * gain
b = solution[-1].astype(np.float32) * gain
return w, b
def build_references(seed: int = 11) -> tuple[np.ndarray, list[str]]:
rows = []
used = []
for label in tgm.LABELS:
lo, hi = BPM_RANGE[label]
bpm = 0.5 * (lo + hi)
audio = synth(label, bpm, 6.0, seed)
rows.append(features_of(audio))
used.append(f"{label}@{bpm:.0f}bpm")
return np.vstack(rows), used
def main() -> int:
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument("--output-dir", type=Path, default=Path(__file__).resolve().parents[1] / "bin")
parser.add_argument("--validate", action="store_true", help="classify unseen renders")
parser.add_argument("--seed", type=int, default=11)
args = parser.parse_args()
features, used = build_references(args.seed)
print("PROTOTYPES: " + ", ".join(used), flush=True)
w, b = fit_head(features)
model = DesignedGenreCNN()
with torch.no_grad():
model.head.weight.copy_(torch.from_numpy(w))
model.head.bias.copy_(torch.from_numpy(b))
model.eval()
# Sanity: the references must come back as themselves (head applied to the
# pooled features, since `features` is already past the convolution).
with torch.no_grad():
probs = model.head(torch.from_numpy(features)).softmax(dim=1)
print("REFERENCE FIT:")
for i, label in enumerate(tgm.LABELS):
pred = tgm.LABELS[int(probs[i].argmax())]
print(f" {label:11s} -> {pred:11s} p={probs[i].max():.3f}")
if args.validate:
print("UNSEEN RENDERS:")
ok = 0
total = 0
for label in tgm.LABELS:
lo, hi = BPM_RANGE[label]
for frac in (0.25, 0.75):
bpm = lo + (hi - lo) * frac
audio = synth(label, bpm, 6.0, args.seed + 100 + total)
f = features_of(audio).reshape(1, -1)
with torch.no_grad():
p = model.head(torch.from_numpy(f)).softmax(dim=1).numpy()[0]
pred = tgm.LABELS[int(np.argmax(p))]
ok += pred == label
total += 1
print(f" {label:11s} {bpm:6.1f}bpm seed={args.seed + 100 + total - 1} -> {pred:11s} p={p.max():.3f}")
print(f"VALIDATION: {ok}/{total}")
if ok < total * 0.6:
print("WARNING: too few unseen renders classified correctly; this model is a scaffold.")
out = args.output_dir
out.mkdir(parents=True, exist_ok=True)
onnx_path = out / "genre_model.onnx"
json_path = out / "genre_model.json"
example = torch.zeros(1, 1, tgm.N_MELS, tgm.FRAMES, dtype=torch.float32)
torch.onnx.export(
model, example, str(onnx_path),
input_names=["logmel"], output_names=["probabilities"],
opset_version=17, dynamo=False,
)
with torch.no_grad():
check = model(example).numpy()[0]
meta = {
"labels": tgm.LABELS,
"mean": MEAN,
"std": STD,
"version": tgm.VERSION,
"source": "designed",
"frontend": {
"fft": 2048, "hann": "periodic", "hop": "sr//100",
"padding": "causal-left-N-hop", "power": "abs(rfft)^2",
"mel": "HTK", "bands": tgm.N_MELS, "fmin": 30, "fmax": 11000,
"area": "divide by full support width in fractional bins", "log_floor": 1e-10,
},
}
json_path.write_text(json.dumps(meta, indent=2), encoding="utf-8")
print(f"MODEL={onnx_path.resolve()} ({onnx_path.stat().st_size} bytes)")
print(f"JSON={json_path.resolve()}")
print(f"SANITY: sum={check.sum():.6f} max={check.max():.3f} label={tgm.LABELS[int(check.argmax())]}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
tools/train_genre_model.py
+361
View File
@@ -0,0 +1,361 @@
"""Train/export the fixed ldcc ONNX contract; no-argument invocation runs self-test.
Python requires only torch, numpy, scipy and onnx. No onnxruntime is needed.
Parity: --verify-dump tensor.bin --data SELFTEST_DIR --rate 48000 (or 44100).
The fixture is SELFTEST_DIR/fixture.wav; --rate reinterprets samples, NOT resamples.
Train and export real data with --data DATA_DIR --output-dir bin.
"""
from __future__ import annotations
import argparse
import copy
import json
import math
import random
import tempfile
from functools import lru_cache
from pathlib import Path
import numpy as np
from scipy.io import wavfile
from scipy.signal import lfilter
import torch
from torch import nn
import onnx
from onnx.reference import ReferenceEvaluator
LABELS = ["latin", "electronic", "hiphop", "rock", "classical"]
N_FFT, N_MELS, FRAMES = 2048, 128, 300
FMIN, FMAX, LOG_FLOOR = 30.0, 11000.0, 1e-10
VERSION = 1
def read_wav(path: Path) -> tuple[np.ndarray, int]:
sr, audio = wavfile.read(path)
if sr not in (44100, 48000):
raise ValueError(f"{path}: only 44100/48000 Hz supported, got {sr}")
if audio.dtype == np.uint8:
audio = (audio.astype(np.float64) - 128.0) / 128.0
elif np.issubdtype(audio.dtype, np.signedinteger):
# scipy returns 24-bit WAV left-justified in int32, matching this scaling.
audio = audio.astype(np.float64) / float(2 ** (audio.dtype.itemsize * 8 - 1))
elif np.issubdtype(audio.dtype, np.floating):
audio = audio.astype(np.float64)
else:
raise ValueError(f"Unsupported WAV dtype: {audio.dtype}")
if audio.ndim == 2:
audio = audio.mean(axis=1)
if audio.ndim != 1 or not np.all(np.isfinite(audio)):
raise ValueError(f"{path}: invalid/nonfinite PCM")
# C# ingestion is float32 mono. Promote only AFTER this rounding for FFT parity.
return audio.astype(np.float32), sr
@lru_cache(maxsize=2)
def mel_bank(sr: int) -> np.ndarray:
if sr not in (44100, 48000):
raise ValueError("Expected 44100 or 48000 Hz")
low = 2595.0 * math.log10(1 + FMIN / 700)
high = 2595.0 * math.log10(1 + FMAX / 700)
# Fractional edges: do NOT round/floor the FFT-bin locations.
edges = np.array([
700 * (10 ** ((low + (high - low) * i / (N_MELS + 1)) / 2595) - 1) * N_FFT / sr
for i in range(N_MELS + 2)
], dtype=np.float64)
bins = np.arange(N_FFT // 2 + 1, dtype=np.float64)
filters = np.zeros((N_MELS, len(bins)), dtype=np.float64)
for m in range(N_MELS):
left, center, right = edges[m:m + 3]
filters[m] = np.maximum(0, np.minimum((bins - left) / (center - left), (right - bins) / (right - center))) / (right - left)
return filters
def stft_manual(x: np.ndarray, sr: int) -> np.ndarray:
"""Causal periodic-Hann complex rFFT, shape [1025, floor(len(x)/hop)]."""
hop = sr // 100
x = np.asarray(x, dtype=np.float32).astype(np.float64)
if x.ndim != 1 or not np.isfinite(x).all():
raise ValueError("Expected finite mono samples")
if len(x) < hop:
return np.empty((N_FFT // 2 + 1, 0), dtype=np.complex128)
padded = np.pad(x, (N_FFT - hop, 0))
frames = np.lib.stride_tricks.sliding_window_view(padded, N_FFT)[::hop]
hann = 0.5 - 0.5 * np.cos(2 * np.pi * np.arange(N_FFT, dtype=np.float64) / N_FFT)
return np.fft.rfft(frames * hann, axis=1).T
def logmel(x: np.ndarray, sr: int) -> np.ndarray:
spectrum = stft_manual(x, sr)
# Raw power: no /N and no doubling of positive-frequency bins.
power = spectrum.real ** 2 + spectrum.imag ** 2
energies = mel_bank(sr) @ power
return np.log10(np.maximum(energies, LOG_FLOOR))
def tensor_frontend(x: np.ndarray, sr: int, mean: float, std: float) -> np.ndarray:
needed = FRAMES * (sr // 100)
if len(x) < needed:
raise ValueError(f"Need {needed} samples for 300 frames, got {len(x)}")
if not np.isfinite(mean) or not np.isfinite(std) or std <= 0:
raise ValueError("Invalid normalization statistics")
result = ((logmel(x[:needed], sr) - mean) / std).astype(np.float32)
if result.shape != (N_MELS, FRAMES) or not np.isfinite(result).all():
raise ValueError("Invalid frontend output")
return result[None, None]
def discover(data: Path) -> list[tuple[Path, int]]:
found = []
for label_id, label in enumerate(LABELS):
folder = data / label
if folder.exists():
found.extend((p.resolve(), label_id) for p in sorted(folder.iterdir()) if p.suffix.lower() == ".wav")
if not found:
raise ValueError("No WAV files in class subfolders")
return found
def split_files(items, seed: int):
rng = random.Random(seed)
train, val = [], []
for label in range(len(LABELS)):
group = [item for item in items if item[1] == label]
if not group:
continue
if len(group) < 2:
raise ValueError(f"Class {LABELS[label]} needs at least two distinct files")
rng.shuffle(group)
nval = max(1, round(len(group) * 0.25))
val.extend(group[:nval]); train.extend(group[nval:])
if set(p for p, _ in train) & set(p for p, _ in val):
raise AssertionError("File leakage between train and validation")
return train, val
def random_crop(audio, sr, rng):
count = FRAMES * (sr // 100)
if len(audio) < count:
raise ValueError("Training WAV shorter than 3 seconds")
offset = int(rng.integers(0, len(audio) - count + 1))
return audio[offset:offset + count].copy()
def training_statistics(items, seed):
"""Global scalar mean/std on TRAIN FILES ONLY; two reproducible crops/file."""
rng = np.random.default_rng(seed)
total = squares = 0.0
count = 0
for path, _ in items:
audio, sr = read_wav(path)
for _ in range(2):
values = logmel(random_crop(audio, sr, rng), sr)
total += float(values.sum()); squares += float(np.square(values).sum()); count += values.size
mean = total / count
std = math.sqrt(max(0, squares / count - mean * mean))
if std < 1e-8:
raise ValueError("Training set has zero variance")
return mean, std
class WavDataset(torch.utils.data.Dataset):
def __init__(self, items, mean, std, seed, augment):
self.items, self.mean, self.std, self.seed, self.augment = items, mean, std, seed, augment
self.rng = np.random.default_rng(seed)
def __len__(self):
return len(self.items)
def __getitem__(self, index):
path, label = self.items[index]
audio, sr = read_wav(path)
# Validation crop fixed across epochs; train crop randomized on each read.
rng = self.rng if self.augment else np.random.default_rng(self.seed + index)
x = random_crop(audio, sr, rng)
if self.augment:
x *= np.float32(10 ** (rng.uniform(-6, 6) / 20))
if rng.random() < 0.5:
x *= -1
return torch.from_numpy(tensor_frontend(x, sr, self.mean, self.std)[0]), label
class GenreCNN(nn.Module):
def __init__(self, active_labels):
super().__init__()
self.features = nn.Sequential(
nn.Conv2d(1, 8, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
nn.Conv2d(8, 16, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
nn.Conv2d(16, 24, 3, padding=1), nn.ReLU(), nn.MaxPool2d(2),
nn.AdaptiveAvgPool2d((1, 1)),
)
self.classifier = nn.Linear(24, 5)
self.register_buffer("active", torch.tensor([i in active_labels for i in range(5)], dtype=torch.bool))
def forward(self, x):
logits = self.classifier(self.features(x).flatten(1))
# Self-test keeps five outputs. Absent classes have EXACT zero probability.
return torch.softmax(logits.masked_fill(~self.active, float("-inf")), dim=1)
def epoch(model, loader, optimizer=None):
training = optimizer is not None
model.train(training)
loss_total, correct, count = 0.0, 0, 0
with torch.set_grad_enabled(training):
for x, labels in loader:
if training:
optimizer.zero_grad()
probs = model(x)
loss = -torch.log(probs[torch.arange(len(labels)), labels].clamp_min(1e-9)).mean()
if training:
loss.backward(); optimizer.step()
loss_total += loss.item() * len(labels)
correct += (probs.argmax(1) == labels).sum().item(); count += len(labels)
return loss_total / count, correct / count
def validate_export(model_path: Path, model: GenreCNN, sample: np.ndarray):
exported = onnx.load(str(model_path))
onnx.checker.check_model(exported)
shape = [dim.dim_value for dim in exported.graph.input[0].type.tensor_type.shape.dim]
if shape != [1, 1, 128, 300]:
raise AssertionError(f"Bad ONNX shape {shape}")
output = ReferenceEvaluator(exported).run(None, {exported.graph.input[0].name: sample})[0]
if output.shape != (1, 5) or not np.isfinite(output).all() or np.any(output < 0) or abs(float(output.sum()) - 1) > 1e-5:
raise AssertionError(f"Invalid ONNX probabilities: {output}")
with torch.no_grad():
expected = model(torch.from_numpy(sample)).numpy()
difference = float(np.max(np.abs(output - expected)))
if difference > 1e-4:
raise AssertionError(f"Torch/ONNX difference {difference}")
inactive = ~model.active.cpu().numpy()
if np.any(output[0, inactive] != 0):
raise AssertionError("Unused classes must be exactly zero")
print(f"ONNX OK: shape={shape}; sum={output.sum():.9f}; torch_max_abs={difference:.9g}; probs={output.tolist()}", flush=True)
def train(data: Path, output: Path, epochs: int, seed: int, batch_size: int, patience: int):
random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)
items = discover(data)
train_items, val_items = split_files(items, seed)
mean, std = training_statistics(train_items, seed)
output.mkdir(parents=True, exist_ok=True)
manifest = {"train": [str(p) for p, _ in train_items], "val": [str(p) for p, _ in val_items]}
(output / "split.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
print(f"SPLIT: train_files={len(train_items)} val_files={len(val_items)} overlap=0; mean={mean:.12g} std={std:.12g}", flush=True)
active = sorted({label for _, label in train_items})
model = GenreCNN(active)
optimizer = torch.optim.Adam(model.parameters(), lr=0.001)
train_loader = torch.utils.data.DataLoader(WavDataset(train_items, mean, std, seed, True), batch_size=batch_size, shuffle=True, num_workers=0)
val_loader = torch.utils.data.DataLoader(WavDataset(val_items, mean, std, seed + 1, False), batch_size=batch_size, num_workers=0)
best, stale, best_state = math.inf, 0, None
checkpoint = output / "genre_model.pt"
for number in range(1, epochs + 1):
loss, accuracy = epoch(model, train_loader, optimizer)
val_loss, val_accuracy = epoch(model, val_loader)
print(f"EPOCH {number}/{epochs}: loss={loss:.6f} accuracy={accuracy:.3f} val_loss={val_loss:.6f} val_accuracy={val_accuracy:.3f}", flush=True)
if val_loss < best:
best, stale = val_loss, 0
best_state = copy.deepcopy(model.state_dict())
torch.save({"state_dict": best_state, "mean": mean, "std": std, "labels": LABELS, "version": VERSION, "epoch": number}, checkpoint)
else:
stale += 1
if stale >= patience:
print(f"EARLY STOP after {number} epochs", flush=True); break
model.load_state_dict(best_state); model.eval()
model_path, json_path = output / "genre_model.onnx", output / "genre_model.json"
example = torch.zeros(1, 1, 128, 300, dtype=torch.float32)
# Explicit legacy exporter: opset17, fixed shape, no dynamo/onnxscript requirement.
torch.onnx.export(model, example, str(model_path), input_names=["logmel"], output_names=["probabilities"],
opset_version=17, dynamo=False, external_data=False)
print("EXPORT: torch.onnx.export(dynamo=False), opset=17", flush=True)
metadata = {"labels": LABELS, "mean": mean, "std": std, "version": VERSION,
"frontend": {"fft": 2048, "hann": "periodic", "hop": "sr//100", "padding": "causal-left-N-hop",
"power": "abs(rfft)^2", "mel": "HTK", "bands": 128, "fmin": 30, "fmax": 11000,
"area": "divide by full support width in fractional bins", "log_floor": 1e-10}}
json_path.write_text(json.dumps(metadata, indent=2), encoding="utf-8")
audio, sr = read_wav(val_items[0][0])
validate_export(model_path, model, tensor_frontend(audio, sr, mean, std))
print(f"MODEL={model_path.resolve()}\nJSON={json_path.resolve()}\nCHECKPOINT={checkpoint.resolve()}", flush=True)
return model_path, json_path
def synthetic_dataset(root: Path, seed: int):
rng = np.random.default_rng(seed)
sr = 48000
time = np.arange(sr * 20, dtype=np.float64) / sr
for label_id, label in enumerate(LABELS[:3]):
folder = root / label; folder.mkdir(parents=True)
for index in range(12):
if label_id == 0:
period = 60 / rng.uniform(85, 110)
phase = np.mod(time, period)
audio = np.exp(-phase * 45) * np.sin(2 * np.pi * (70 + index * 3) * time)
audio += 0.2 * rng.standard_normal(len(time)) * np.exp(-np.mod(time + period * 0.375, period) * 80)
elif label_id == 1:
frequency = rng.uniform(150, 450)
audio = sum(np.sin(2 * np.pi * frequency * harmonic * time + rng.uniform(0, 6)) / harmonic for harmonic in range(1, 6))
audio *= 0.6 + 0.4 * np.sin(2 * np.pi * rng.uniform(0.5, 2) * time)
else:
audio = lfilter([1, -0.8], [1, -rng.uniform(0.1, 0.6)], rng.standard_normal(len(time)))
audio *= 0.4 + 0.6 * np.sin(2 * np.pi * rng.uniform(1, 4) * time) ** 2
audio = audio / max(float(np.max(np.abs(audio))), 1e-9) * rng.uniform(0.3, 0.8)
wavfile.write(folder / f"synthetic_{index:02d}.wav", sr, np.round(audio * 32767).astype(np.int16))
# A fixed, broadband fixture exercises low-energy bins without depending on a player.
n = sr * 4
t = np.arange(n) / sr
fixture = 0.15 * np.sin(2 * np.pi * 440 * t) + 0.1 * np.sin(2 * np.pi * 4500 * t) + 0.03 * rng.standard_normal(n)
wavfile.write(root / "fixture.wav", sr, np.round(np.clip(fixture, -1, 1) * 32767).astype(np.int16))
print(f"SELFTEST_DIR={root.resolve()}\nFIXTURE={(root / 'fixture.wav').resolve()}", flush=True)
def verify_dump(path: Path, data: Path, rate: int | None, metadata_path: Path | None):
if data is None:
raise ValueError("--verify-dump needs --data pointing to the printed self-test directory")
metadata = json.loads((metadata_path or data / "genre_model.json").read_text(encoding="utf-8"))
if metadata["labels"] != LABELS or metadata["version"] != VERSION:
raise ValueError("Model metadata contract mismatch")
audio, sr = read_wav(data / "fixture.wav")
if rate is not None:
sr = rate
expected = tensor_frontend(audio, sr, metadata["mean"], metadata["std"])
if path.stat().st_size != 4 * N_MELS * FRAMES:
raise ValueError("Dump must contain exactly 153600 bytes (38400 float32 values)")
actual = np.fromfile(path, dtype="<f4").reshape(1, 1, N_MELS, FRAMES)
if not np.isfinite(actual).all():
raise ValueError("Non-finite tensor dump")
difference = float(np.max(np.abs(actual.astype(np.float64) - expected.astype(np.float64))))
print(f"PARITY rate={sr} shape={actual.shape} max_abs={difference:.12g} threshold=0.0001", flush=True)
if difference >= 1e-4:
raise AssertionError(f"Frontend parity failed: {difference}")
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--data", type=Path)
parser.add_argument("--output-dir", type=Path, default=Path(__file__).resolve().parents[1] / "bin")
parser.add_argument("--epochs", type=int, default=20)
parser.add_argument("--batch-size", type=int, default=4)
parser.add_argument("--patience", type=int, default=5)
parser.add_argument("--seed", type=int, default=12345)
parser.add_argument("--threads", type=int, default=4)
parser.add_argument("--selftest", action="store_true")
parser.add_argument("--verify-dump", type=Path)
parser.add_argument("--metadata-json", type=Path)
parser.add_argument("--rate", type=int, choices=[44100, 48000])
args = parser.parse_args()
if min(args.epochs, args.batch_size, args.patience, args.threads) < 1:
parser.error("epochs/batch-size/patience/threads must be positive")
torch.set_num_threads(args.threads)
if args.verify_dump:
verify_dump(args.verify_dump, args.data, args.rate, args.metadata_json)
elif args.selftest or args.data is None:
root = Path(tempfile.mkdtemp(prefix="ldcc-genre-selftest-"))
synthetic_dataset(root, args.seed)
train(root, root, 3, args.seed, args.batch_size, args.patience)
print("SELFTEST PASS: synthetic only, not a usable musical genre model", flush=True)
else:
train(args.data, args.output_dir, args.epochs, args.seed, args.batch_size, args.patience)
if __name__ == "__main__":
main()