add onnx and metadata
This commit is contained in:
Overside srl committed
1 parent
98d9fa24cf
commit
5f3f915258
18 files changed
+2229
-78
No files matched your search
@@ -0,0 +1,330 @@
|
||||
"""
|
||||
Designed (non-trained) genre CNN for the ldcc bridge.
|
||||
|
||||
The weights are WRITTEN, not learned: there is no dataset and no gradient descent.
|
||||
Everything is deterministic and reproducible.
|
||||
|
||||
Design
|
||||
------
|
||||
1. Conv layer with fixed, interpretable filters: eight uniform averages over
|
||||
contiguous 16-band mel groups. No randomness, no training.
|
||||
2. Temporal statistics per band group: mean, standard deviation and mean absolute
|
||||
frame-to-frame difference (a cheap onset/flux proxy). 8 x 3 = 24 features.
|
||||
3. Linear head obtained in CLOSED FORM (ridge) so that the 24 features of five
|
||||
reference sounds map to five one-hot vectors. The references are synthesized
|
||||
right here, not recorded, so no external data and no licensing issues.
|
||||
|
||||
What this is and is not
|
||||
-----------------------
|
||||
- It is: a complete, valid ONNX model that satisfies the bridge contract, with
|
||||
weights derived from an explicit procedure and inspectable in the source.
|
||||
- It is not: a trained classifier. Accuracy on real music will be roughly the
|
||||
accuracy of the five reference prototypes. Train with tools/train_genre_model.py
|
||||
on real labelled audio to beat it.
|
||||
|
||||
Usage
|
||||
-----
|
||||
python tools/design_cnn.py --output-dir bin
|
||||
python tools/design_cnn.py --validate # checks unseen renders
|
||||
|
||||
Contract (identical to the bridge):
|
||||
input [1, 1, 128, 300] float32 = (batch, channel, mel bands, frames)
|
||||
output [1, 5] float32 = probabilities, label order LABELS
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
||||
import train_genre_model as tgm # noqa: E402 (front-end + contract, single source)
|
||||
|
||||
SR = 48000
|
||||
CROP_SECONDS = 3.0
|
||||
# Normalization constants are a design choice: they must match the json shipped
|
||||
# with the model, because the bridge applies them before the tensor.
|
||||
MEAN = -4.0
|
||||
STD = 4.0
|
||||
N_GROUPS = tgm.N_MELS // 16 # 128 / 16 = 8 mel groups
|
||||
N_FEATURES = N_GROUPS * 3 # mean, std, flux
|
||||
BPM_RANGE = {"latin": (85, 105), "electronic": (115, 135), "hiphop": (80, 95),
|
||||
"rock": (125, 150), "classical": (70, 80)}
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Synthesis of reference sounds (deterministic, no external data)
|
||||
# --------------------------------------------------------------------------- #
|
||||
def _add(buf, i0, wave):
|
||||
"""Somma un burst nel buffer, troncando oltre la fine (niente broadcast negativo)."""
|
||||
if i0 >= buf.size:
|
||||
return
|
||||
m = min(wave.size, buf.size - i0)
|
||||
buf[i0:i0 + m] += wave[:m]
|
||||
|
||||
|
||||
def _kick(buf, sr, t0, amp, rng):
|
||||
n = int(0.35 * sr)
|
||||
i0 = int(t0 * sr)
|
||||
t = np.arange(n) / sr
|
||||
wave = np.sin(2 * np.pi * (55 * t + (1 - np.exp(-t * 40)) / 40)) * np.exp(-t * 16) * amp
|
||||
_add(buf, i0, wave)
|
||||
|
||||
|
||||
|
||||
def _snare(buf, sr, t0, amp, rng):
|
||||
n = int(0.15 * sr)
|
||||
i0 = int(t0 * sr)
|
||||
t = np.arange(n) / sr
|
||||
noise = (rng.standard_normal(n) * 2 - 1) * np.exp(-t * 28)
|
||||
body = np.sin(2 * np.pi * 190 * t) * np.exp(-t * 30) * 0.4
|
||||
wave = (noise + body) * amp
|
||||
_add(buf, i0, wave)
|
||||
|
||||
|
||||
|
||||
def _hat(buf, sr, t0, amp, rng):
|
||||
n = int(0.04 * sr)
|
||||
i0 = int(t0 * sr)
|
||||
t = np.arange(n) / sr
|
||||
noise = rng.standard_normal(n)
|
||||
hp = np.diff(noise, prepend=0.0)
|
||||
wave = hp * np.exp(-t * 90) * amp
|
||||
_add(buf, i0, wave)
|
||||
|
||||
|
||||
|
||||
def _chord(buf, sr, t0, dur, amp):
|
||||
n = int(dur * sr)
|
||||
i0 = int(t0 * sr)
|
||||
t = np.arange(n) / sr
|
||||
env = np.exp(-t * 3.0)
|
||||
v = np.zeros(n)
|
||||
for root in (110.0, 146.83, 164.81):
|
||||
for h in range(1, 21):
|
||||
v += np.sin(2 * np.pi * root * h * t + h) / h
|
||||
wave = np.tanh(v * 6.0) * env * amp / 10.0
|
||||
_add(buf, i0, wave)
|
||||
|
||||
|
||||
|
||||
def _tones(buf, sr, seconds, rng):
|
||||
notes = (220.0, 261.63, 329.63, 392.0, 329.63, 261.63)
|
||||
t = 0.0
|
||||
i = 0
|
||||
while t < seconds:
|
||||
f = notes[i % len(notes)]
|
||||
dur = 1.6 + rng.random() * 1.2
|
||||
amp = 0.12 + rng.random() * 0.5
|
||||
n = int(dur * sr)
|
||||
i0 = int(t * sr)
|
||||
tt = np.arange(n) / sr
|
||||
env = np.exp(-tt * 1.2) * (1 - np.exp(-tt * 60))
|
||||
wave = (np.sin(2 * np.pi * f * tt) + 0.3 * np.sin(4 * np.pi * f * tt)) * env * amp
|
||||
_add(buf, i0, wave)
|
||||
t += dur * 0.85
|
||||
i += 1
|
||||
|
||||
|
||||
def synth(kind: str, bpm: float, seconds: float, seed: int) -> np.ndarray:
|
||||
rng = np.random.default_rng(seed)
|
||||
sr = SR
|
||||
buf = np.zeros(int(sr * seconds), dtype=np.float64)
|
||||
sixteenth = 60.0 / bpm / 4.0
|
||||
kick, snare, hat, chord = [], [], [], False
|
||||
if kind == "latin":
|
||||
kick, snare = [0, 8], [0, 3, 6, 8, 11, 14]
|
||||
elif kind == "electronic":
|
||||
kick, hat = [0, 4, 8, 12], [2, 6, 10, 14]
|
||||
elif kind == "hiphop":
|
||||
kick, snare, hat = [0, 6], [4, 12], [2, 6, 10, 14]
|
||||
elif kind == "rock":
|
||||
kick, snare, hat = [0, 6, 8], [4, 12], [0, 2, 4, 6, 8, 10, 12, 14]
|
||||
chord = True
|
||||
elif kind == "classical":
|
||||
_tones(buf, sr, seconds, rng)
|
||||
return np.clip(buf, -1, 1).astype(np.float32)
|
||||
else:
|
||||
raise ValueError(kind)
|
||||
|
||||
bar = 0
|
||||
while bar * 16 * sixteenth < seconds:
|
||||
base = bar * 16 * sixteenth
|
||||
for s in kick:
|
||||
_kick(buf, sr, base + s * sixteenth, 0.85, rng)
|
||||
for s in snare:
|
||||
_snare(buf, sr, base + s * sixteenth, 0.45, rng)
|
||||
for s in hat:
|
||||
_hat(buf, sr, base + s * sixteenth, 0.18, rng)
|
||||
if chord:
|
||||
_chord(buf, sr, base, sixteenth * 8, 0.30)
|
||||
bar += 1
|
||||
return np.clip(buf, -1, 1).astype(np.float32)
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Fixed convolution + temporal statistics
|
||||
# --------------------------------------------------------------------------- #
|
||||
def band_group_weights() -> torch.Tensor:
|
||||
"""Uno filtro per gruppo di bande: kernel [8,1,128,1], canale g attivo (1/16) sulle
|
||||
righe mel 16g..16g+15. Deterministico e leggibile, nessuna inizializzazione casuale."""
|
||||
w = torch.zeros(N_GROUPS, 1, tgm.N_MELS, 1, dtype=torch.float32)
|
||||
for g in range(N_GROUPS):
|
||||
w[g, 0, g * 16:(g + 1) * 16, 0] = 1.0 / 16.0
|
||||
return w
|
||||
|
||||
|
||||
class DesignedGenreCNN(nn.Module):
|
||||
"""Conv(fixed) -> (mean, std, flux) -> Linear(fitted)."""
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__()
|
||||
self.bands = nn.Conv2d(1, N_GROUPS, (tgm.N_MELS, 1), stride=(1, 1), bias=False)
|
||||
self.bands.weight = nn.Parameter(band_group_weights(), requires_grad=False)
|
||||
self.head = nn.Linear(N_FEATURES, len(tgm.LABELS))
|
||||
|
||||
def pool(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# x: [N, G, 1, T]
|
||||
x = x.squeeze(2) # [N, G, T]
|
||||
mean = x.mean(dim=2) # [N, G]
|
||||
var = torch.clamp((x * x).mean(dim=2) - mean * mean, min=0.0)
|
||||
std = torch.sqrt(var + 1e-8) # [N, G]
|
||||
flux = torch.abs(x[:, :, 1:] - x[:, :, :-1]).mean(dim=2)
|
||||
return torch.cat([mean, std, flux], dim=1) # [N, 24]
|
||||
|
||||
def embed(self, x: torch.Tensor) -> torch.Tensor:
|
||||
# Invarianza al livello: il bridge normalizza con una costante globale, ma il
|
||||
# volume di riproduzione sposta tutto lo spettro. Sottrarre la media della
|
||||
# finestra rende il modello insensibile a un guadagno costante.
|
||||
return self.pool(self.bands(x - x.mean()))
|
||||
|
||||
def forward(self, x: torch.Tensor) -> torch.Tensor:
|
||||
return torch.softmax(self.head(self.embed(x)), dim=1)
|
||||
|
||||
|
||||
def features_of(audio: np.ndarray) -> np.ndarray:
|
||||
"""Run the bridge front-end on a raw clip and return the 24 pooled features."""
|
||||
model = DesignedGenreCNN()
|
||||
ten = torch.from_numpy(tgm.tensor_frontend(audio, SR, MEAN, STD))
|
||||
with torch.no_grad():
|
||||
return model.embed(ten).numpy()[0]
|
||||
|
||||
|
||||
# --------------------------------------------------------------------------- #
|
||||
# Closed-form head
|
||||
# --------------------------------------------------------------------------- #
|
||||
def fit_head(features: np.ndarray, gain: float = 2.5) -> tuple[np.ndarray, np.ndarray]:
|
||||
"""Minimum-norm least-squares fit of the 5 reference rows to one-hot targets.
|
||||
|
||||
Closed form, deterministic, no iterative training. `gain` sharpens the logits:
|
||||
the minimum-norm solution is deliberately small, and without a gain the softmax
|
||||
would stay close to uniform.
|
||||
"""
|
||||
x = np.concatenate([features, np.ones((features.shape[0], 1), dtype=np.float64)], axis=1)
|
||||
y = np.eye(len(tgm.LABELS))
|
||||
solution, *_ = np.linalg.lstsq(x, y, rcond=None) # [features+1, classes]
|
||||
w = solution[:-1].T.astype(np.float32) * gain
|
||||
b = solution[-1].astype(np.float32) * gain
|
||||
return w, b
|
||||
|
||||
|
||||
def build_references(seed: int = 11) -> tuple[np.ndarray, list[str]]:
|
||||
rows = []
|
||||
used = []
|
||||
for label in tgm.LABELS:
|
||||
lo, hi = BPM_RANGE[label]
|
||||
bpm = 0.5 * (lo + hi)
|
||||
audio = synth(label, bpm, 6.0, seed)
|
||||
rows.append(features_of(audio))
|
||||
used.append(f"{label}@{bpm:.0f}bpm")
|
||||
return np.vstack(rows), used
|
||||
|
||||
|
||||
def main() -> int:
|
||||
parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
|
||||
parser.add_argument("--output-dir", type=Path, default=Path(__file__).resolve().parents[1] / "bin")
|
||||
parser.add_argument("--validate", action="store_true", help="classify unseen renders")
|
||||
parser.add_argument("--seed", type=int, default=11)
|
||||
args = parser.parse_args()
|
||||
|
||||
features, used = build_references(args.seed)
|
||||
print("PROTOTYPES: " + ", ".join(used), flush=True)
|
||||
w, b = fit_head(features)
|
||||
|
||||
model = DesignedGenreCNN()
|
||||
with torch.no_grad():
|
||||
model.head.weight.copy_(torch.from_numpy(w))
|
||||
model.head.bias.copy_(torch.from_numpy(b))
|
||||
model.eval()
|
||||
|
||||
# Sanity: the references must come back as themselves (head applied to the
|
||||
# pooled features, since `features` is already past the convolution).
|
||||
with torch.no_grad():
|
||||
probs = model.head(torch.from_numpy(features)).softmax(dim=1)
|
||||
print("REFERENCE FIT:")
|
||||
for i, label in enumerate(tgm.LABELS):
|
||||
pred = tgm.LABELS[int(probs[i].argmax())]
|
||||
print(f" {label:11s} -> {pred:11s} p={probs[i].max():.3f}")
|
||||
|
||||
if args.validate:
|
||||
print("UNSEEN RENDERS:")
|
||||
ok = 0
|
||||
total = 0
|
||||
for label in tgm.LABELS:
|
||||
lo, hi = BPM_RANGE[label]
|
||||
for frac in (0.25, 0.75):
|
||||
bpm = lo + (hi - lo) * frac
|
||||
audio = synth(label, bpm, 6.0, args.seed + 100 + total)
|
||||
f = features_of(audio).reshape(1, -1)
|
||||
with torch.no_grad():
|
||||
p = model.head(torch.from_numpy(f)).softmax(dim=1).numpy()[0]
|
||||
pred = tgm.LABELS[int(np.argmax(p))]
|
||||
ok += pred == label
|
||||
total += 1
|
||||
print(f" {label:11s} {bpm:6.1f}bpm seed={args.seed + 100 + total - 1} -> {pred:11s} p={p.max():.3f}")
|
||||
print(f"VALIDATION: {ok}/{total}")
|
||||
if ok < total * 0.6:
|
||||
print("WARNING: too few unseen renders classified correctly; this model is a scaffold.")
|
||||
|
||||
out = args.output_dir
|
||||
out.mkdir(parents=True, exist_ok=True)
|
||||
onnx_path = out / "genre_model.onnx"
|
||||
json_path = out / "genre_model.json"
|
||||
example = torch.zeros(1, 1, tgm.N_MELS, tgm.FRAMES, dtype=torch.float32)
|
||||
torch.onnx.export(
|
||||
model, example, str(onnx_path),
|
||||
input_names=["logmel"], output_names=["probabilities"],
|
||||
opset_version=17, dynamo=False,
|
||||
)
|
||||
with torch.no_grad():
|
||||
check = model(example).numpy()[0]
|
||||
meta = {
|
||||
"labels": tgm.LABELS,
|
||||
"mean": MEAN,
|
||||
"std": STD,
|
||||
"version": tgm.VERSION,
|
||||
"source": "designed",
|
||||
"frontend": {
|
||||
"fft": 2048, "hann": "periodic", "hop": "sr//100",
|
||||
"padding": "causal-left-N-hop", "power": "abs(rfft)^2",
|
||||
"mel": "HTK", "bands": tgm.N_MELS, "fmin": 30, "fmax": 11000,
|
||||
"area": "divide by full support width in fractional bins", "log_floor": 1e-10,
|
||||
},
|
||||
}
|
||||
json_path.write_text(json.dumps(meta, indent=2), encoding="utf-8")
|
||||
print(f"MODEL={onnx_path.resolve()} ({onnx_path.stat().st_size} bytes)")
|
||||
print(f"JSON={json_path.resolve()}")
|
||||
print(f"SANITY: sum={check.sum():.6f} max={check.max():.3f} label={tgm.LABELS[int(check.argmax())]}")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in new issue
Block a user