1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
|
"""Export PretrainedSED frame_mn10_strong_1.pt to the C library's weight blob.
Requires torch, torchaudio, and numpy; no upstream checkout is needed.
Architecture and frontend follow PretrainedSED revision
1aa47e482f7e89904cba2338999345025d8b4e36 (models/frame_mn and
models/frame_passt/preprocess.py). Only the strong (frame-level) head is used.
"""
import argparse
import hashlib
from pathlib import Path
import struct
import tempfile
import urllib.request
URL = ("https://github.com/fschmid56/PretrainedSED/releases/download/"
"v0.0.1/frame_mn10_strong_1.pt")
CHECKPOINT_SHA256 = "4a0fe320d5369987b772394c51881fb20a602967e6842f72f3e5c8181065ece7"
def export_tensors(checkpoint):
import torch
import torch.nn.functional as F
from torchaudio.compliance.kaldi import get_mel_banks
# weights_only avoids arbitrary pickle code execution. Pin the checkpoint
# digest as well: other MobileNet variants are not supported by the C code.
digest = hashlib.sha256(checkpoint.read_bytes()).hexdigest()
if digest != CHECKPOINT_SHA256:
raise ValueError(f"unexpected checkpoint SHA-256: {digest}")
state = torch.load(checkpoint, map_location="cpu", weights_only=True)
tensors = {}
def add(name, tensor):
if not torch.isfinite(tensor).all():
raise ValueError(f"nonfinite tensor: {name}")
tensors[name] = tensor.detach().cpu().numpy().astype("<f4").tobytes()
def conv(name, prefix):
# Upstream BatchNorm2d uses eps=0.001. Convolutions have no bias.
weight = state[prefix + ".0.weight"]
bn = prefix + ".1."
scale = state[bn + "weight"] / torch.sqrt(state[bn + "running_var"] + 0.001)
add(name + "_w", weight * scale[:, None, None, None])
add(name + "_b", state[bn + "bias"] - state[bn + "running_mean"] * scale)
add("window", torch.hann_window(400, periodic=False))
# Evaluation fmax is sr/2 - fmax_aug_range/2 = 7000 Hz, not 8000.
mel, _ = get_mel_banks(128, 512, 16000, 0.0, 7000.0,
vtln_low=100.0, vtln_high=-500.0, vtln_warp_factor=1.0)
add("mel", F.pad(mel, (0, 1)))
base = "model.frame_mn.features."
conv("c0", base + "0")
se_blocks = {4, 5, 6, 11, 12, 13, 14, 15}
for index in range(1, 16):
prefix = base + f"{index}.block."
layer = 0
if index != 1:
conv(f"b{index}e", prefix + str(layer))
layer += 1
conv(f"b{index}d", prefix + str(layer))
layer += 1
if index in se_blocks:
se = prefix + f"{layer}.conc_se_layers.0."
for fc in (1, 2):
add(f"b{index}s{fc}w", state[se + f"fc{fc}.weight"])
add(f"b{index}s{fc}b", state[se + f"fc{fc}.bias"])
layer += 1
conv(f"b{index}p", prefix + str(layer))
conv("cf", base + "16")
if tuple(state["strong_head.weight"].shape) != (447, 960):
raise ValueError("expected a 447-class, 960-channel strong head")
add("out_w", state["strong_head.weight"])
add("out_b", state["strong_head.bias"])
return tensors
def fnv1a(data):
digest = 0x811C9DC5
for byte in data:
digest = ((digest ^ byte) * 0x01000193) & 0xFFFFFFFF
return digest
def pack_blob(tensors):
# model_blob.c: 16-byte header, 36-byte entries, 64-byte aligned payloads.
blob = bytearray(16 + 36 * len(tensors))
for index, (name, payload) in enumerate(tensors.items()):
blob.extend(bytes((-len(blob)) % 64))
offset = len(blob)
blob.extend(payload)
struct.pack_into("<24sIII", blob, 16 + 36 * index,
name.encode("ascii"), offset, len(payload), fnv1a(payload))
blob.extend(bytes((-len(blob)) % 64))
struct.pack_into("<4sIII", blob, 0, b"SDC1", 1, len(tensors), len(blob))
return blob
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--checkpoint", type=Path,
help="local upstream checkpoint; otherwise download to a temporary directory")
parser.add_argument("--output", type=Path, default=Path("models/framesed_weights.bin"))
args = parser.parse_args()
with tempfile.TemporaryDirectory(prefix="framesed-export-") as directory:
checkpoint = args.checkpoint
if checkpoint is None:
checkpoint = Path(directory) / "frame_mn10_strong_1.pt"
print(f"Downloading {URL}")
urllib.request.urlretrieve(URL, checkpoint)
blob = pack_blob(export_tensors(checkpoint))
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_bytes(blob)
print(f"Wrote {args.output}: {len(blob)} bytes, SHA-256 {hashlib.sha256(blob).hexdigest()}")
if __name__ == "__main__":
main()
|