aboutsummaryrefslogtreecommitdiff
path: root/devtools/export_framesed_weights.py
blob: cdce3a956e3e41e4963f8bba8a8161b8dfe78fe2 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
"""Export PretrainedSED frame_mn10_strong_1.pt to the C library's weight blob.

Requires torch, torchaudio, and numpy; no upstream checkout is needed.
Architecture and frontend follow PretrainedSED revision
1aa47e482f7e89904cba2338999345025d8b4e36 (models/frame_mn and
models/frame_passt/preprocess.py). Only the strong (frame-level) head is used.
"""

import argparse
import hashlib
from pathlib import Path
import struct
import tempfile
import urllib.request

URL = ("https://github.com/fschmid56/PretrainedSED/releases/download/"
       "v0.0.1/frame_mn10_strong_1.pt")
CHECKPOINT_SHA256 = "4a0fe320d5369987b772394c51881fb20a602967e6842f72f3e5c8181065ece7"


def export_tensors(checkpoint):
    import torch
    import torch.nn.functional as F
    from torchaudio.compliance.kaldi import get_mel_banks

    # weights_only avoids arbitrary pickle code execution. Pin the checkpoint
    # digest as well: other MobileNet variants are not supported by the C code.
    digest = hashlib.sha256(checkpoint.read_bytes()).hexdigest()
    if digest != CHECKPOINT_SHA256:
        raise ValueError(f"unexpected checkpoint SHA-256: {digest}")
    state = torch.load(checkpoint, map_location="cpu", weights_only=True)
    tensors = {}

    def add(name, tensor):
        if not torch.isfinite(tensor).all():
            raise ValueError(f"nonfinite tensor: {name}")
        tensors[name] = tensor.detach().cpu().numpy().astype("<f4").tobytes()

    def conv(name, prefix):
        # Upstream BatchNorm2d uses eps=0.001. Convolutions have no bias.
        weight = state[prefix + ".0.weight"]
        bn = prefix + ".1."
        scale = state[bn + "weight"] / torch.sqrt(state[bn + "running_var"] + 0.001)
        add(name + "_w", weight * scale[:, None, None, None])
        add(name + "_b", state[bn + "bias"] - state[bn + "running_mean"] * scale)

    add("window", torch.hann_window(400, periodic=False))
    # Evaluation fmax is sr/2 - fmax_aug_range/2 = 7000 Hz, not 8000.
    mel, _ = get_mel_banks(128, 512, 16000, 0.0, 7000.0,
                           vtln_low=100.0, vtln_high=-500.0, vtln_warp_factor=1.0)
    add("mel", F.pad(mel, (0, 1)))
    base = "model.frame_mn.features."
    conv("c0", base + "0")
    se_blocks = {4, 5, 6, 11, 12, 13, 14, 15}
    for index in range(1, 16):
        prefix = base + f"{index}.block."
        layer = 0
        if index != 1:
            conv(f"b{index}e", prefix + str(layer))
            layer += 1
        conv(f"b{index}d", prefix + str(layer))
        layer += 1
        if index in se_blocks:
            se = prefix + f"{layer}.conc_se_layers.0."
            for fc in (1, 2):
                add(f"b{index}s{fc}w", state[se + f"fc{fc}.weight"])
                add(f"b{index}s{fc}b", state[se + f"fc{fc}.bias"])
            layer += 1
        conv(f"b{index}p", prefix + str(layer))
    conv("cf", base + "16")
    if tuple(state["strong_head.weight"].shape) != (447, 960):
        raise ValueError("expected a 447-class, 960-channel strong head")
    add("out_w", state["strong_head.weight"])
    add("out_b", state["strong_head.bias"])
    return tensors


def fnv1a(data):
    digest = 0x811C9DC5
    for byte in data:
        digest = ((digest ^ byte) * 0x01000193) & 0xFFFFFFFF
    return digest


def pack_blob(tensors):
    # model_blob.c: 16-byte header, 36-byte entries, 64-byte aligned payloads.
    blob = bytearray(16 + 36 * len(tensors))
    for index, (name, payload) in enumerate(tensors.items()):
        blob.extend(bytes((-len(blob)) % 64))
        offset = len(blob)
        blob.extend(payload)
        struct.pack_into("<24sIII", blob, 16 + 36 * index,
                         name.encode("ascii"), offset, len(payload), fnv1a(payload))
    blob.extend(bytes((-len(blob)) % 64))
    struct.pack_into("<4sIII", blob, 0, b"SDC1", 1, len(tensors), len(blob))
    return blob


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument("--checkpoint", type=Path,
                        help="local upstream checkpoint; otherwise download to a temporary directory")
    parser.add_argument("--output", type=Path, default=Path("models/framesed_weights.bin"))
    args = parser.parse_args()
    with tempfile.TemporaryDirectory(prefix="framesed-export-") as directory:
        checkpoint = args.checkpoint
        if checkpoint is None:
            checkpoint = Path(directory) / "frame_mn10_strong_1.pt"
            print(f"Downloading {URL}")
            urllib.request.urlretrieve(URL, checkpoint)
        blob = pack_blob(export_tensors(checkpoint))
    args.output.parent.mkdir(parents=True, exist_ok=True)
    args.output.write_bytes(blob)
    print(f"Wrote {args.output}: {len(blob)} bytes, SHA-256 {hashlib.sha256(blob).hexdigest()}")


if __name__ == "__main__":
    main()