diff options
| -rw-r--r-- | .gitignore | 7 | ||||
| -rw-r--r-- | LICENSE | 20 | ||||
| -rw-r--r-- | Makefile | 125 | ||||
| -rw-r--r-- | README.md | 119 | ||||
| -rw-r--r-- | devtools/export_framesed_classes.py | 46 | ||||
| -rw-r--r-- | devtools/export_framesed_weights.py | 118 | ||||
| -rw-r--r-- | framesed/README.md | 19 | ||||
| -rw-r--r-- | framesed/classes.csv | 448 | ||||
| -rw-r--r-- | framesed/framesed.c | 1102 | ||||
| -rw-r--r-- | framesed/framesed.h | 100 | ||||
| -rw-r--r-- | framesed/framesed_classes.inc | 448 | ||||
| -rw-r--r-- | framesed/framesed_policy.inc | 85 | ||||
| -rw-r--r-- | model_blob.c | 111 | ||||
| -rw-r--r-- | model_blob.h | 36 | ||||
| -rw-r--r-- | testdata/framesed_reference.bin | bin | 0 -> 9852 bytes | |||
| -rw-r--r-- | tests/test_classes.c | 124 | ||||
| -rw-r--r-- | tests/test_cli.py | 76 | ||||
| -rw-r--r-- | tests/test_framesed.c | 280 | ||||
| -rw-r--r-- | tests/test_policy.c | 140 | ||||
| -rw-r--r-- | tools/framesed.c | 138 | ||||
| -rw-r--r-- | wav.c | 138 | ||||
| -rw-r--r-- | wav.h | 23 |
22 files changed, 3703 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ec13c96 --- /dev/null +++ b/.gitignore @@ -0,0 +1,7 @@ +/build/ +/libframesed.a +/models/*.bin +/.venv/ +__pycache__/ +*.pyc +.DS_Store @@ -0,0 +1,20 @@ +MIT License + +Copyright (c) 2026 The framesed authors + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal in +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of +the Software, and to permit persons to whom the Software is furnished to do so, +subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR +COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER +IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..d451190 --- /dev/null +++ b/Makefile @@ -0,0 +1,125 @@ +CC ?= cc +AR ?= ar +PYTHON ?= python3 +MODE ?= release +ARCH_FLAGS ?= +TARGET ?= libframesed.a +OBJDIR := build/obj +BUILD_CONFIG := build/.build-config + +CPPFLAGS += -I. -Iframesed +WARNING_FLAGS ?= -Wall -Wextra -Werror=implicit-function-declaration +ifeq ($(MODE),release) +OPT_FLAGS ?= -O3 -DNDEBUG +else ifeq ($(MODE),debug) +OPT_FLAGS ?= -O0 -g3 -fno-omit-frame-pointer +else ifeq ($(MODE),profile) +OPT_FLAGS ?= -O2 -g -fno-omit-frame-pointer +else +$(error MODE must be release, debug, or profile) +endif +CFLAGS ?= $(OPT_FLAGS) $(WARNING_FLAGS) -std=gnu11 $(ARCH_FLAGS) +ARFLAGS := rcs + +ifeq ($(shell uname -s),Darwin) +AUDIO_BLAS ?= accelerate +else +AUDIO_BLAS ?= none +endif +ifeq ($(AUDIO_BLAS),accelerate) +CPPFLAGS += -DFRAMESED_ACCELERATE -DACCELERATE_NEW_LAPACK +AUDIO_LDLIBS := -framework Accelerate +else ifeq ($(AUDIO_BLAS),openblas) +CPPFLAGS += -DFRAMESED_OPENBLAS $(shell pkg-config --cflags openblas 2>/dev/null) +AUDIO_LDLIBS := $(shell pkg-config --libs openblas 2>/dev/null || echo -lopenblas) +else ifneq ($(AUDIO_BLAS),none) +$(error AUDIO_BLAS must be none, accelerate, or openblas) +endif +BASE_LDLIBS = -lm -lpthread $(AUDIO_LDLIBS) $(LDLIBS) + +LIB_OBJECTS := $(OBJDIR)/framesed/framesed.o $(OBJDIR)/model_blob.o +CLI_OBJECTS := $(OBJDIR)/tools/framesed.o $(OBJDIR)/wav.o +TEST_OBJECTS := $(addprefix $(OBJDIR)/tests/,test_framesed.o test_classes.o test_policy.o) +OBJECTS := $(LIB_OBJECTS) $(CLI_OBJECTS) $(TEST_OBJECTS) +FRAMESED_TEST := build/tests/test_framesed +CLASS_TEST := build/tests/test_classes +POLICY_TEST := build/tests/test_policy +FRAMESED_WEIGHTS ?= models/framesed_weights.bin +FRAMESED_REFERENCE ?= testdata/framesed_reference.bin +SANITIZER_CFLAGS ?= -O1 -g -Wall -Wextra -std=gnu11 -fsanitize=address,undefined \ + -fno-omit-frame-pointer -fno-sanitize-recover=all +SANITIZER_LDFLAGS ?= -fsanitize=address,undefined + +.PHONY: all framesed tools cli libframesed release debug profile test test-framesed check \ + check-sanitize test-classes test-policy test-cli check-classes regenerate-classes clean FORCE + +all: $(TARGET) build/framesed +framesed tools cli: build/framesed +libframesed: $(TARGET) + +release debug profile: + $(MAKE) MODE=$@ all + +$(TARGET): $(LIB_OBJECTS) + $(RM) $@ + $(AR) $(ARFLAGS) $@ $^ + +build/framesed: $(CLI_OBJECTS) $(TARGET) + $(CC) $(CFLAGS) $(CLI_OBJECTS) $(TARGET) $(LDFLAGS) $(BASE_LDLIBS) -o $@ + +$(FRAMESED_TEST) $(CLASS_TEST): build/tests/%: $(OBJDIR)/tests/%.o $(TARGET) + @mkdir -p $(dir $@) + $(CC) $(CFLAGS) $< $(TARGET) $(LDFLAGS) $(BASE_LDLIBS) -o $@ + +# The policy test includes framesed.c to exercise private score-to-event contracts. +$(POLICY_TEST): $(OBJDIR)/tests/test_policy.o $(OBJDIR)/model_blob.o + @mkdir -p $(dir $@) + $(CC) $(CFLAGS) $^ $(LDFLAGS) $(BASE_LDLIBS) -o $@ + +test check: test-framesed test-classes test-policy test-cli check-classes + +test-framesed: $(FRAMESED_TEST) $(FRAMESED_WEIGHTS) $(FRAMESED_REFERENCE) + $(FRAMESED_TEST) "$(FRAMESED_WEIGHTS)" "$(FRAMESED_REFERENCE)" + +test-classes: $(CLASS_TEST) $(FRAMESED_WEIGHTS) + $(CLASS_TEST) "$(FRAMESED_WEIGHTS)" + +test-policy: $(POLICY_TEST) + $(POLICY_TEST) + +test-cli: build/framesed $(FRAMESED_WEIGHTS) + $(PYTHON) tests/test_cli.py "$(FRAMESED_WEIGHTS)" + +regenerate-classes: + $(PYTHON) devtools/export_framesed_classes.py framesed/classes.csv framesed/framesed_classes.inc + +check-classes: + $(PYTHON) devtools/export_framesed_classes.py framesed/classes.csv framesed/framesed_classes.inc --check + +check-sanitize: + $(MAKE) CFLAGS='$(SANITIZER_CFLAGS)' LDFLAGS='$(SANITIZER_LDFLAGS)' all test + +$(OBJECTS): $(BUILD_CONFIG) + +$(OBJDIR)/%.o: %.c + @mkdir -p $(dir $@) + $(CC) $(CPPFLAGS) $(CFLAGS) -MMD -MP -c $< -o $@ + +# Rebuild on configuration changes, but leave unchanged builds incremental. +$(BUILD_CONFIG): FORCE | build + @printf '%s\n' 'MODE=$(MODE)' 'AUDIO_BLAS=$(AUDIO_BLAS)' \ + 'CC=$(CC)' 'AR=$(AR)' 'ARFLAGS=$(ARFLAGS)' \ + 'CPPFLAGS=$(CPPFLAGS)' 'CFLAGS=$(CFLAGS)' \ + 'LDFLAGS=$(LDFLAGS)' 'BASE_LDLIBS=$(BASE_LDLIBS)' > $@.new + @cmp -s $@.new $@ || cp $@.new $@ + @rm -f $@.new + +FORCE: + +build: + mkdir -p $@ + +clean: + rm -rf build $(TARGET) + +-include $(OBJECTS:.o=.d) diff --git a/README.md b/README.md new file mode 100644 index 0000000..00ae9f4 --- /dev/null +++ b/README.md @@ -0,0 +1,119 @@ +# framesed + +Frame-level sound event detection using `PretrainedSED/frame_mn10`. + +## Build and run + +```sh +make # libframesed.a and build/framesed +make test # model parity, policies, threading, CLI, class metadata +build/framesed --weights models/framesed_weights.bin audio.wav +``` + +A C11 compiler, Make, and POSIX threads are required. Tests and metadata generation also require Python 3 (standard library only). The public API is `framesed/framesed.h`. + +## CLI + +```text +build/framesed [--all-classes] [--threads N] [--track-step SECONDS] --weights FILE INPUT.wav +build/framesed --list-classes +``` + +Input is 16-bit PCM WAV at 16 kHz; multiple channels are averaged to mono. +The CLI does not resample. `--threads` defaults to 1 and must be positive. +`--track-step` enables score tracks and must be at least 0.04 seconds; the +returned step rounds down to a multiple of 0.04 seconds. + +Default detection uses 59 caption groups from `framesed/framesed_policy.inc`. +`--all-classes` detects each of the 447 model classes independently, without +caption grouping or suppression. This includes speech and background/context +classes, not just sound effects. + +```sh +build/framesed --all-classes --weights models/framesed_weights.bin audio.wav +build/framesed --list-classes # tab-separated index, AudioSet ID, label; no weights needed +``` + +Class-mode gates are uniform: start score 0.50, release score 0.35, minimum +duration 0.24 s, allowed gap 0.32 s. They are not individually calibrated; +recognition quality varies by class. Tracks are returned for classes reaching +0.35, even when no event survives the duration gate. + +Output is one record per line, with times in seconds: + +```text +event START END SCORE GROUP LABEL +track TIME SCORE GROUP LABEL +``` + +`GROUP` is a caption-group index by default, or a model class index (0–446) +with `--all-classes`. Labels may contain spaces and commas. + +## API + +Set `framesed_options.all_classes = 1` for individual-class detection; zero or +NULL options select caption groups. `framesed_class_label(index)` and +`framesed_class_mid(index)` return model metadata, or NULL for an invalid index. +`framesed_group_label(index)` always refers to caption groups. + +With `keep_planes = 1`, class-mode results contain 447 probability rows in +model order, at 0.04 s resolution. `framesed_infer_s16()` returns raw logits for one +10-second window. Free detection results with `framesed_result_free()` and logits +with `framesed_logits_free()`. + +The vocabulary is in [`framesed/classes.csv`](framesed/classes.csv). Its indices follow +PretrainedSED's [`as_strong_train_classes`](https://github.com/fschmid56/PretrainedSED/blob/1aa47e482f7e89904cba2338999345025d8b4e36/data_util/audioset_classes.py); +AudioSet IDs come from its `hf_dataset_gen/metadata` label tables. Class order +must match the weights; editing labels does not teach the model new sounds. +`make regenerate-classes` generates the compiled table; `make check-classes` +checks that it matches the CSV. Neither command requires PyTorch. + +## Recreating the weights + +`models/framesed_weights.bin` is an inference export of PretrainedSED's +[`frame_mn10_strong_1.pt`](https://github.com/fschmid56/PretrainedSED/releases/download/v0.0.1/frame_mn10_strong_1.pt), +not a separately trained model. The exporter folds batch normalization into +convolution weights, includes the Hann window and Kaldi mel filterbank, and +keeps the 447-class strong head (the weak head is omitted). + +From the project root, in a Python environment with `torch`, `torchaudio`, and +`numpy` installed (use matching torch/torchaudio versions): + +```sh +python3 devtools/export_framesed_weights.py +make test +``` + +The script downloads and verifies the upstream checkpoint before writing +`models/framesed_weights.bin`. For offline use or a different output path: + +```sh +python3 devtools/export_framesed_weights.py \ + --checkpoint /path/to/frame_mn10_strong_1.pt --output /tmp/framesed_weights.bin +``` + +The exporter follows upstream revision `1aa47e482f7e89904cba2338999345025d8b4e36`. + +`devtools/export_framesed_classes.py` exports class metadata. + +## Build options + +- `make debug` / `make profile` / `make release` +- `make AUDIO_BLAS=none all test` for portable CPU inference +- `make AUDIO_BLAS=openblas all test` for OpenBLAS (requires OpenBLAS; + discovered through `pkg-config`) +- `make check-sanitize` for AddressSanitizer and UndefinedBehaviorSanitizer +- `make clean` removes build outputs, not weights or test data + +macOS defaults to Accelerate; other systems default to portable CPU. +Configuration and header changes automatically trigger rebuilds. +`FRAMESED_WEIGHTS` and `FRAMESED_REFERENCE` override test fixture paths. + +Link `libframesed.a` with the math and pthread libraries, plus Accelerate or OpenBLAS +when enabled. The WAV reader is used only by the CLI, not the library. + +## License + +MIT License. + +The model weights are `PretrainedSED/frame_mn10` by Floarian Schmid; see the [model repository](https://github.com/fschmid56/PretrainedSED) for details. diff --git a/devtools/export_framesed_classes.py b/devtools/export_framesed_classes.py new file mode 100644 index 0000000..e02752a --- /dev/null +++ b/devtools/export_framesed_classes.py @@ -0,0 +1,46 @@ +"""Generate C class metadata from the model-ordered index,mid,display_name CSV.""" + +import argparse +import csv +import json +from pathlib import Path + + +def render(path: Path) -> str: + with path.open(newline="", encoding="utf-8") as source: + rows = list(csv.DictReader(source)) + if len(rows) != 447: + raise ValueError("frame_mn10 requires exactly 447 classes") + mids = set() + labels = set() + lines = ["/* Model output order; regenerate with make regenerate-classes. */"] + for index, row in enumerate(rows): + mid, label = row["mid"], row["display_name"] + if int(row["index"]) != index: + raise ValueError("class indices must be contiguous and ordered from zero") + if not mid or not label or mid in mids or label in labels: + raise ValueError("class IDs and labels must be nonempty and unique") + if any(ord(c) < 32 for c in mid + label): + raise ValueError("class metadata must not contain control characters") + mids.add(mid) + labels.add(label) + lines.append(f" {{{json.dumps(mid)}, {json.dumps(label)}}},") + return "\n".join(lines) + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("labels", type=Path) + parser.add_argument("output", type=Path) + parser.add_argument("--check", action="store_true") + args = parser.parse_args() + content = render(args.labels) + if args.check: + if args.output.read_text(encoding="utf-8") != content: + parser.exit(1, f"{args.output}: regenerate with make regenerate-classes\n") + else: + args.output.write_text(content, encoding="utf-8") + + +if __name__ == "__main__": + main() diff --git a/devtools/export_framesed_weights.py b/devtools/export_framesed_weights.py new file mode 100644 index 0000000..cdce3a9 --- /dev/null +++ b/devtools/export_framesed_weights.py @@ -0,0 +1,118 @@ +"""Export PretrainedSED frame_mn10_strong_1.pt to the C library's weight blob. + +Requires torch, torchaudio, and numpy; no upstream checkout is needed. +Architecture and frontend follow PretrainedSED revision +1aa47e482f7e89904cba2338999345025d8b4e36 (models/frame_mn and +models/frame_passt/preprocess.py). Only the strong (frame-level) head is used. +""" + +import argparse +import hashlib +from pathlib import Path +import struct +import tempfile +import urllib.request + +URL = ("https://github.com/fschmid56/PretrainedSED/releases/download/" + "v0.0.1/frame_mn10_strong_1.pt") +CHECKPOINT_SHA256 = "4a0fe320d5369987b772394c51881fb20a602967e6842f72f3e5c8181065ece7" + + +def export_tensors(checkpoint): + import torch + import torch.nn.functional as F + from torchaudio.compliance.kaldi import get_mel_banks + + # weights_only avoids arbitrary pickle code execution. Pin the checkpoint + # digest as well: other MobileNet variants are not supported by the C code. + digest = hashlib.sha256(checkpoint.read_bytes()).hexdigest() + if digest != CHECKPOINT_SHA256: + raise ValueError(f"unexpected checkpoint SHA-256: {digest}") + state = torch.load(checkpoint, map_location="cpu", weights_only=True) + tensors = {} + + def add(name, tensor): + if not torch.isfinite(tensor).all(): + raise ValueError(f"nonfinite tensor: {name}") + tensors[name] = tensor.detach().cpu().numpy().astype("<f4").tobytes() + + def conv(name, prefix): + # Upstream BatchNorm2d uses eps=0.001. Convolutions have no bias. + weight = state[prefix + ".0.weight"] + bn = prefix + ".1." + scale = state[bn + "weight"] / torch.sqrt(state[bn + "running_var"] + 0.001) + add(name + "_w", weight * scale[:, None, None, None]) + add(name + "_b", state[bn + "bias"] - state[bn + "running_mean"] * scale) + + add("window", torch.hann_window(400, periodic=False)) + # Evaluation fmax is sr/2 - fmax_aug_range/2 = 7000 Hz, not 8000. + mel, _ = get_mel_banks(128, 512, 16000, 0.0, 7000.0, + vtln_low=100.0, vtln_high=-500.0, vtln_warp_factor=1.0) + add("mel", F.pad(mel, (0, 1))) + base = "model.frame_mn.features." + conv("c0", base + "0") + se_blocks = {4, 5, 6, 11, 12, 13, 14, 15} + for index in range(1, 16): + prefix = base + f"{index}.block." + layer = 0 + if index != 1: + conv(f"b{index}e", prefix + str(layer)) + layer += 1 + conv(f"b{index}d", prefix + str(layer)) + layer += 1 + if index in se_blocks: + se = prefix + f"{layer}.conc_se_layers.0." + for fc in (1, 2): + add(f"b{index}s{fc}w", state[se + f"fc{fc}.weight"]) + add(f"b{index}s{fc}b", state[se + f"fc{fc}.bias"]) + layer += 1 + conv(f"b{index}p", prefix + str(layer)) + conv("cf", base + "16") + if tuple(state["strong_head.weight"].shape) != (447, 960): + raise ValueError("expected a 447-class, 960-channel strong head") + add("out_w", state["strong_head.weight"]) + add("out_b", state["strong_head.bias"]) + return tensors + + +def fnv1a(data): + digest = 0x811C9DC5 + for byte in data: + digest = ((digest ^ byte) * 0x01000193) & 0xFFFFFFFF + return digest + + +def pack_blob(tensors): + # model_blob.c: 16-byte header, 36-byte entries, 64-byte aligned payloads. + blob = bytearray(16 + 36 * len(tensors)) + for index, (name, payload) in enumerate(tensors.items()): + blob.extend(bytes((-len(blob)) % 64)) + offset = len(blob) + blob.extend(payload) + struct.pack_into("<24sIII", blob, 16 + 36 * index, + name.encode("ascii"), offset, len(payload), fnv1a(payload)) + blob.extend(bytes((-len(blob)) % 64)) + struct.pack_into("<4sIII", blob, 0, b"SDC1", 1, len(tensors), len(blob)) + return blob + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--checkpoint", type=Path, + help="local upstream checkpoint; otherwise download to a temporary directory") + parser.add_argument("--output", type=Path, default=Path("models/framesed_weights.bin")) + args = parser.parse_args() + with tempfile.TemporaryDirectory(prefix="framesed-export-") as directory: + checkpoint = args.checkpoint + if checkpoint is None: + checkpoint = Path(directory) / "frame_mn10_strong_1.pt" + print(f"Downloading {URL}") + urllib.request.urlretrieve(URL, checkpoint) + blob = pack_blob(export_tensors(checkpoint)) + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_bytes(blob) + print(f"Wrote {args.output}: {len(blob)} bytes, SHA-256 {hashlib.sha256(blob).hexdigest()}") + + +if __name__ == "__main__": + main() diff --git a/framesed/README.md b/framesed/README.md new file mode 100644 index 0000000..00d60ca --- /dev/null +++ b/framesed/README.md @@ -0,0 +1,19 @@ +# FRAMESED sound events + +This directory contains the standalone 16 kHz sound-event model and the +deterministic policy that turns class scores into 59 caption groups. Detection +can instead return all 447 model classes independently. Both modes support +unsuppressed score tracks and per-frame probability planes. + +Build and run from the project root: + +```sh +make build/framesed +build/framesed --weights models/framesed_weights.bin audio.wav +build/framesed --all-classes --weights models/framesed_weights.bin audio.wav +build/framesed --list-classes +``` + +The public API is `framesed.h`. Input is mono signed 16-bit PCM at 16 kHz; callers +own decoding and resampling. See the [project reference](../README.md) for +class-mode thresholds, metadata, and output formats. diff --git a/framesed/classes.csv b/framesed/classes.csv new file mode 100644 index 0000000..c518714 --- /dev/null +++ b/framesed/classes.csv @@ -0,0 +1,448 @@ +index,mid,display_name +0,/m/07q2z82,"Accelerating, revving, vroom" +1,/m/0gvgw0,Air brake +2,/m/025wky1,Air conditioning +3,/m/05x_td,"Air horn, truck horn" +4,/m/0k5j,Aircraft +5,/m/014yck,Aircraft engine +6,/m/07pp_mv,Alarm +7,/m/046dlr,Alarm clock +8,/t/dd00144,Alert +9,/m/012n7d,Ambulance (siren) +10,/m/0jbk,Animal +11,/m/028ght,Applause +12,/m/0dl83,Arrow +13,/m/0_1c,Artillery fire +14,/t/dd00142,Audio logo +15,/m/0261r1,Babbling +16,/t/dd00002,"Baby cry, infant cry" +17,/t/dd00001,Baby laughter +18,/m/093_4n,Background noise +19,/m/07pws3f,Bang +20,/m/05tny_,Bark +21,/m/018w8,Basketball bounce +22,/m/01j2bj,Bathroom sounds +23,/m/03dnzn,Bathtub (filling or washing) +24,/m/04gy_2,Battle cry +25,/m/01h3n,"Bee, wasp, etc." +26,/m/02fs_r,"Beep, bleep" +27,/m/0395lw,Bell +28,/m/07q4ntr,Bellow +29,/m/07sq110,Belly laugh +30,/m/0gy1t2s,Bicycle bell +31,/m/0199g,"Bicycle, tricycle" +32,/m/015p6,Bird +33,/m/05_wcq,"Bird flight, flapping wings" +34,/m/020bb7,"Bird vocalization, bird call, bird song" +35,/m/07pdhp0,Biting +36,/m/07q0h5t,Bleat +37,/m/02pjr4,"Blender, food processor" +38,/m/019jd,"Boat, Water vehicle" +39,/m/0dv3j,Boiling +40,/t/dd00121,Boing +41,/m/04v5dt,Booing +42,/m/07qqyl4,Boom +43,/m/07pqn27,Bouncing +44,/m/07rc7d9,Bow-wow +45,/m/07pc8lb,Breaking +46,/m/0lyf6,Breathing +47,/t/dd00138,Brief tone +48,/m/03q5_w,"Burping, eructation" +49,/m/07qsvvw,"Burst, pop" +50,/m/01bjv,Bus +51,/m/01z47d,Busy signal +52,/m/07pjwq1,Buzz +53,/m/030rvx,Buzzer +54,/m/07szfh9,Cacophony +55,/m/0dv5r,Camera +56,/m/01z5f,"Canidae, wild dogs, wolves" +57,/m/073cg4,Cap gun +58,/m/0k4j,Car +59,/m/02mfyn,Car alarm +60,/t/dd00134,Car passing by +61,/m/08dckq,"Carbon monoxide detector, CO detector" +62,/m/018p4k,Cart +63,/m/024dl,Cash register +64,/m/01yrx,Cat +65,/m/07r81j2,Caterwaul +66,/m/01xq0k1,"Cattle, bovinae" +67,/m/07r5c2p,Caw +68,/m/01sb50,"Cellphone buzz, vibrating alert" +69,/m/0269r2s,Chain +70,/m/01j4z9,Chainsaw +71,/m/0150b9,Change ringing (campanology) +72,/t/dd00123,"Channel, environment and background" +73,/m/02bk07,Chant +74,/m/053hz1,Cheering +75,/m/03cczk,"Chewing, mastication" +76,/m/09b5t,"Chicken, rooster" +77,/t/dd00005,Child singing +78,/m/0ytgt,"Child speech, kid speaking" +79,/t/dd00013,Children playing +80,/t/dd00135,Children shouting +81,/m/0f8s22,Chime +82,/m/02021,Chipmunk +83,/m/0hdsk,Chirp tone +84,/m/07pggtn,"Chirp, tweet" +85,/m/0l14jd,Choir +86,/m/07pczhz,Chop +87,/m/07pn_8q,Chopping (food) +88,/m/07m2kt,Chorus effect +89,/m/07rgt08,"Chuckle, chortle" +90,/m/03w41f,Church bell +91,/m/0dgbq,Civil defense siren +92,/m/07rv4dm,Clang +93,/m/0l15bq,Clapping +94,/m/07qmpdm,Clatter +95,/m/07rwm0c,Clickety-clack +96,/m/07qc9xj,Clicking +97,/m/07rv9rh,Clip-clop +98,/m/01x3z,Clock +99,/m/07st89h,Cluck +100,/t/dd00108,Clunk +101,/m/0242l,Coin (dropping) +102,/m/01m2v,Computer keyboard +103,/m/01h8n0,Conversation +104,/m/07r_25d,Coo +105,/m/01b_21,Cough +106,/m/0239kh,Cowbell +107,/m/07qs1cx,Crack +108,/m/07pzfmf,Crackle +109,/m/07qh7jl,Creak +110,/m/09xqv,Cricket +111,/m/07st88b,Croak +112,/m/0hgq8df,Crockery breaking and smashing +113,/m/04s8yn,Crow +114,/m/03qtwd,Crowd +115,/m/07qn5dc,"Crowing, cock-a-doodle-doo" +116,/t/dd00112,"Crumpling, crinkling" +117,/m/07s12q4,Crunch +118,/m/07plct2,Crushing +119,/m/0463cq4,"Crying, sobbing" +120,/m/0642b4,Cupboard open or close +121,/m/023pjk,"Cutlery, silverware" +122,/t/dd00110,Deformable shell +123,/m/08j51y,"Dental drill, dentist's drill" +124,/m/015jpf,Dial tone +125,/m/0160x5,Digestive +126,/m/07phxs1,Ding +127,/m/07r67yg,Ding-dong +128,/m/04brg2,"Dishes, pots, and pans" +129,/m/0g12c5,Distortion +130,/m/0bt9lr,Dog +131,/m/068hy,"Domestic animals, pets" +132,/t/dd00147,"Dong, bong" +133,/m/0ffhf,"Donkey, ass" +134,/m/02dgv,Door +135,/m/03wwcy,Doorbell +136,/m/0fqfqc,Drawer open or close +137,/m/01d380,Drill +138,/m/07r5v4s,Drip +139,/m/04179zz,Duck call (hunting tool) +140,/m/09ddx,"Ducks, geese, waterfowl" +141,/m/01jnbd,Echo +142,/m/02rr_,Effects unit +143,/m/0c1tlg,"Electric rotor drone, quadcopter" +144,/m/02g901,"Electric shaver, electric razor" +145,/m/04fgwm,Electric toothbrush +146,/m/0b_fwt,Electronic tuner +147,/m/03j1ly,Emergency vehicle +148,/m/02mk9,Engine +149,/m/01h82_,Engine knocking +150,/t/dd00130,Engine starting +151,/m/06_y0by,Environmental noise +152,/m/098_xr,Error signal +153,/m/07pxg6y,Eruption +154,/m/014zdl,Explosion +155,/m/02_nn,Fart +156,/t/dd00004,Female singing +157,/m/02zsn,"Female speech, woman speaking" +158,/m/02p01q,Filing (rasp) +159,/m/07p7b8y,Fill (with liquid) +160,/m/025_jnm,Finger snapping +161,/m/02_41,Fire +162,/m/0c3f7m,Fire alarm +163,/m/012ndj,"Fire engine, fire truck (siren)" +164,/g/122z_qxw,Firecracker +165,/m/0g6b5,Fireworks +166,/m/0cmf2,"Fixed-wing aircraft, airplane" +167,/t/dd00118,Fizz +168,/m/07pyf11,Flap +169,/m/0h2mp,"Fly, housefly" +170,/m/04fq5q,Foghorn +171,/m/025rv6n,Fowl +172,/m/09ld4,Frog +173,/m/0dxrf,Frying (food) +174,/m/02z32qm,Fusillade +175,/m/0939n_,Gargling +176,/m/07s0dtb,Gasp +177,/m/0l7xg,Gears +178,/t/dd00099,Generic impact sounds +179,/m/07r660_,Giggle +180,/m/039jq,Glass +181,/m/07q7njn,"Glass chink, clink" +182,/m/07rn7sz,Glass shatter +183,/m/03fwl,Goat +184,/m/07svc2k,Gobble +185,/m/07pt6mm,Grind +186,/m/07s2xch,Groan +187,/m/0ghcn6,Growling +188,/m/07r4k75,Grunt +189,/m/01dwxx,"Gull, seagull" +190,/m/032s66,"Gunshot, gunfire" +191,/m/07swgks,"Gurgling, bubbling" +192,/t/dd00088,Gush +193,/m/03wvsk,Hair dryer +194,/m/03l9g,Hammer +195,/m/0k65p,Hands +196,/m/01jg02,"Heart sounds, heartbeat" +197,/t/dd00067,Heavy engine (low frequency) +198,/m/09ct_,Helicopter +199,/m/02p3nc,Hiccup +200,/m/07rjwbb,Hiss +201,/m/07qwf61,Honk +202,/m/07r_80w,Hoot +203,/m/03k3r,Horse +204,/m/07qf0zm,Howl +205,/m/07q8f3b,Howl (wind) +206,/m/07qfr4h,"Hubbub, speech noise, speech babble" +207,/m/07rcgpl,Hum +208,/t/dd00012,Human group actions +209,/m/0bpl036,Human locomotion +210,/m/0dgw9r,Human sounds +211,/m/09l8g,Human voice +212,/m/02fxyj,Humming +213,/m/03cl9h,"Ice cream truck, ice cream van" +214,/m/07pb8fc,Idling +215,/m/03vt0,Insect +216,/t/dd00126,"Inside, large room or hall" +217,/t/dd00127,"Inside, public space" +218,/t/dd00125,"Inside, small room" +219,/m/03p19w,Jackhammer +220,/m/04229,Jet engine +221,/m/027m70_,Jingle bell +222,/m/07qfgpx,"Jingle, tinkle" +223,/g/11b630rrvh,Kettle whistle +224,/m/056r_1,Keypress tone +225,/m/03v3yw,Keys jangling +226,/m/0d4wf,Kitchen and dining room sounds +227,/m/04ctx,Knife +228,/m/07r4wb8,Knock +229,/m/01j3sz,Laughter +230,/m/01yg9g,Lawn mower +231,/t/dd00065,Light engine (high frequency) +232,/m/04k94,Liquid +233,/m/0ch8v,"Livestock, farm animals, working animals" +234,/m/02ll1_,Lock +235,/m/04zjc,Machine gun +236,/m/06xkwv,Mains hum +237,/t/dd00003,Male singing +238,/m/05zppz,"Male speech, man speaking" +239,/m/01c194,Mantra +240,/m/06cyt0,Mechanical bell +241,/m/02x984l,Mechanical fan +242,/t/dd00077,Mechanisms +243,/t/dd00066,Medium engine (mid frequency) +244,/m/07qrkrw,Meow +245,/m/0hg7b,Microphone +246,/m/0fx9l,Microwave oven +247,/m/07rpkh9,Moo +248,/m/09f96,Mosquito +249,/m/012f08,Motor vehicle (road) +250,/m/02rlv9,"Motorboat, speedboat" +251,/m/04_sv,Motorcycle +252,/m/04rmv,Mouse +253,/m/04rlf,Music +254,/m/02qldy,"Narration, monologue" +255,/m/07q5rw0,"Neigh, whinny" +256,/m/096m7z,Noise +257,/t/dd00061,Non-motorized land vehicle +258,/m/05kq4,Ocean +259,/t/dd00018,Oink +260,/t/dd00122,Other sourceless +261,/t/dd00128,"Outside, urban or manmade" +262,/m/09d5_,Owl +263,/m/05mxj0q,"Packing tape, duct tape" +264,/m/07pyy8b,Pant +265,/t/dd00141,Pant (dog) +266,/m/0641k,Paper rustling +267,/m/07r4gkf,Patter +268,/m/068zj,Pig +269,/m/0h0rv,"Pigeon, dove" +270,/m/07qwdck,Ping +271,/m/07qyrcz,Plop +272,/m/04qvtq,Police car (siren) +273,/m/07prgkl,Pour +274,/m/0md09,"Power saw, circular saw, table saw" +275,/m/0_ksk,Power tool +276,/m/04gxbd,"Power windows, electric windows" +277,/m/01m4t,Printer +278,/m/02l6bg,"Propeller, airscrew" +279,/m/07q34h3,Puff +280,/m/05zc1,Pulleys +281,/m/07pt_g0,Pulse +282,/m/07ptzwd,Pump (liquid) +283,/m/02yds9,Purr +284,/m/07qdb04,Quack +285,/m/0ltv,"Race car, auto racing" +286,/m/06bz3,Radio +287,/m/06d_3,Rail transport +288,/m/01g50p,"Railroad car, train wagon" +289,/m/06mb1,Rain +290,/t/dd00038,Rain on surface +291,/m/07r10fb,Raindrop +292,/m/06bxc,Rapping +293,/m/02bm9n,"Ratchet, pawl" +294,/m/07qn4z3,Rattle +295,/m/040b_t,Refrigerator +296,/m/09hlz4,Respiratory sounds +297,/m/01b9nn,Reverberation +298,/m/02rhddq,Reversing beeps +299,/m/0bcdqg,"Ringing tone, ringback tone" +300,/m/01hnzm,Ringtone +301,/m/04cvmfc,Roar +302,/m/0cdnk,"Roaring cats (lions, tigers)" +303,/m/06hps,"Rodents, rats, mice" +304,/m/07s8j8t,Roll +305,/m/056ks2,"Rowboat, canoe, kayak" +306,/m/07pdjhy,Rub +307,/m/07phhsh,Rumble +308,/m/06h7j,Run +309,/m/07qwyj0,Rustle +310,/m/0hsrw,"Sailboat, sailing ship" +311,/m/023vsd,Sanding +312,/m/01b82r,Sawing +313,/m/01lsmm,Scissors +314,/m/07qv4k0,Scrape +315,/m/07qb_dv,Scratch +316,/m/03qc9zr,Screaming +317,/m/07q8k13,Screech +318,/m/0llzx,Sewing machine +319,/m/07bgp,Sheep +320,/m/06q74,Ship +321,/m/07p6fty,Shout +322,/m/02f9f_,Shower +323,/m/07qv_x_,Shuffle +324,/m/05rj2,Shuffling cards +325,/m/07plz5l,Sigh +326,/m/01v_m0,Sine wave +327,/m/015lz1,Singing +328,/m/07bjf,Single-lens reflex camera +329,/m/0130jx,Sink (filling or washing) +330,/m/03kmc9,Siren +331,/m/07p9k1k,Sizzle +332,/m/06_fw,Skateboard +333,/m/07rjzl8,Slam +334,/m/07ryjzk,"Slap, smack" +335,/m/02y_763,Sliding door +336,/m/07p6mqd,Slosh +337,/m/07pqmly,"Slurp, drinking straw" +338,/m/07pjjrj,"Smash, crash" +339,/m/01y3hg,"Smoke detector, smoke alarm" +340,/m/078jl,Snake +341,/m/07pc8l3,Snap +342,/m/01hsr_,Sneeze +343,/m/07s04w4,Snicker +344,/m/07ppn3j,Sniff +345,/m/01d3sd,Snoring +346,/m/07q0yl5,Snort +347,/t/dd00139,Snort (horse) +348,/m/0790c,Sonar +349,/m/0193bn,Sonic boom +350,/m/0c1dj,Sound effect +351,/t/dd00091,Sound equipment +352,/m/07bm98,Sound reproduction +353,/m/09x0r,Speech +354,/m/0brhx,Speech synthesizer +355,/m/07rrlb6,"Splash, splatter" +356,/m/07pl1bw,Splinter +357,/m/07qlf79,Spray +358,/m/07sx8x_,Squawk +359,/m/07q6cd_,Squeak +360,/m/07s02z0,Squeal +361,/m/07qlwh6,Squish +362,/m/01lynh,Stairs +363,/m/07rgkc5,Static +364,/m/06wzb,Steam +365,/m/06hck5,Steam whistle +366,/m/07ptfmf,Stir +367,/m/01g90h,Stomach rumble +368,/m/07sk0jz,"Stomp, stamp" +369,/m/0j6m2,"Stream, river" +370,/m/0195fx,"Subway, metro, underground" +371,/t/dd00109,Surface contact +372,/m/07s13rg,Sweeping +373,/t/dd00006,Synthetic singing +374,/m/07qcpgn,Tap +375,/m/0fw86,Tap dance +376,/m/09d1b1,Tape hiss +377,/m/07qcx4z,Tearing +378,/m/07cx4,Telephone +379,/m/07pp8cl,Telephone bell ringing +380,/m/02c8p,"Telephone dialing, DTMF" +381,/m/07c52,Television +382,/m/0dl9sf8,Throat clearing +383,/m/07qnq_y,"Thump, thud" +384,/m/0ngt1,Thunder +385,/m/0jb2l,Thunderstorm +386,/m/07rrh0c,Thunk +387,/m/07qjznt,Tick +388,/m/07qjznl,Tick-tock +389,/m/0h9mv,"Tire squeal, skidding" +390,/m/01jt3m,Toilet flush +391,/m/07k1x,Tools +392,/m/012xff,Toothbrush +393,/m/0btp2,"Traffic noise, roadway noise" +394,/m/07jdr,Train +395,/m/0284vy3,Train horn +396,/t/dd00048,Train wheels squealing +397,/m/04zmvq,Train whistle +398,/m/07pqc89,"Trickle, dribble" +399,/m/07r04,Truck +400,/m/07n_g,Tuning fork +401,/m/01rd7k,Turkey +402,/m/0c2wf,Typewriter +403,/m/0316dw,Typing +404,/t/dd00143,Unknown sound +405,/m/0d31p,Vacuum cleaner +406,/m/07yv9,Vehicle +407,/m/0912c9,"Vehicle horn, car horn, honking, toot" +408,/m/0zmy2j9,"Velcro, hook and loop fastener" +409,/m/0bzvm2,Video game sound +410,/m/07qw_06,"Wail, moan" +411,/m/07pbtc8,"Walk, footsteps" +412,/m/0174k2,Washing machine +413,/m/0838f,Water +414,/m/02jz0l,"Water tap, faucet" +415,/m/0j2kx,Waterfall +416,/m/034srq,"Waves, surf" +417,/m/07rdhzs,"Whack, thwack" +418,/m/032n05,Whale vocalization +419,/m/07mzm6,Wheeze +420,/m/07qz6j3,Whimper +421,/t/dd00136,Whimper (dog) +422,/m/07rbp7_,Whip +423,/m/07s34ls,Whir +424,/m/02rtxlg,Whispering +425,/m/0l156k,Whistle +426,/m/01w250,Whistling +427,/m/0chx_,"White noise, pink noise" +428,/m/07rwj3x,Whoop +429,/m/07rqsjt,"Whoosh, swoosh, swish" +430,/m/01280g,Wild animals +431,/m/0fjy1,Wildfire +432,/m/03m9d0z,Wind +433,/m/026fgl,Wind chime +434,/t/dd00092,Wind noise (microphone) +435,/m/02417f,"Windscreen wiper, windshield wiper" +436,/m/07pk7mg,Wobble +437,/m/079vc8,Wolf-whistling +438,/m/083vt,Wood +439,/m/081rb,Writing +440,/m/01hhp3,Yak +441,/m/01j423,Yawn +442,/m/07sr1lc,Yell +443,/m/07r_k2n,Yip +444,/m/01swy6,Yodeling +445,/m/07p78v5,Zing +446,/m/01s0vc,Zipper (clothing) diff --git a/framesed/framesed.c b/framesed/framesed.c new file mode 100644 index 0000000..36eb87f --- /dev/null +++ b/framesed/framesed.c @@ -0,0 +1,1102 @@ +#include "framesed.h" + +#include <math.h> +#include <pthread.h> +#include <stdatomic.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#if defined(FRAMESED_ACCELERATE) +# include <Accelerate/Accelerate.h> +#elif defined(FRAMESED_OPENBLAS) +# include <cblas.h> +#endif + +#include "model_blob.h" + +#define MEL_BINS 128 +#define FFT_SIZE 512 +#define FFT_BINS 257 +#define FFT_BITS 9 +#define WINDOW_SIZE 400 +#define WINDOW_OFFSET 56 +#define HOP_SIZE 160 +#define MODEL_SECONDS 10u +#define MODEL_SAMPLES (MODEL_SECONDS * FRAMESED_SAMPLE_RATE) +#define MODEL_FRAMES 250u +#define FINAL_CHANNELS 960 +#define FRAMESED_FRAME_SAMPLES 640u +#define INFERENCE_HOP_SECONDS 5u +#define INFERENCE_HOP_SAMPLES (INFERENCE_HOP_SECONDS * FRAMESED_SAMPLE_RATE) +#define INFERENCE_HOP_FRAMES 125u +#define INTERNAL_KEEP_START 63u +#define INTERNAL_KEEP_END (INTERNAL_KEEP_START + INFERENCE_HOP_FRAMES) + +typedef enum { + ACT_NONE, + ACT_RELU, + ACT_HSWISH, +} activation; + +typedef struct { + const float *weight; + const float *bias; + int input_channels; + int output_channels; + int kernel; + int stride_y; + int stride_x; +} conv; + +typedef struct { + int input_channels; + int expanded_channels; + int output_channels; + int kernel; + int stride_y; + int stride_x; + int squeeze_channels; + activation act; + int residual; + conv expand; + conv depthwise; + conv project; + const float *se1_weight; + const float *se1_bias; + const float *se2_weight; + const float *se2_bias; +} block; + +typedef struct { + float *data; + int channels; + int height; + int width; +} tensor; + +typedef struct { + int start; + int count; + const float *weights; +} mel_band; + +struct framesed { + framesed_model_blob blob; + void *mapping; + size_t mapping_size; + const float *window; + const float *mel; + mel_band bands[MEL_BINS]; + conv initial; + block blocks[15]; + conv final; + conv output; + float twiddle_real[FFT_SIZE / 2]; + float twiddle_imag[FFT_SIZE / 2]; + uint16_t reverse[FFT_SIZE]; + char backend[24]; +}; + +typedef struct { + int input; + int expanded; + int output; + int kernel; + int stride_y; + int stride_x; + int squeeze; + activation act; +} block_config; + +typedef struct { + const char *label; + float threshold; + float release; + float minimum; + float gap; + uint16_t class_offset; + uint16_t class_count; + int16_t parent; + uint64_t suppresses; +} framesed_group_definition; + +#include "framesed_policy.inc" + +_Static_assert(FRAMESED_GROUP_COUNT <= 64, "FRAMESED suppression mask is too small"); +_Static_assert(FRAMESED_GROUP_COUNT <= FRAMESED_CLASS_COUNT, "FRAMESED track presence array is too small"); + +static const struct { + const char *mid; + const char *label; +} framesed_classes[] = { +#include "framesed_classes.inc" +}; + +_Static_assert(sizeof(framesed_classes) / sizeof(framesed_classes[0]) == FRAMESED_CLASS_COUNT, + "FRAMESED labels must match the model output count"); + +static size_t detection_group_count(int all_classes) { + return all_classes ? FRAMESED_CLASS_COUNT : FRAMESED_GROUP_COUNT; +} + +static framesed_group_definition detection_group(size_t group, int all_classes) { + if (!all_classes) + return framesed_groups[group]; + return (framesed_group_definition){ + .label = framesed_classes[group].label, + .threshold = 0.50f, + .release = 0.35f, + .minimum = 0.24f, + .gap = 0.32f, + .class_offset = (uint16_t)group, + .class_count = 1, + .parent = -1, + }; +} + +static const block_config block_configs[15] = { + {16, 16, 16, 3, 1, 1, 0, ACT_RELU}, {16, 64, 24, 3, 2, 2, 0, ACT_RELU}, + {24, 72, 24, 3, 2, 1, 0, ACT_RELU}, {24, 72, 40, 5, 1, 1, 24, ACT_RELU}, + {40, 120, 40, 5, 1, 1, 32, ACT_RELU}, {40, 120, 40, 5, 2, 1, 32, ACT_RELU}, + {40, 240, 80, 3, 1, 1, 0, ACT_HSWISH}, {80, 200, 80, 3, 2, 1, 0, ACT_HSWISH}, + {80, 184, 80, 3, 1, 1, 0, ACT_HSWISH}, {80, 184, 80, 3, 1, 1, 0, ACT_HSWISH}, + {80, 480, 112, 3, 2, 1, 120, ACT_HSWISH}, {112, 672, 112, 3, 1, 1, 168, ACT_HSWISH}, + {112, 672, 160, 5, 1, 1, 168, ACT_HSWISH}, {160, 960, 160, 5, 1, 1, 240, ACT_HSWISH}, + {160, 960, 160, 5, 2, 1, 240, ACT_HSWISH}, +}; + +static int checked_product(size_t a, size_t b, size_t *result) { + if (a && b > SIZE_MAX / a) + return -1; + *result = a * b; + return 0; +} + +static int tensor_allocate(tensor *value, int channels, int height, int width) { + if (channels <= 0 || height <= 0 || width <= 0) + return -1; + size_t count; + if (checked_product((size_t)channels, (size_t)height, &count) != 0 || + checked_product(count, (size_t)width, &count) != 0 || count > SIZE_MAX / sizeof(float)) + return -1; + void *data = NULL; + if (posix_memalign(&data, 64, count * sizeof(float)) != 0) + return -1; + value->data = data; + value->channels = channels; + value->height = height; + value->width = width; + return 0; +} + +static void tensor_release(tensor *value) { + free(value->data); + memset(value, 0, sizeof(*value)); +} + +static float activate(float value, activation act) { + if (act == ACT_RELU) + return value > 0.0f ? value : 0.0f; + if (act == ACT_HSWISH) { + float gate = value + 3.0f; + if (gate < 0.0f) + gate = 0.0f; + if (gate > 6.0f) + gate = 6.0f; + return value * gate * (1.0f / 6.0f); + } + return value; +} + +static void matrix_multiply( + const float *weight, const float *input, float *output, int rows, int columns, int inner) { +#if defined(FRAMESED_ACCELERATE) || defined(FRAMESED_OPENBLAS) + cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, columns, inner, 1.0f, weight, inner, + input, columns, 0.0f, output, columns); +#else + for (int row = 0; row < rows; ++row) { + const float *w = weight + (size_t)row * inner; + float *out = output + (size_t)row * columns; + memset(out, 0, (size_t)columns * sizeof(float)); + for (int k = 0; k < inner; ++k) { + const float scale = w[k]; + const float *in = input + (size_t)k * columns; + for (int column = 0; column < columns; ++column) + out[column] += scale * in[column]; + } + } +#endif +} + +static int pointwise(const conv *layer, const tensor *input, activation act, tensor *output) { + if (layer->kernel != 1 || layer->stride_y != 1 || layer->stride_x != 1 || + layer->input_channels != input->channels || + tensor_allocate(output, layer->output_channels, input->height, input->width) != 0) + return -1; + const int spatial = input->height * input->width; + matrix_multiply(layer->weight, input->data, output->data, layer->output_channels, spatial, + layer->input_channels); + for (int channel = 0; channel < output->channels; ++channel) { + float *row = output->data + (size_t)channel * spatial; + const float bias = layer->bias[channel]; + for (int index = 0; index < spatial; ++index) + row[index] = activate(row[index] + bias, act); + } + return 0; +} + +static int +spatial_conv(const conv *layer, const tensor *input, activation act, int depthwise, tensor *output) { + const int padding = layer->kernel / 2; + const int out_height = (input->height + 2 * padding - layer->kernel) / layer->stride_y + 1; + const int out_width = (input->width + 2 * padding - layer->kernel) / layer->stride_x + 1; + if (layer->input_channels != input->channels || + (depthwise && layer->output_channels != input->channels) || + tensor_allocate(output, layer->output_channels, out_height, out_width) != 0) + return -1; + + for (int output_channel = 0; output_channel < output->channels; ++output_channel) { + const int first_input = depthwise ? output_channel : 0; + const int last_input = depthwise ? output_channel + 1 : input->channels; + const float *kernel = + layer->weight + + (size_t)output_channel * (depthwise ? layer->kernel * layer->kernel + : input->channels * layer->kernel * layer->kernel); + float *destination = output->data + (size_t)output_channel * out_height * out_width; + for (int y = 0; y < out_height; ++y) { + for (int x = 0; x < out_width; ++x) { + float sum = layer->bias[output_channel]; + for (int input_channel = first_input; input_channel < last_input; ++input_channel) { + const float *source = + input->data + (size_t)input_channel * input->height * input->width; + const int kernel_channel = + depthwise ? 0 : input_channel * layer->kernel * layer->kernel; + for (int ky = 0; ky < layer->kernel; ++ky) { + const int sy = y * layer->stride_y + ky - padding; + if (sy < 0 || sy >= input->height) + continue; + for (int kx = 0; kx < layer->kernel; ++kx) { + const int sx = x * layer->stride_x + kx - padding; + if (sx < 0 || sx >= input->width) + continue; + sum += source[(size_t)sy * input->width + sx] * + kernel[kernel_channel + ky * layer->kernel + kx]; + } + } + } + destination[(size_t)y * out_width + x] = activate(sum, act); + } + } + } + return 0; +} + +static int squeeze_excitation(const block *layer, tensor *input) { + if (!layer->squeeze_channels) + return 0; + const int spatial = input->height * input->width; + float *mean = malloc((size_t)input->channels * sizeof(float)); + float *hidden = malloc((size_t)layer->squeeze_channels * sizeof(float)); + float *scale = malloc((size_t)input->channels * sizeof(float)); + if (!mean || !hidden || !scale) { + free(mean); + free(hidden); + free(scale); + return -1; + } + for (int channel = 0; channel < input->channels; ++channel) { + const float *row = input->data + (size_t)channel * spatial; + float sum = 0.0f; + for (int index = 0; index < spatial; ++index) + sum += row[index]; + mean[channel] = sum / spatial; + } + for (int row = 0; row < layer->squeeze_channels; ++row) { + const float *weight = layer->se1_weight + (size_t)row * input->channels; + float sum = layer->se1_bias[row]; + for (int column = 0; column < input->channels; ++column) + sum += weight[column] * mean[column]; + hidden[row] = sum > 0.0f ? sum : 0.0f; + } + for (int row = 0; row < input->channels; ++row) { + const float *weight = layer->se2_weight + (size_t)row * layer->squeeze_channels; + float sum = layer->se2_bias[row]; + for (int column = 0; column < layer->squeeze_channels; ++column) + sum += weight[column] * hidden[column]; + scale[row] = 1.0f / (1.0f + expf(-sum)); + } + for (int channel = 0; channel < input->channels; ++channel) { + float *row = input->data + (size_t)channel * spatial; + for (int index = 0; index < spatial; ++index) + row[index] *= scale[channel]; + } + free(mean); + free(hidden); + free(scale); + return 0; +} + +static int run_block(const block *layer, tensor *input, tensor *output) { + tensor expanded = {0}; + tensor depth = {0}; + tensor projected = {0}; + const tensor *depth_input = input; + if (layer->expanded_channels != layer->input_channels) { + if (pointwise(&layer->expand, input, layer->act, &expanded) != 0) + goto failed; + depth_input = &expanded; + } + if (spatial_conv(&layer->depthwise, depth_input, layer->act, 1, &depth) != 0 || + squeeze_excitation(layer, &depth) != 0 || + pointwise(&layer->project, &depth, ACT_NONE, &projected) != 0) + goto failed; + + if (layer->residual) { + const size_t count = (size_t)input->channels * input->height * input->width; + for (size_t index = 0; index < count; ++index) + projected.data[index] += input->data[index]; + } + tensor_release(&expanded); + tensor_release(&depth); + *output = projected; + return 0; + +failed: + tensor_release(&expanded); + tensor_release(&depth); + tensor_release(&projected); + return -1; +} + +static void fft(framesed_t *engine, float *real, float *imag) { + for (int length = 2; length <= FFT_SIZE; length <<= 1) { + const int half = length / 2; + const int stride = FFT_SIZE / length; + for (int base = 0; base < FFT_SIZE; base += length) { + for (int j = 0; j < half; ++j) { + const float wr = engine->twiddle_real[j * stride]; + const float wi = engine->twiddle_imag[j * stride]; + const int a = base + j; + const int b = a + half; + const float tr = real[b] * wr - imag[b] * wi; + const float ti = real[b] * wi + imag[b] * wr; + real[b] = real[a] - tr; + imag[b] = imag[a] - ti; + real[a] += tr; + imag[a] += ti; + } + } + } +} + +static size_t reflect_index(int64_t index, size_t length) { + if (length <= 1) + return 0; + const int64_t period = (int64_t)(2 * length - 2); + index %= period; + if (index < 0) + index += period; + if ((size_t)index >= length) + index = period - index; + return (size_t)index; +} + +static float preemphasized(const int16_t *samples, size_t index) { + return ((float)samples[index + 1] - 0.97f * (float)samples[index]) * (1.0f / 32768.0f); +} + +static int frontend(framesed_t *engine, const int16_t *samples, size_t num_samples, tensor *features) { + if (num_samples < 2) + return -1; + const size_t emphasized = num_samples - 1; + const size_t frames_size = 1 + emphasized / HOP_SIZE; + if (frames_size > INT32_MAX || tensor_allocate(features, 1, MEL_BINS, (int)frames_size) != 0) + return -1; + const int frames = (int)frames_size; + + float real[FFT_SIZE]; + float imag[FFT_SIZE]; + float power[FFT_BINS]; + for (int frame = 0; frame < frames; ++frame) { + memset(real, 0, sizeof(real)); + memset(imag, 0, sizeof(imag)); + const int64_t center = (int64_t)frame * HOP_SIZE; + for (int index = 0; index < WINDOW_SIZE; ++index) { + const size_t source = reflect_index(center - WINDOW_SIZE / 2 + index, emphasized); + const int fft_index = WINDOW_OFFSET + index; + real[engine->reverse[fft_index]] = preemphasized(samples, source) * engine->window[index]; + } + fft(engine, real, imag); + for (int bin = 0; bin < FFT_BINS; ++bin) + power[bin] = real[bin] * real[bin] + imag[bin] * imag[bin]; + + for (int mel = 0; mel < MEL_BINS; ++mel) { + const mel_band *band = &engine->bands[mel]; + float sum = 0.0f; + for (int bin = 0; bin < band->count; ++bin) + sum += power[band->start + bin] * band->weights[bin]; + if (sum < 1e-7f) + sum = 1e-7f; + features->data[(size_t)mel * frames + frame] = (logf(sum) + 4.5f) * 0.2f; + } + } + return 0; +} + +static int load_conv(framesed_model_blob *blob, + conv *layer, + const char *name, + int input, + int output, + int kernel, + int stride_y, + int stride_x) { + char weight_name[24]; + char bias_name[24]; + if (snprintf(weight_name, sizeof(weight_name), "%s_w", name) >= (int)sizeof(weight_name) || + snprintf(bias_name, sizeof(bias_name), "%s_b", name) >= (int)sizeof(bias_name)) + return -1; + layer->weight = framesed_model_blob_take(blob, weight_name, (size_t)output * input * kernel * kernel); + layer->bias = framesed_model_blob_take(blob, bias_name, (size_t)output); + layer->input_channels = input; + layer->output_channels = output; + layer->kernel = kernel; + layer->stride_y = stride_y; + layer->stride_x = stride_x; + return layer->weight && layer->bias ? 0 : -1; +} + +static int load_block(framesed_model_blob *blob, block *layer, int index, const block_config *config) { + char name[24]; + memset(layer, 0, sizeof(*layer)); + layer->input_channels = config->input; + layer->expanded_channels = config->expanded; + layer->output_channels = config->output; + layer->kernel = config->kernel; + layer->stride_y = config->stride_y; + layer->stride_x = config->stride_x; + layer->squeeze_channels = config->squeeze; + layer->act = config->act; + layer->residual = + config->stride_y == 1 && config->stride_x == 1 && config->input == config->output; + + if (config->expanded != config->input) { + snprintf(name, sizeof(name), "b%de", index); + if (load_conv(blob, &layer->expand, name, config->input, config->expanded, 1, 1, 1) != 0) + return -1; + } + snprintf(name, sizeof(name), "b%dd", index); + if (load_conv(blob, &layer->depthwise, name, 1, config->expanded, config->kernel, + config->stride_y, config->stride_x) != 0) + return -1; + layer->depthwise.input_channels = config->expanded; + + if (config->squeeze) { + snprintf(name, sizeof(name), "b%ds1w", index); + layer->se1_weight = + framesed_model_blob_take(blob, name, (size_t)config->squeeze * config->expanded); + snprintf(name, sizeof(name), "b%ds1b", index); + layer->se1_bias = framesed_model_blob_take(blob, name, config->squeeze); + snprintf(name, sizeof(name), "b%ds2w", index); + layer->se2_weight = + framesed_model_blob_take(blob, name, (size_t)config->expanded * config->squeeze); + snprintf(name, sizeof(name), "b%ds2b", index); + layer->se2_bias = framesed_model_blob_take(blob, name, config->expanded); + if (!layer->se1_weight || !layer->se1_bias || !layer->se2_weight || !layer->se2_bias) + return -1; + } + snprintf(name, sizeof(name), "b%dp", index); + return load_conv(blob, &layer->project, name, config->expanded, config->output, 1, 1, 1); +} + +framesed_t *framesed_create(const char *weights_path) { + if (!weights_path) + return NULL; + framesed_t *engine = calloc(1, sizeof(*engine)); + if (!engine) + return NULL; + if (framesed_model_blob_map(weights_path, 1, &engine->blob, &engine->mapping, &engine->mapping_size) != + 0) + goto failed; + + engine->window = framesed_model_blob_take(&engine->blob, "window", WINDOW_SIZE); + engine->mel = framesed_model_blob_take(&engine->blob, "mel", MEL_BINS * FFT_BINS); + if (!engine->window || !engine->mel || + load_conv(&engine->blob, &engine->initial, "c0", 1, 16, 3, 2, 2) != 0) + goto failed; + for (int index = 0; index < 15; ++index) + if (load_block(&engine->blob, &engine->blocks[index], index + 1, &block_configs[index]) != 0) + goto failed; + if (load_conv(&engine->blob, &engine->final, "cf", 160, FINAL_CHANNELS, 1, 1, 1) != 0 || + load_conv(&engine->blob, &engine->output, "out", FINAL_CHANNELS, FRAMESED_CLASS_COUNT, 1, 1, 1) != + 0 || + engine->blob.failed) + goto failed; + + for (int mel = 0; mel < MEL_BINS; ++mel) { + const float *row = engine->mel + (size_t)mel * FFT_BINS; + int start = 0; + while (start < FFT_BINS && row[start] == 0.0f) + ++start; + int end = FFT_BINS; + while (end > start && row[end - 1] == 0.0f) + --end; + engine->bands[mel].start = start; + engine->bands[mel].count = end - start; + engine->bands[mel].weights = row + start; + } + for (int index = 0; index < FFT_SIZE / 2; ++index) { + const double angle = -2.0 * M_PI * index / FFT_SIZE; + engine->twiddle_real[index] = (float)cos(angle); + engine->twiddle_imag[index] = (float)sin(angle); + } + for (int index = 0; index < FFT_SIZE; ++index) { + unsigned reversed = 0; + for (int bit = 0; bit < FFT_BITS; ++bit) + reversed |= ((unsigned)(index >> bit) & 1u) << (FFT_BITS - 1 - bit); + engine->reverse[index] = (uint16_t)reversed; + } +#if defined(FRAMESED_ACCELERATE) + strcpy(engine->backend, "accelerate"); +#elif defined(FRAMESED_OPENBLAS) + strcpy(engine->backend, "openblas"); +#else + strcpy(engine->backend, "scalar"); +#endif + return engine; + +failed: + framesed_destroy(engine); + return NULL; +} + +void framesed_destroy(framesed_t *engine) { + if (!engine) + return; + framesed_model_blob_unmap(engine->mapping, engine->mapping_size); + free(engine); +} + +int framesed_infer_s16(framesed_t *engine, const int16_t *samples, size_t num_samples, framesed_logits *output) { + if (!engine || !samples || !output || num_samples != MODEL_SAMPLES) + return -1; + memset(output, 0, sizeof(*output)); + tensor current = {0}; + tensor next = {0}; + if (frontend(engine, samples, num_samples, ¤t) != 0 || + spatial_conv(&engine->initial, ¤t, ACT_HSWISH, 0, &next) != 0) + goto failed; + tensor_release(¤t); + current = next; + memset(&next, 0, sizeof(next)); + + for (int index = 0; index < 15; ++index) { + if (run_block(&engine->blocks[index], ¤t, &next) != 0) + goto failed; + tensor_release(¤t); + current = next; + memset(&next, 0, sizeof(next)); + } + if (pointwise(&engine->final, ¤t, ACT_HSWISH, &next) != 0) + goto failed; + tensor_release(¤t); + current = next; + memset(&next, 0, sizeof(next)); + + if (current.height != 1) + goto failed; + const int frames = current.width; + size_t logit_count; + if (checked_product(FRAMESED_CLASS_COUNT, (size_t)frames, &logit_count) != 0 || + logit_count > SIZE_MAX / sizeof(float)) + goto failed; + float *logits = malloc(logit_count * sizeof(float)); + if (!logits) + goto failed; + matrix_multiply(engine->output.weight, current.data, logits, FRAMESED_CLASS_COUNT, frames, + FINAL_CHANNELS); + for (int class_index = 0; class_index < FRAMESED_CLASS_COUNT; ++class_index) { + float *row = logits + (size_t)class_index * frames; + const float bias = engine->output.bias[class_index]; + for (int frame = 0; frame < frames; ++frame) + row[frame] += bias; + } + tensor_release(¤t); + output->logits = logits; + output->frames = (size_t)frames; + return 0; + +failed: + tensor_release(¤t); + tensor_release(&next); + return -1; +} + +void framesed_logits_free(framesed_logits *output) { + if (!output) + return; + free(output->logits); + memset(output, 0, sizeof(*output)); +} + +static size_t output_frames(size_t samples) { + return samples ? (samples + FRAMESED_FRAME_SAMPLES - 1) / FRAMESED_FRAME_SAMPLES : 0; +} + +static float sigmoid(float value) { + if (value >= 0.0f) + return 1.0f / (1.0f + expf(-value)); + const float exponential = expf(value); + return exponential / (1.0f + exponential); +} + +static int group_probabilities(const framesed_logits *logits, float *scores, int all_classes) { + if (!logits || !logits->logits || !scores || !logits->frames) + return -1; + const size_t frames = logits->frames; + for (size_t group = 0; group < detection_group_count(all_classes); ++group) { + float *destination = scores + group * frames; + for (size_t frame = 0; frame < frames; ++frame) + destination[frame] = 0.0f; + const framesed_group_definition definition = detection_group(group, all_classes); + for (uint16_t member = 0; member < definition.class_count; ++member) { + const size_t class_index = + all_classes ? group : framesed_group_classes[definition.class_offset + member]; + const float *source = logits->logits + (size_t)class_index * frames; + for (size_t frame = 0; frame < frames; ++frame) { + const float probability = sigmoid(source[frame]); + if (probability > destination[frame]) + destination[frame] = probability; + } + } + } + return 0; +} + +static int append_event(framesed_result *result, + size_t *capacity, + size_t group, + const char *label, + size_t start_frame, + size_t end_frame, + float score, + double duration) { + if (result->num_events == *capacity) { + size_t grown = *capacity ? *capacity * 2 : 32; + if (grown < *capacity || grown > SIZE_MAX / sizeof(*result->events)) + return -1; + framesed_event *events = realloc(result->events, grown * sizeof(*result->events)); + if (!events) + return -1; + result->events = events; + *capacity = grown; + } + double start = start_frame * FRAMESED_FRAME_STEP; + double end = end_frame * FRAMESED_FRAME_STEP; + if (end > duration) + end = duration; + result->events[result->num_events++] = (framesed_event){ + .start = start, + .end = end, + .score = score, + .label = label, + .class_group = (int)group, + }; + return 0; +} + +static int compare_event(const void *left, const void *right) { + const framesed_event *a = left; + const framesed_event *b = right; + if (a->start != b->start) + return (a->start > b->start) - (a->start < b->start); + if (a->end != b->end) + return (a->end > b->end) - (a->end < b->end); + if (a->score != b->score) + return (a->score < b->score) - (a->score > b->score); + return a->class_group - b->class_group; +} + +/* Caption suppression uses a 64-bit group mask; model classes never enter + * this path because their outputs are independent, including parent classes. */ +static void suppress_groups(float *scores, size_t frames) { + for (size_t child = 0; child < FRAMESED_GROUP_COUNT; ++child) { + const float *child_scores = scores + child * frames; + uint64_t suppressed = framesed_groups[child].suppresses; + int parent = framesed_groups[child].parent; + while (parent >= 0) { + suppressed |= UINT64_C(1) << parent; + float *parent_scores = scores + (size_t)parent * frames; + for (size_t frame = 0; frame < frames; ++frame) + if (child_scores[frame] >= framesed_groups[child].release) + parent_scores[frame] = 0.0f; + parent = framesed_groups[parent].parent; + } + for (size_t group = 0; group < FRAMESED_GROUP_COUNT; ++group) { + if (!(suppressed & (UINT64_C(1) << group))) + continue; + float *row = scores + group * frames; + for (size_t frame = 0; frame < frames; ++frame) + if (child_scores[frame] >= framesed_groups[child].release) + row[frame] = 0.0f; + } + } +} + +static int extract_events(float *scores, size_t frames, double duration, framesed_result *result, + int all_classes) { + if (!all_classes) + suppress_groups(scores, frames); + size_t capacity = 0; + for (size_t group = 0; group < detection_group_count(all_classes); ++group) { + const framesed_group_definition definition = detection_group(group, all_classes); + const float *row = scores + group * frames; + size_t frame = 0; + while (frame < frames) { + while (frame < frames && row[frame] < definition.threshold) + ++frame; + if (frame == frames) + break; + const size_t start = frame; + size_t last_active = frame; + float maximum = row[frame]; + ++frame; + while (frame < frames) { + if (row[frame] >= definition.release) { + last_active = frame; + if (row[frame] > maximum) + maximum = row[frame]; + } else if ((frame - last_active) * FRAMESED_FRAME_STEP > definition.gap) { + break; + } + ++frame; + } + const size_t end = last_active + 1; + if ((end - start) * FRAMESED_FRAME_STEP >= definition.minimum && + append_event(result, &capacity, group, definition.label, start, end, + maximum, duration) != 0) + return -1; + } + } + if (result->num_events > 1) + qsort(result->events, result->num_events, sizeof(*result->events), compare_event); + return 0; +} + +/* Number of model windows the recording is covered by. Windows hop by + * INFERENCE_HOP_SAMPLES, and the last one is the first whose remaining + * samples fit inside a single window. */ +static size_t chunk_count_for(size_t num_samples) { + if (num_samples <= MODEL_SAMPLES) + return 1; + const size_t beyond = num_samples - MODEL_SAMPLES; + return 1 + (beyond + INFERENCE_HOP_SAMPLES - 1) / INFERENCE_HOP_SAMPLES; +} + +/* Runs one model window and writes its kept frames into the recording-wide + * score plane. The destination range follows from the window index alone, so + * windows never write the same frame and the result does not depend on the + * order in which they run. */ +static int run_chunk(framesed_t *engine, + const int16_t *samples, + size_t num_samples, + size_t chunk_index, + size_t frames, + float *scores, + int16_t *padded, + int all_classes) { + const size_t chunk_start = chunk_index * INFERENCE_HOP_SAMPLES; + if (chunk_start >= num_samples) + return -1; + size_t chunk_count = num_samples - chunk_start; + if (chunk_count > MODEL_SAMPLES) + chunk_count = MODEL_SAMPLES; + const int first = chunk_index == 0; + const int last = chunk_start + chunk_count == num_samples; + const int16_t *chunk = samples + chunk_start; + if (chunk_count < MODEL_SAMPLES) { + memcpy(padded, chunk, chunk_count * sizeof(*padded)); + memset(padded + chunk_count, 0, (MODEL_SAMPLES - chunk_count) * sizeof(*padded)); + chunk = padded; + } + + framesed_logits logits = {0}; + if (framesed_infer_s16(engine, chunk, MODEL_SAMPLES, &logits) != 0 || logits.frames != MODEL_FRAMES) { + framesed_logits_free(&logits); + return -1; + } + size_t local_count; + if (checked_product(detection_group_count(all_classes), logits.frames, &local_count) != 0 || + local_count > SIZE_MAX / sizeof(float)) { + framesed_logits_free(&logits); + return -1; + } + float *local_scores = malloc(local_count * sizeof(float)); + if (!local_scores || group_probabilities(&logits, local_scores, all_classes) != 0) { + free(local_scores); + framesed_logits_free(&logits); + return -1; + } + const size_t actual_frames = output_frames(chunk_count); + const size_t keep_start = first ? 0 : INTERNAL_KEEP_START; + const size_t keep_end = last ? actual_frames : INTERNAL_KEEP_END; + if (keep_start > keep_end || keep_end > logits.frames) { + free(local_scores); + framesed_logits_free(&logits); + return -1; + } + const size_t global_offset = chunk_start / FRAMESED_FRAME_SAMPLES; + for (size_t group = 0; group < detection_group_count(all_classes); ++group) { + const float *source = local_scores + group * logits.frames; + float *destination = scores + group * frames; + for (size_t frame = keep_start; frame < keep_end; ++frame) { + const size_t global_frame = global_offset + frame; + if (global_frame < frames) + destination[global_frame] = source[frame]; + } + } + free(local_scores); + framesed_logits_free(&logits); + return 0; +} + +typedef struct { + framesed_t *engine; + const int16_t *samples; + size_t num_samples; + size_t num_chunks; + size_t frames; + float *scores; + int all_classes; + atomic_size_t next_chunk; + atomic_int failed; + void (*progress)(void *context, size_t done, size_t total); + void *progress_context; + pthread_mutex_t progress_lock; + size_t finished; /* guarded by progress_lock */ +} detect_pool; + +/* Claims windows until the recording is covered. The engine holds only + * read-only weights and every window allocates its own working tensors, so + * one engine serves every worker. */ +static void *detect_worker(void *opaque) { + detect_pool *pool = opaque; + int16_t *padded = calloc(MODEL_SAMPLES, sizeof(*padded)); + if (!padded) { + atomic_store(&pool->failed, 1); + return NULL; + } + for (;;) { + const size_t index = atomic_fetch_add(&pool->next_chunk, 1); + if (index >= pool->num_chunks || atomic_load(&pool->failed)) + break; + if (run_chunk(pool->engine, pool->samples, pool->num_samples, index, pool->frames, + pool->scores, padded, pool->all_classes) != 0) { + atomic_store(&pool->failed, 1); + break; + } + if (pool->progress) { + pthread_mutex_lock(&pool->progress_lock); + pool->progress(pool->progress_context, ++pool->finished, pool->num_chunks); + pthread_mutex_unlock(&pool->progress_lock); + } + } + free(padded); + return NULL; +} + +/* Reduces a group's frame scores to one value per output step by taking the + * maximum, which is how an event's reported score is defined and keeps a + * short peak from disappearing into a longer step. */ +static int build_tracks(const float *scores, size_t frames, double track_step, framesed_result *result, + int all_classes) { + /* The step snaps down to whole model frames, so the returned resolution is + * never coarser than the caller asked for. */ + const size_t per_step = (size_t)(track_step / FRAMESED_FRAME_STEP); + const size_t stride = per_step ? per_step : 1; + const size_t count = (frames + stride - 1) / stride; + if (!count) + return 0; + + /* A group earns a track when its score reaches the release threshold + * anywhere in the recording. That covers every group that produced an + * event and also the sustained near misses, which are the cases where an + * event boundary alone does not say how present a sound is. */ + unsigned char present[FRAMESED_CLASS_COUNT] = {0}; + const size_t num_groups = detection_group_count(all_classes); + size_t num_tracks = 0; + for (size_t group = 0; group < num_groups; ++group) { + const framesed_group_definition definition = detection_group(group, all_classes); + const float *row = scores + group * frames; + for (size_t frame = 0; frame < frames; ++frame) + if (row[frame] >= definition.release) { + present[group] = 1; + ++num_tracks; + break; + } + } + if (!num_tracks) + return 0; + + framesed_track *tracks = calloc(num_tracks, sizeof(*tracks)); + if (!tracks) + return -1; + size_t index = 0; + for (size_t group = 0; group < num_groups; ++group) { + if (!present[group]) + continue; + float *values = malloc(count * sizeof(*values)); + if (!values) { + for (size_t i = 0; i < index; ++i) + free(tracks[i].values); + free(tracks); + return -1; + } + const float *row = scores + group * frames; + for (size_t step = 0; step < count; ++step) { + const size_t start = step * stride; + size_t end = start + stride; + if (end > frames) + end = frames; + float maximum = row[start]; + for (size_t frame = start + 1; frame < end; ++frame) + if (row[frame] > maximum) + maximum = row[frame]; + values[step] = maximum; + } + tracks[index++] = (framesed_track){ + .label = detection_group(group, all_classes).label, + .class_group = (int)group, + .values = values, + .count = count, + }; + } + result->tracks = tracks; + result->num_tracks = num_tracks; + result->track_step = (double)stride * FRAMESED_FRAME_STEP; + return 0; +} + +const char *framesed_group_label(size_t group) { + return group < FRAMESED_GROUP_COUNT ? framesed_groups[group].label : NULL; +} + +size_t framesed_group_count(void) { return FRAMESED_GROUP_COUNT; } + +const char *framesed_class_label(size_t class_index) { + return class_index < FRAMESED_CLASS_COUNT ? framesed_classes[class_index].label : NULL; +} + +const char *framesed_class_mid(size_t class_index) { + return class_index < FRAMESED_CLASS_COUNT ? framesed_classes[class_index].mid : NULL; +} + +int framesed_detect_s16(framesed_t *engine, + const int16_t *samples, + size_t num_samples, + const framesed_options *options, + framesed_result *output) { + const framesed_options defaults = {.threads = 1}; + if (!options) + options = &defaults; + const int threads = options->threads > 0 ? options->threads : 1; + const double track_step = options->track_step; + if (!engine || (!samples && num_samples) || !output) + return -1; + memset(output, 0, sizeof(*output)); + if (num_samples < 2) + return 0; + + const size_t frames = output_frames(num_samples); + const int all_classes = options->all_classes; + const size_t num_groups = detection_group_count(all_classes); + size_t score_count; + if (checked_product(num_groups, frames, &score_count) != 0 || + score_count > SIZE_MAX / sizeof(float)) + return -1; + float *scores = calloc(score_count, sizeof(float)); + if (!scores) + return -1; + + detect_pool pool = { + .engine = engine, + .samples = samples, + .num_samples = num_samples, + .num_chunks = chunk_count_for(num_samples), + .frames = frames, + .scores = scores, + .all_classes = all_classes, + }; + atomic_init(&pool.next_chunk, 0); + atomic_init(&pool.failed, 0); + pool.progress = options->progress; + pool.progress_context = options->progress_context; + pthread_mutex_init(&pool.progress_lock, NULL); + + size_t workers = (size_t)threads; + if (workers > pool.num_chunks) + workers = pool.num_chunks; + pthread_t *pool_threads = workers > 1 ? calloc(workers - 1, sizeof(*pool_threads)) : NULL; + size_t started = 0; + if (pool_threads) + for (size_t i = 0; i + 1 < workers; ++i) + if (pthread_create(&pool_threads[i], NULL, detect_worker, &pool) == 0) + ++started; + detect_worker(&pool); + for (size_t i = 0; i < started; ++i) + pthread_join(pool_threads[i], NULL); + free(pool_threads); + pthread_mutex_destroy(&pool.progress_lock); + + if (atomic_load(&pool.failed)) { + free(scores); + return -1; + } + + /* Caption suppression must not modify the evidence returned as tracks or + * planes. All-class extraction leaves these scores unsuppressed. */ + float *unsuppressed = NULL; + if (track_step > 0.0 || options->keep_planes) { + unsuppressed = malloc(score_count * sizeof(float)); + if (!unsuppressed) { + free(scores); + return -1; + } + memcpy(unsuppressed, scores, score_count * sizeof(float)); + } + + const double duration = (double)num_samples / FRAMESED_SAMPLE_RATE; + if (extract_events(scores, frames, duration, output, all_classes) != 0 || + (unsuppressed && track_step > 0.0 && + build_tracks(unsuppressed, frames, track_step, output, all_classes) != 0)) { + framesed_result_free(output); + free(unsuppressed); + free(scores); + return -1; + } + free(scores); + + if (options->keep_planes) { + output->planes = unsuppressed; + output->num_groups = num_groups; + output->num_frames = frames; + } else { + free(unsuppressed); + } + return 0; +} + +void framesed_result_free(framesed_result *output) { + if (!output) + return; + free(output->events); + for (size_t i = 0; i < output->num_tracks; ++i) + free(output->tracks[i].values); + free(output->tracks); + free(output->planes); + memset(output, 0, sizeof(*output)); +} + +const char *framesed_backend(const framesed_t *engine) { return engine ? engine->backend : "unavailable"; } diff --git a/framesed/framesed.h b/framesed/framesed.h new file mode 100644 index 0000000..cc77607 --- /dev/null +++ b/framesed/framesed.h @@ -0,0 +1,100 @@ +#ifndef OFFLINE_AUDIO_FRAMESED_H +#define OFFLINE_AUDIO_FRAMESED_H + +#include <stddef.h> +#include <stdint.h> + +#define FRAMESED_SAMPLE_RATE 16000 +#define FRAMESED_CLASS_COUNT 447 +#define FRAMESED_FRAME_STEP 0.04 + +typedef struct framesed framesed_t; + +typedef struct { + float *logits; /* [FRAMESED_CLASS_COUNT][frames] */ + size_t frames; +} framesed_logits; + +typedef struct { + double start; + double end; + float score; + const char *label; + int class_group; /* caption group index, or model class index in all_classes mode */ +} framesed_event; + +/* A per-group or per-class detection score, downsampled from the model's 40 ms + * frames by taking the maximum in each output step. These are the scores + * before caption suppression, so a continuous background sound keeps its + * score while a more specific event is being captioned. */ +typedef struct { + const char *label; + int class_group; /* same index namespace as framesed_event.class_group */ + float *values; + size_t count; +} framesed_track; + +typedef struct { + framesed_event *events; + size_t num_events; + framesed_track *tracks; + size_t num_tracks; + double track_step; + /* Present only with keep_planes: unsuppressed scores [num_groups][num_frames] + * at FRAMESED_FRAME_STEP seconds, in framesed_group_label order, or framesed_class_label + * order with all_classes. In that mode num_groups is FRAMESED_CLASS_COUNT. */ + float *planes; + size_t num_groups; + size_t num_frames; +} framesed_result; + +/* A zeroed struct selects caption detection, serial execution, and no tracks. */ +typedef struct { + int threads; /* model windows in flight; 0 or 1 is serial */ + double track_step; /* seconds per score-track value; 0 returns no tracks */ + int keep_planes; /* retain the unsuppressed group/class score plane */ + /* Called once per finished model window from a worker thread, serialized + * by the engine. NULL disables it. */ + void (*progress)(void *context, size_t done, size_t total); + void *progress_context; + /* Nonzero returns each model class independently, without caption grouping + * or suppression. Uniform gates: start 0.50, release 0.35, minimum 0.24 s, + * gap 0.32 s. These are not per-class calibrated thresholds. */ + int all_classes; +} framesed_options; + +/* Caption-group label (regardless of detection mode), or NULL out of range. */ +const char *framesed_group_label(size_t group); +size_t framesed_group_count(void); + +/* Model output metadata, indexed 0..FRAMESED_CLASS_COUNT-1. Returned strings have + * static lifetime; invalid indices return NULL. IDs are AudioSet ontology IDs. */ +const char *framesed_class_label(size_t class_index); +const char *framesed_class_mid(size_t class_index); + +/* Loads the PretrainedSED frame_mn10 weight blob. */ +framesed_t *framesed_create(const char *weights_path); +void framesed_destroy(framesed_t *engine); + +/* Runs one 10-second model window and returns its native frame scores. */ +int framesed_infer_s16(framesed_t *engine, const int16_t *samples, size_t num_samples, framesed_logits *output); +void framesed_logits_free(framesed_logits *output); + +/* Detects caption-ready sound events, or individual model classes when + * all_classes is set, with bounded inference chunks. Events may overlap. + * Model windows run independently on up to `options->threads` workers; the result + * does not depend on the worker count. `options` may be NULL. + * + * A positive `track_step` in seconds also returns a score track for every + * group (or class) reaching its release threshold. The returned step is rounded + * down to whole model frames, with a minimum of FRAMESED_FRAME_STEP. */ +int framesed_detect_s16(framesed_t *engine, + const int16_t *samples, + size_t num_samples, + const framesed_options *options, + framesed_result *output); +void framesed_result_free(framesed_result *output); + +const char *framesed_backend(const framesed_t *engine); + +#endif diff --git a/framesed/framesed_classes.inc b/framesed/framesed_classes.inc new file mode 100644 index 0000000..a407651 --- /dev/null +++ b/framesed/framesed_classes.inc @@ -0,0 +1,448 @@ +/* Model output order; regenerate with make regenerate-classes. */ + {"/m/07q2z82", "Accelerating, revving, vroom"}, + {"/m/0gvgw0", "Air brake"}, + {"/m/025wky1", "Air conditioning"}, + {"/m/05x_td", "Air horn, truck horn"}, + {"/m/0k5j", "Aircraft"}, + {"/m/014yck", "Aircraft engine"}, + {"/m/07pp_mv", "Alarm"}, + {"/m/046dlr", "Alarm clock"}, + {"/t/dd00144", "Alert"}, + {"/m/012n7d", "Ambulance (siren)"}, + {"/m/0jbk", "Animal"}, + {"/m/028ght", "Applause"}, + {"/m/0dl83", "Arrow"}, + {"/m/0_1c", "Artillery fire"}, + {"/t/dd00142", "Audio logo"}, + {"/m/0261r1", "Babbling"}, + {"/t/dd00002", "Baby cry, infant cry"}, + {"/t/dd00001", "Baby laughter"}, + {"/m/093_4n", "Background noise"}, + {"/m/07pws3f", "Bang"}, + {"/m/05tny_", "Bark"}, + {"/m/018w8", "Basketball bounce"}, + {"/m/01j2bj", "Bathroom sounds"}, + {"/m/03dnzn", "Bathtub (filling or washing)"}, + {"/m/04gy_2", "Battle cry"}, + {"/m/01h3n", "Bee, wasp, etc."}, + {"/m/02fs_r", "Beep, bleep"}, + {"/m/0395lw", "Bell"}, + {"/m/07q4ntr", "Bellow"}, + {"/m/07sq110", "Belly laugh"}, + {"/m/0gy1t2s", "Bicycle bell"}, + {"/m/0199g", "Bicycle, tricycle"}, + {"/m/015p6", "Bird"}, + {"/m/05_wcq", "Bird flight, flapping wings"}, + {"/m/020bb7", "Bird vocalization, bird call, bird song"}, + {"/m/07pdhp0", "Biting"}, + {"/m/07q0h5t", "Bleat"}, + {"/m/02pjr4", "Blender, food processor"}, + {"/m/019jd", "Boat, Water vehicle"}, + {"/m/0dv3j", "Boiling"}, + {"/t/dd00121", "Boing"}, + {"/m/04v5dt", "Booing"}, + {"/m/07qqyl4", "Boom"}, + {"/m/07pqn27", "Bouncing"}, + {"/m/07rc7d9", "Bow-wow"}, + {"/m/07pc8lb", "Breaking"}, + {"/m/0lyf6", "Breathing"}, + {"/t/dd00138", "Brief tone"}, + {"/m/03q5_w", "Burping, eructation"}, + {"/m/07qsvvw", "Burst, pop"}, + {"/m/01bjv", "Bus"}, + {"/m/01z47d", "Busy signal"}, + {"/m/07pjwq1", "Buzz"}, + {"/m/030rvx", "Buzzer"}, + {"/m/07szfh9", "Cacophony"}, + {"/m/0dv5r", "Camera"}, + {"/m/01z5f", "Canidae, wild dogs, wolves"}, + {"/m/073cg4", "Cap gun"}, + {"/m/0k4j", "Car"}, + {"/m/02mfyn", "Car alarm"}, + {"/t/dd00134", "Car passing by"}, + {"/m/08dckq", "Carbon monoxide detector, CO detector"}, + {"/m/018p4k", "Cart"}, + {"/m/024dl", "Cash register"}, + {"/m/01yrx", "Cat"}, + {"/m/07r81j2", "Caterwaul"}, + {"/m/01xq0k1", "Cattle, bovinae"}, + {"/m/07r5c2p", "Caw"}, + {"/m/01sb50", "Cellphone buzz, vibrating alert"}, + {"/m/0269r2s", "Chain"}, + {"/m/01j4z9", "Chainsaw"}, + {"/m/0150b9", "Change ringing (campanology)"}, + {"/t/dd00123", "Channel, environment and background"}, + {"/m/02bk07", "Chant"}, + {"/m/053hz1", "Cheering"}, + {"/m/03cczk", "Chewing, mastication"}, + {"/m/09b5t", "Chicken, rooster"}, + {"/t/dd00005", "Child singing"}, + {"/m/0ytgt", "Child speech, kid speaking"}, + {"/t/dd00013", "Children playing"}, + {"/t/dd00135", "Children shouting"}, + {"/m/0f8s22", "Chime"}, + {"/m/02021", "Chipmunk"}, + {"/m/0hdsk", "Chirp tone"}, + {"/m/07pggtn", "Chirp, tweet"}, + {"/m/0l14jd", "Choir"}, + {"/m/07pczhz", "Chop"}, + {"/m/07pn_8q", "Chopping (food)"}, + {"/m/07m2kt", "Chorus effect"}, + {"/m/07rgt08", "Chuckle, chortle"}, + {"/m/03w41f", "Church bell"}, + {"/m/0dgbq", "Civil defense siren"}, + {"/m/07rv4dm", "Clang"}, + {"/m/0l15bq", "Clapping"}, + {"/m/07qmpdm", "Clatter"}, + {"/m/07rwm0c", "Clickety-clack"}, + {"/m/07qc9xj", "Clicking"}, + {"/m/07rv9rh", "Clip-clop"}, + {"/m/01x3z", "Clock"}, + {"/m/07st89h", "Cluck"}, + {"/t/dd00108", "Clunk"}, + {"/m/0242l", "Coin (dropping)"}, + {"/m/01m2v", "Computer keyboard"}, + {"/m/01h8n0", "Conversation"}, + {"/m/07r_25d", "Coo"}, + {"/m/01b_21", "Cough"}, + {"/m/0239kh", "Cowbell"}, + {"/m/07qs1cx", "Crack"}, + {"/m/07pzfmf", "Crackle"}, + {"/m/07qh7jl", "Creak"}, + {"/m/09xqv", "Cricket"}, + {"/m/07st88b", "Croak"}, + {"/m/0hgq8df", "Crockery breaking and smashing"}, + {"/m/04s8yn", "Crow"}, + {"/m/03qtwd", "Crowd"}, + {"/m/07qn5dc", "Crowing, cock-a-doodle-doo"}, + {"/t/dd00112", "Crumpling, crinkling"}, + {"/m/07s12q4", "Crunch"}, + {"/m/07plct2", "Crushing"}, + {"/m/0463cq4", "Crying, sobbing"}, + {"/m/0642b4", "Cupboard open or close"}, + {"/m/023pjk", "Cutlery, silverware"}, + {"/t/dd00110", "Deformable shell"}, + {"/m/08j51y", "Dental drill, dentist's drill"}, + {"/m/015jpf", "Dial tone"}, + {"/m/0160x5", "Digestive"}, + {"/m/07phxs1", "Ding"}, + {"/m/07r67yg", "Ding-dong"}, + {"/m/04brg2", "Dishes, pots, and pans"}, + {"/m/0g12c5", "Distortion"}, + {"/m/0bt9lr", "Dog"}, + {"/m/068hy", "Domestic animals, pets"}, + {"/t/dd00147", "Dong, bong"}, + {"/m/0ffhf", "Donkey, ass"}, + {"/m/02dgv", "Door"}, + {"/m/03wwcy", "Doorbell"}, + {"/m/0fqfqc", "Drawer open or close"}, + {"/m/01d380", "Drill"}, + {"/m/07r5v4s", "Drip"}, + {"/m/04179zz", "Duck call (hunting tool)"}, + {"/m/09ddx", "Ducks, geese, waterfowl"}, + {"/m/01jnbd", "Echo"}, + {"/m/02rr_", "Effects unit"}, + {"/m/0c1tlg", "Electric rotor drone, quadcopter"}, + {"/m/02g901", "Electric shaver, electric razor"}, + {"/m/04fgwm", "Electric toothbrush"}, + {"/m/0b_fwt", "Electronic tuner"}, + {"/m/03j1ly", "Emergency vehicle"}, + {"/m/02mk9", "Engine"}, + {"/m/01h82_", "Engine knocking"}, + {"/t/dd00130", "Engine starting"}, + {"/m/06_y0by", "Environmental noise"}, + {"/m/098_xr", "Error signal"}, + {"/m/07pxg6y", "Eruption"}, + {"/m/014zdl", "Explosion"}, + {"/m/02_nn", "Fart"}, + {"/t/dd00004", "Female singing"}, + {"/m/02zsn", "Female speech, woman speaking"}, + {"/m/02p01q", "Filing (rasp)"}, + {"/m/07p7b8y", "Fill (with liquid)"}, + {"/m/025_jnm", "Finger snapping"}, + {"/m/02_41", "Fire"}, + {"/m/0c3f7m", "Fire alarm"}, + {"/m/012ndj", "Fire engine, fire truck (siren)"}, + {"/g/122z_qxw", "Firecracker"}, + {"/m/0g6b5", "Fireworks"}, + {"/m/0cmf2", "Fixed-wing aircraft, airplane"}, + {"/t/dd00118", "Fizz"}, + {"/m/07pyf11", "Flap"}, + {"/m/0h2mp", "Fly, housefly"}, + {"/m/04fq5q", "Foghorn"}, + {"/m/025rv6n", "Fowl"}, + {"/m/09ld4", "Frog"}, + {"/m/0dxrf", "Frying (food)"}, + {"/m/02z32qm", "Fusillade"}, + {"/m/0939n_", "Gargling"}, + {"/m/07s0dtb", "Gasp"}, + {"/m/0l7xg", "Gears"}, + {"/t/dd00099", "Generic impact sounds"}, + {"/m/07r660_", "Giggle"}, + {"/m/039jq", "Glass"}, + {"/m/07q7njn", "Glass chink, clink"}, + {"/m/07rn7sz", "Glass shatter"}, + {"/m/03fwl", "Goat"}, + {"/m/07svc2k", "Gobble"}, + {"/m/07pt6mm", "Grind"}, + {"/m/07s2xch", "Groan"}, + {"/m/0ghcn6", "Growling"}, + {"/m/07r4k75", "Grunt"}, + {"/m/01dwxx", "Gull, seagull"}, + {"/m/032s66", "Gunshot, gunfire"}, + {"/m/07swgks", "Gurgling, bubbling"}, + {"/t/dd00088", "Gush"}, + {"/m/03wvsk", "Hair dryer"}, + {"/m/03l9g", "Hammer"}, + {"/m/0k65p", "Hands"}, + {"/m/01jg02", "Heart sounds, heartbeat"}, + {"/t/dd00067", "Heavy engine (low frequency)"}, + {"/m/09ct_", "Helicopter"}, + {"/m/02p3nc", "Hiccup"}, + {"/m/07rjwbb", "Hiss"}, + {"/m/07qwf61", "Honk"}, + {"/m/07r_80w", "Hoot"}, + {"/m/03k3r", "Horse"}, + {"/m/07qf0zm", "Howl"}, + {"/m/07q8f3b", "Howl (wind)"}, + {"/m/07qfr4h", "Hubbub, speech noise, speech babble"}, + {"/m/07rcgpl", "Hum"}, + {"/t/dd00012", "Human group actions"}, + {"/m/0bpl036", "Human locomotion"}, + {"/m/0dgw9r", "Human sounds"}, + {"/m/09l8g", "Human voice"}, + {"/m/02fxyj", "Humming"}, + {"/m/03cl9h", "Ice cream truck, ice cream van"}, + {"/m/07pb8fc", "Idling"}, + {"/m/03vt0", "Insect"}, + {"/t/dd00126", "Inside, large room or hall"}, + {"/t/dd00127", "Inside, public space"}, + {"/t/dd00125", "Inside, small room"}, + {"/m/03p19w", "Jackhammer"}, + {"/m/04229", "Jet engine"}, + {"/m/027m70_", "Jingle bell"}, + {"/m/07qfgpx", "Jingle, tinkle"}, + {"/g/11b630rrvh", "Kettle whistle"}, + {"/m/056r_1", "Keypress tone"}, + {"/m/03v3yw", "Keys jangling"}, + {"/m/0d4wf", "Kitchen and dining room sounds"}, + {"/m/04ctx", "Knife"}, + {"/m/07r4wb8", "Knock"}, + {"/m/01j3sz", "Laughter"}, + {"/m/01yg9g", "Lawn mower"}, + {"/t/dd00065", "Light engine (high frequency)"}, + {"/m/04k94", "Liquid"}, + {"/m/0ch8v", "Livestock, farm animals, working animals"}, + {"/m/02ll1_", "Lock"}, + {"/m/04zjc", "Machine gun"}, + {"/m/06xkwv", "Mains hum"}, + {"/t/dd00003", "Male singing"}, + {"/m/05zppz", "Male speech, man speaking"}, + {"/m/01c194", "Mantra"}, + {"/m/06cyt0", "Mechanical bell"}, + {"/m/02x984l", "Mechanical fan"}, + {"/t/dd00077", "Mechanisms"}, + {"/t/dd00066", "Medium engine (mid frequency)"}, + {"/m/07qrkrw", "Meow"}, + {"/m/0hg7b", "Microphone"}, + {"/m/0fx9l", "Microwave oven"}, + {"/m/07rpkh9", "Moo"}, + {"/m/09f96", "Mosquito"}, + {"/m/012f08", "Motor vehicle (road)"}, + {"/m/02rlv9", "Motorboat, speedboat"}, + {"/m/04_sv", "Motorcycle"}, + {"/m/04rmv", "Mouse"}, + {"/m/04rlf", "Music"}, + {"/m/02qldy", "Narration, monologue"}, + {"/m/07q5rw0", "Neigh, whinny"}, + {"/m/096m7z", "Noise"}, + {"/t/dd00061", "Non-motorized land vehicle"}, + {"/m/05kq4", "Ocean"}, + {"/t/dd00018", "Oink"}, + {"/t/dd00122", "Other sourceless"}, + {"/t/dd00128", "Outside, urban or manmade"}, + {"/m/09d5_", "Owl"}, + {"/m/05mxj0q", "Packing tape, duct tape"}, + {"/m/07pyy8b", "Pant"}, + {"/t/dd00141", "Pant (dog)"}, + {"/m/0641k", "Paper rustling"}, + {"/m/07r4gkf", "Patter"}, + {"/m/068zj", "Pig"}, + {"/m/0h0rv", "Pigeon, dove"}, + {"/m/07qwdck", "Ping"}, + {"/m/07qyrcz", "Plop"}, + {"/m/04qvtq", "Police car (siren)"}, + {"/m/07prgkl", "Pour"}, + {"/m/0md09", "Power saw, circular saw, table saw"}, + {"/m/0_ksk", "Power tool"}, + {"/m/04gxbd", "Power windows, electric windows"}, + {"/m/01m4t", "Printer"}, + {"/m/02l6bg", "Propeller, airscrew"}, + {"/m/07q34h3", "Puff"}, + {"/m/05zc1", "Pulleys"}, + {"/m/07pt_g0", "Pulse"}, + {"/m/07ptzwd", "Pump (liquid)"}, + {"/m/02yds9", "Purr"}, + {"/m/07qdb04", "Quack"}, + {"/m/0ltv", "Race car, auto racing"}, + {"/m/06bz3", "Radio"}, + {"/m/06d_3", "Rail transport"}, + {"/m/01g50p", "Railroad car, train wagon"}, + {"/m/06mb1", "Rain"}, + {"/t/dd00038", "Rain on surface"}, + {"/m/07r10fb", "Raindrop"}, + {"/m/06bxc", "Rapping"}, + {"/m/02bm9n", "Ratchet, pawl"}, + {"/m/07qn4z3", "Rattle"}, + {"/m/040b_t", "Refrigerator"}, + {"/m/09hlz4", "Respiratory sounds"}, + {"/m/01b9nn", "Reverberation"}, + {"/m/02rhddq", "Reversing beeps"}, + {"/m/0bcdqg", "Ringing tone, ringback tone"}, + {"/m/01hnzm", "Ringtone"}, + {"/m/04cvmfc", "Roar"}, + {"/m/0cdnk", "Roaring cats (lions, tigers)"}, + {"/m/06hps", "Rodents, rats, mice"}, + {"/m/07s8j8t", "Roll"}, + {"/m/056ks2", "Rowboat, canoe, kayak"}, + {"/m/07pdjhy", "Rub"}, + {"/m/07phhsh", "Rumble"}, + {"/m/06h7j", "Run"}, + {"/m/07qwyj0", "Rustle"}, + {"/m/0hsrw", "Sailboat, sailing ship"}, + {"/m/023vsd", "Sanding"}, + {"/m/01b82r", "Sawing"}, + {"/m/01lsmm", "Scissors"}, + {"/m/07qv4k0", "Scrape"}, + {"/m/07qb_dv", "Scratch"}, + {"/m/03qc9zr", "Screaming"}, + {"/m/07q8k13", "Screech"}, + {"/m/0llzx", "Sewing machine"}, + {"/m/07bgp", "Sheep"}, + {"/m/06q74", "Ship"}, + {"/m/07p6fty", "Shout"}, + {"/m/02f9f_", "Shower"}, + {"/m/07qv_x_", "Shuffle"}, + {"/m/05rj2", "Shuffling cards"}, + {"/m/07plz5l", "Sigh"}, + {"/m/01v_m0", "Sine wave"}, + {"/m/015lz1", "Singing"}, + {"/m/07bjf", "Single-lens reflex camera"}, + {"/m/0130jx", "Sink (filling or washing)"}, + {"/m/03kmc9", "Siren"}, + {"/m/07p9k1k", "Sizzle"}, + {"/m/06_fw", "Skateboard"}, + {"/m/07rjzl8", "Slam"}, + {"/m/07ryjzk", "Slap, smack"}, + {"/m/02y_763", "Sliding door"}, + {"/m/07p6mqd", "Slosh"}, + {"/m/07pqmly", "Slurp, drinking straw"}, + {"/m/07pjjrj", "Smash, crash"}, + {"/m/01y3hg", "Smoke detector, smoke alarm"}, + {"/m/078jl", "Snake"}, + {"/m/07pc8l3", "Snap"}, + {"/m/01hsr_", "Sneeze"}, + {"/m/07s04w4", "Snicker"}, + {"/m/07ppn3j", "Sniff"}, + {"/m/01d3sd", "Snoring"}, + {"/m/07q0yl5", "Snort"}, + {"/t/dd00139", "Snort (horse)"}, + {"/m/0790c", "Sonar"}, + {"/m/0193bn", "Sonic boom"}, + {"/m/0c1dj", "Sound effect"}, + {"/t/dd00091", "Sound equipment"}, + {"/m/07bm98", "Sound reproduction"}, + {"/m/09x0r", "Speech"}, + {"/m/0brhx", "Speech synthesizer"}, + {"/m/07rrlb6", "Splash, splatter"}, + {"/m/07pl1bw", "Splinter"}, + {"/m/07qlf79", "Spray"}, + {"/m/07sx8x_", "Squawk"}, + {"/m/07q6cd_", "Squeak"}, + {"/m/07s02z0", "Squeal"}, + {"/m/07qlwh6", "Squish"}, + {"/m/01lynh", "Stairs"}, + {"/m/07rgkc5", "Static"}, + {"/m/06wzb", "Steam"}, + {"/m/06hck5", "Steam whistle"}, + {"/m/07ptfmf", "Stir"}, + {"/m/01g90h", "Stomach rumble"}, + {"/m/07sk0jz", "Stomp, stamp"}, + {"/m/0j6m2", "Stream, river"}, + {"/m/0195fx", "Subway, metro, underground"}, + {"/t/dd00109", "Surface contact"}, + {"/m/07s13rg", "Sweeping"}, + {"/t/dd00006", "Synthetic singing"}, + {"/m/07qcpgn", "Tap"}, + {"/m/0fw86", "Tap dance"}, + {"/m/09d1b1", "Tape hiss"}, + {"/m/07qcx4z", "Tearing"}, + {"/m/07cx4", "Telephone"}, + {"/m/07pp8cl", "Telephone bell ringing"}, + {"/m/02c8p", "Telephone dialing, DTMF"}, + {"/m/07c52", "Television"}, + {"/m/0dl9sf8", "Throat clearing"}, + {"/m/07qnq_y", "Thump, thud"}, + {"/m/0ngt1", "Thunder"}, + {"/m/0jb2l", "Thunderstorm"}, + {"/m/07rrh0c", "Thunk"}, + {"/m/07qjznt", "Tick"}, + {"/m/07qjznl", "Tick-tock"}, + {"/m/0h9mv", "Tire squeal, skidding"}, + {"/m/01jt3m", "Toilet flush"}, + {"/m/07k1x", "Tools"}, + {"/m/012xff", "Toothbrush"}, + {"/m/0btp2", "Traffic noise, roadway noise"}, + {"/m/07jdr", "Train"}, + {"/m/0284vy3", "Train horn"}, + {"/t/dd00048", "Train wheels squealing"}, + {"/m/04zmvq", "Train whistle"}, + {"/m/07pqc89", "Trickle, dribble"}, + {"/m/07r04", "Truck"}, + {"/m/07n_g", "Tuning fork"}, + {"/m/01rd7k", "Turkey"}, + {"/m/0c2wf", "Typewriter"}, + {"/m/0316dw", "Typing"}, + {"/t/dd00143", "Unknown sound"}, + {"/m/0d31p", "Vacuum cleaner"}, + {"/m/07yv9", "Vehicle"}, + {"/m/0912c9", "Vehicle horn, car horn, honking, toot"}, + {"/m/0zmy2j9", "Velcro, hook and loop fastener"}, + {"/m/0bzvm2", "Video game sound"}, + {"/m/07qw_06", "Wail, moan"}, + {"/m/07pbtc8", "Walk, footsteps"}, + {"/m/0174k2", "Washing machine"}, + {"/m/0838f", "Water"}, + {"/m/02jz0l", "Water tap, faucet"}, + {"/m/0j2kx", "Waterfall"}, + {"/m/034srq", "Waves, surf"}, + {"/m/07rdhzs", "Whack, thwack"}, + {"/m/032n05", "Whale vocalization"}, + {"/m/07mzm6", "Wheeze"}, + {"/m/07qz6j3", "Whimper"}, + {"/t/dd00136", "Whimper (dog)"}, + {"/m/07rbp7_", "Whip"}, + {"/m/07s34ls", "Whir"}, + {"/m/02rtxlg", "Whispering"}, + {"/m/0l156k", "Whistle"}, + {"/m/01w250", "Whistling"}, + {"/m/0chx_", "White noise, pink noise"}, + {"/m/07rwj3x", "Whoop"}, + {"/m/07rqsjt", "Whoosh, swoosh, swish"}, + {"/m/01280g", "Wild animals"}, + {"/m/0fjy1", "Wildfire"}, + {"/m/03m9d0z", "Wind"}, + {"/m/026fgl", "Wind chime"}, + {"/t/dd00092", "Wind noise (microphone)"}, + {"/m/02417f", "Windscreen wiper, windshield wiper"}, + {"/m/07pk7mg", "Wobble"}, + {"/m/079vc8", "Wolf-whistling"}, + {"/m/083vt", "Wood"}, + {"/m/081rb", "Writing"}, + {"/m/01hhp3", "Yak"}, + {"/m/01j423", "Yawn"}, + {"/m/07sr1lc", "Yell"}, + {"/m/07r_k2n", "Yip"}, + {"/m/01swy6", "Yodeling"}, + {"/m/07p78v5", "Zing"}, + {"/m/01s0vc", "Zipper (clothing)"}, diff --git a/framesed/framesed_policy.inc b/framesed/framesed_policy.inc new file mode 100644 index 0000000..fc6d6ee --- /dev/null +++ b/framesed/framesed_policy.inc @@ -0,0 +1,85 @@ +/* Generated by tools/export_framesed_policy.py. */ +static const uint16_t framesed_group_classes[] = { + 27, 30, 71, 73, 81, 85, 90, 106, 221, 239, 253, 400, 433, 11, 93, 74, + 114, 17, 29, 89, 179, 229, 343, 16, 119, 420, 16, 24, 28, 80, 316, 321, + 428, 442, 424, 73, 77, 85, 156, 237, 239, 292, 327, 373, 444, 325, 176, 186, + 46, 176, 264, 345, 346, 419, 345, 105, 382, 342, 75, 20, 44, 130, 187, 204, + 421, 443, 64, 65, 187, 200, 244, 283, 32, 33, 34, 67, 84, 104, 113, 189, + 202, 262, 269, 358, 25, 52, 110, 169, 215, 248, 10, 36, 56, 66, 76, 82, + 97, 99, 106, 111, 115, 131, 133, 140, 171, 172, 183, 184, 201, 203, 233, 247, + 252, 255, 259, 267, 268, 284, 294, 301, 302, 303, 319, 340, 347, 401, 418, 430, + 440, 289, 290, 291, 384, 385, 205, 432, 434, 416, 108, 161, 431, 191, 200, 258, + 364, 369, 413, 415, 9, 91, 163, 272, 330, 407, 394, 395, 397, 4, 5, 166, + 220, 278, 198, 1, 3, 9, 30, 31, 38, 50, 58, 59, 60, 147, 163, 213, + 249, 250, 251, 257, 272, 276, 285, 287, 288, 298, 305, 310, 320, 332, 370, 389, + 393, 396, 399, 406, 407, 0, 70, 123, 148, 149, 150, 197, 214, 220, 230, 231, + 243, 411, 27, 30, 71, 81, 90, 106, 221, 400, 433, 223, 365, 425, 127, 134, + 135, 333, 335, 359, 374, 228, 51, 68, 124, 300, 378, 379, 380, 3, 6, 7, + 9, 30, 53, 59, 91, 127, 135, 162, 163, 170, 223, 272, 330, 339, 365, 407, + 425, 98, 387, 388, 102, 439, 128, 405, 390, 225, 446, 55, 328, 70, 312, 42, + 49, 153, 154, 349, 13, 57, 174, 190, 235, 164, 165, 180, 181, 182, 19, +}; + +static const framesed_group_definition framesed_groups[] = { + {"music", 0.50f, 0.35f, 0.80f, 0.56f, 0, 13, -1, UINT64_C(0x0)}, + {"applause", 0.50f, 0.35f, 0.48f, 0.40f, 13, 2, 3, UINT64_C(0x400000000000000)}, + {"cheering", 0.55f, 0.40f, 0.48f, 0.40f, 15, 1, 3, UINT64_C(0x0)}, + {"crowd", 0.62f, 0.47f, 0.80f, 0.56f, 16, 1, -1, UINT64_C(0x0)}, + {"laughter", 0.50f, 0.35f, 0.48f, 0.40f, 17, 6, -1, UINT64_C(0x0)}, + {"crying", 0.55f, 0.40f, 0.48f, 0.40f, 23, 3, -1, UINT64_C(0x0)}, + {"baby crying", 0.55f, 0.40f, 0.48f, 0.40f, 26, 1, 5, UINT64_C(0x0)}, + {"shouting", 0.58f, 0.43f, 0.48f, 0.40f, 27, 7, -1, UINT64_C(0x0)}, + {"whispering", 0.62f, 0.47f, 0.48f, 0.40f, 34, 1, -1, UINT64_C(0x0)}, + {"singing", 0.55f, 0.40f, 0.80f, 0.56f, 35, 10, -1, UINT64_C(0x0)}, + {"sighing", 0.60f, 0.45f, 0.48f, 0.40f, 45, 1, -1, UINT64_C(0x0)}, + {"gasping", 0.60f, 0.45f, 0.48f, 0.40f, 46, 1, -1, UINT64_C(0x0)}, + {"groaning", 0.60f, 0.45f, 0.48f, 0.40f, 47, 1, -1, UINT64_C(0x0)}, + {"breathing", 0.65f, 0.50f, 0.48f, 0.40f, 48, 6, -1, UINT64_C(0x0)}, + {"snoring", 0.58f, 0.43f, 0.80f, 0.56f, 54, 1, -1, UINT64_C(0x0)}, + {"coughing", 0.58f, 0.43f, 0.24f, 0.32f, 55, 2, -1, UINT64_C(0x0)}, + {"sneezing", 0.58f, 0.43f, 0.24f, 0.32f, 57, 1, -1, UINT64_C(0x0)}, + {"chewing", 0.62f, 0.47f, 0.48f, 0.40f, 58, 1, -1, UINT64_C(0x0)}, + {"dog", 0.55f, 0.40f, 0.48f, 0.40f, 59, 7, 22, UINT64_C(0x0)}, + {"cat", 0.55f, 0.40f, 0.48f, 0.40f, 66, 6, 22, UINT64_C(0x0)}, + {"birds", 0.55f, 0.40f, 0.48f, 0.40f, 72, 12, 22, UINT64_C(0x0)}, + {"insects", 0.58f, 0.43f, 0.48f, 0.40f, 84, 6, 22, UINT64_C(0x0)}, + {"animal", 0.65f, 0.50f, 0.80f, 0.56f, 90, 39, -1, UINT64_C(0x0)}, + {"rain", 0.58f, 0.43f, 0.80f, 0.56f, 129, 3, 28, UINT64_C(0x0)}, + {"thunder", 0.58f, 0.43f, 0.48f, 0.40f, 132, 2, -1, UINT64_C(0x0)}, + {"wind", 0.60f, 0.45f, 0.80f, 0.56f, 134, 3, -1, UINT64_C(0x0)}, + {"waves", 0.58f, 0.43f, 0.80f, 0.56f, 137, 1, 28, UINT64_C(0x0)}, + {"fire", 0.60f, 0.45f, 0.80f, 0.56f, 138, 3, -1, UINT64_C(0x0)}, + {"water", 0.65f, 0.50f, 0.80f, 0.56f, 141, 7, -1, UINT64_C(0x0)}, + {"siren", 0.55f, 0.40f, 0.48f, 0.40f, 148, 5, -1, UINT64_C(0x0)}, + {"horn", 0.58f, 0.43f, 0.24f, 0.32f, 153, 1, -1, UINT64_C(0x400000000000000)}, + {"train", 0.58f, 0.43f, 0.80f, 0.56f, 154, 3, 34, UINT64_C(0x0)}, + {"aircraft", 0.60f, 0.45f, 0.80f, 0.56f, 157, 5, 34, UINT64_C(0x0)}, + {"helicopter", 0.55f, 0.40f, 0.80f, 0.56f, 162, 1, 32, UINT64_C(0x0)}, + {"vehicle", 0.65f, 0.50f, 0.80f, 0.56f, 163, 34, -1, UINT64_C(0x0)}, + {"engine", 0.60f, 0.45f, 0.80f, 0.56f, 197, 12, 34, UINT64_C(0x0)}, + {"footsteps", 0.58f, 0.43f, 0.48f, 0.40f, 209, 1, -1, UINT64_C(0x0)}, + {"bell ringing", 0.58f, 0.43f, 0.48f, 0.40f, 210, 9, -1, UINT64_C(0x0)}, + {"whistle", 0.60f, 0.45f, 0.48f, 0.40f, 219, 3, -1, UINT64_C(0x0)}, + {"door", 0.60f, 0.45f, 0.48f, 0.40f, 222, 7, -1, UINT64_C(0x0)}, + {"knocking", 0.58f, 0.43f, 0.24f, 0.32f, 229, 1, 39, UINT64_C(0x400000000000000)}, + {"telephone", 0.58f, 0.43f, 0.48f, 0.40f, 230, 7, -1, UINT64_C(0x0)}, + {"alarm", 0.58f, 0.43f, 0.48f, 0.40f, 237, 20, -1, UINT64_C(0x0)}, + {"clock", 0.62f, 0.47f, 0.48f, 0.40f, 257, 3, -1, UINT64_C(0x0)}, + {"typing", 0.58f, 0.43f, 0.48f, 0.40f, 260, 1, -1, UINT64_C(0x0)}, + {"writing", 0.62f, 0.47f, 0.48f, 0.40f, 261, 1, -1, UINT64_C(0x0)}, + {"dishes", 0.60f, 0.45f, 0.48f, 0.40f, 262, 1, -1, UINT64_C(0x0)}, + {"vacuum cleaner", 0.58f, 0.43f, 0.80f, 0.56f, 263, 1, -1, UINT64_C(0x0)}, + {"toilet flush", 0.58f, 0.43f, 0.48f, 0.40f, 264, 1, 28, UINT64_C(0x0)}, + {"keys jangling", 0.60f, 0.45f, 0.48f, 0.40f, 265, 1, -1, UINT64_C(0x0)}, + {"zipper", 0.62f, 0.47f, 0.48f, 0.40f, 266, 1, -1, UINT64_C(0x0)}, + {"camera shutter", 0.60f, 0.45f, 0.24f, 0.32f, 267, 2, -1, UINT64_C(0x0)}, + {"chainsaw", 0.58f, 0.43f, 0.80f, 0.56f, 269, 1, -1, UINT64_C(0x800000000)}, + {"sawing", 0.60f, 0.45f, 0.48f, 0.40f, 270, 1, -1, UINT64_C(0x0)}, + {"explosion", 0.60f, 0.45f, 0.24f, 0.32f, 271, 5, -1, UINT64_C(0x400000000000000)}, + {"gunfire", 0.60f, 0.45f, 0.24f, 0.32f, 276, 5, 54, UINT64_C(0x400000000000000)}, + {"fireworks", 0.58f, 0.43f, 0.24f, 0.32f, 281, 2, 54, UINT64_C(0x400000000000000)}, + {"glass breaking", 0.58f, 0.43f, 0.24f, 0.32f, 283, 3, -1, UINT64_C(0x400000000000000)}, + {"impact", 0.65f, 0.50f, 0.24f, 0.32f, 286, 1, -1, UINT64_C(0x0)}, +}; + +#define FRAMESED_GROUP_COUNT (sizeof(framesed_groups) / sizeof(framesed_groups[0])) diff --git a/model_blob.c b/model_blob.c new file mode 100644 index 0000000..a13d113 --- /dev/null +++ b/model_blob.c @@ -0,0 +1,111 @@ +/* The file contains a header, a fixed-size table of named tensors, and tensor + * data at 64-byte boundaries. */ + +#include "model_blob.h" + +#include <fcntl.h> +#include <stdint.h> +#include <string.h> +#include <sys/mman.h> +#include <sys/stat.h> +#include <unistd.h> + +#define MAGIC "SDC1" +#define VERSION 1u +#define NAME_LEN 24 +#define MAX_TENSORS 1024 + +typedef struct { + char magic[4]; + uint32_t version; + uint32_t count; + uint32_t total; +} blob_header; + +typedef struct { + char name[NAME_LEN]; + uint32_t offset; + uint32_t nbytes; + uint32_t digest; +} blob_entry; + +static uint32_t fnv1a(const unsigned char *p, size_t n) { + uint32_t h = 0x811C9DC5u; + for (size_t i = 0; i < n; ++i) + h = (h ^ p[i]) * 0x01000193u; + return h; +} + +int framesed_model_blob_parse(const void *data, size_t len, int verify, framesed_model_blob *out) { + if (!data || len < sizeof(blob_header)) + return -1; + const blob_header *h = (const blob_header *)data; + if (memcmp(h->magic, MAGIC, 4) != 0 || h->version != VERSION) + return -1; + if (h->count == 0 || h->count > MAX_TENSORS) + return -1; + const size_t table_bytes = (size_t)h->count * sizeof(blob_entry); + if (len < sizeof(blob_header) + table_bytes || len < h->total) + return -1; + + out->base = (const unsigned char *)data; + out->len = len; + out->table = (const unsigned char *)data + sizeof *h; + out->count = h->count; + out->verify = verify; + out->failed = 0; + return 0; +} + +const float *framesed_model_blob_take(framesed_model_blob *blob, const char *name, size_t floats) { + if (blob->failed) + return NULL; + if (floats > SIZE_MAX / sizeof(float)) { + blob->failed = 1; + return NULL; + } + const blob_entry *table = (const blob_entry *)blob->table; + for (unsigned i = 0; i < blob->count; ++i) { + const blob_entry *e = &table[i]; + if (strncmp(e->name, name, NAME_LEN) != 0) + continue; + if (e->nbytes != floats * sizeof(float) || e->offset % 64 != 0 || e->offset > blob->len || + e->nbytes > blob->len - e->offset) + break; + const unsigned char *payload = blob->base + e->offset; + if (blob->verify && fnv1a(payload, e->nbytes) != e->digest) + break; + return (const float *)(const void *)payload; + } + blob->failed = 1; + return NULL; +} + +int framesed_model_blob_map( + const char *path, int verify, framesed_model_blob *out, void **map, size_t *map_len) { + const int fd = open(path, O_RDONLY); + if (fd < 0) + return -1; + struct stat st; + if (fstat(fd, &st) != 0 || st.st_size <= 0) { + close(fd); + return -1; + } + const size_t len = (size_t)st.st_size; + void *addr = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0); + close(fd); + if (addr == MAP_FAILED) + return -1; + if (framesed_model_blob_parse(addr, len, verify, out) != 0) { + munmap(addr, len); + return -1; + } + *map = addr; + *map_len = len; + return 0; +} + +void framesed_model_blob_unmap(void *map, size_t len) { + if (map) + munmap(map, len); +} diff --git a/model_blob.h b/model_blob.h new file mode 100644 index 0000000..433947d --- /dev/null +++ b/model_blob.h @@ -0,0 +1,36 @@ +/* Access to packed, named float tensors in a read-only model blob. */ + +#ifndef FRAMESED_MODEL_BLOB_H +#define FRAMESED_MODEL_BLOB_H + +#include <stddef.h> + +/* Tensor lookup returns NULL and records a failure, so a loader can request + * every tensor in a straight line and check the blob once at the end. */ +typedef struct { + const unsigned char *base; + size_t len; + const void *table; + unsigned count; + int verify; + int failed; +} framesed_model_blob; + +/* Returns 0 when the header and table are well formed. Tensor contents are + * only checked as they are taken. */ +int framesed_model_blob_parse(const void *data, size_t len, int verify, framesed_model_blob *out); + +/* The tensor called `name`, which must be exactly `floats` long. Returns NULL + * and marks the blob failed on any mismatch, so a blob exported from a + * different checkpoint or a different layout is rejected at load time rather + * than producing plausible wrong numbers. */ +const float *framesed_model_blob_take(framesed_model_blob *blob, const char *name, size_t floats); + +/* Maps `path` read-only and parses it. On success the caller must keep the + * mapping alive for as long as any pointer taken from it is used. */ +int framesed_model_blob_map( + const char *path, int verify, framesed_model_blob *out, void **map, size_t *map_len); + +void framesed_model_blob_unmap(void *map, size_t len); + +#endif diff --git a/testdata/framesed_reference.bin b/testdata/framesed_reference.bin Binary files differnew file mode 100644 index 0000000..b7d7216 --- /dev/null +++ b/testdata/framesed_reference.bin diff --git a/tests/test_classes.c b/tests/test_classes.c new file mode 100644 index 0000000..6dcf19f --- /dev/null +++ b/tests/test_classes.c @@ -0,0 +1,124 @@ +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "framesed.h" + +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ + } \ +} while (0) + +static int check_result(const framesed_result *result, double duration) { + CHECK(result->planes && result->num_groups == FRAMESED_CLASS_COUNT); + CHECK(result->num_frames == (size_t)ceil(duration / FRAMESED_FRAME_STEP)); + for (size_t i = 0; i < result->num_groups * result->num_frames; ++i) + CHECK(result->planes[i] >= 0.0f && result->planes[i] <= 1.0f); + int music = 0; + for (size_t i = 0; i < result->num_events; ++i) { + const framesed_event *event = &result->events[i]; + CHECK(event->class_group >= 0 && event->class_group < FRAMESED_CLASS_COUNT); + CHECK(event->label == framesed_class_label((size_t)event->class_group)); + CHECK(event->start >= 0.0 && event->end > event->start && event->end <= duration); + CHECK(event->score >= 0.50f && event->score <= 1.0f); + CHECK(i == 0 || result->events[i - 1].start <= event->start); + if (event->class_group == 253) + music = 1; + } + CHECK(music); + CHECK(result->num_tracks > 0 && result->track_step == 0.2); + for (size_t i = 0; i < result->num_tracks; ++i) { + const framesed_track *track = &result->tracks[i]; + CHECK(track->class_group >= 0 && track->class_group < FRAMESED_CLASS_COUNT); + CHECK(track->label == framesed_class_label((size_t)track->class_group)); + CHECK(track->count == (result->num_frames + 4) / 5); + CHECK(i == 0 || result->tracks[i - 1].class_group < track->class_group); + const float *row = result->planes + (size_t)track->class_group * result->num_frames; + for (size_t j = 0; j < track->count; ++j) { + float maximum = 0.0f; + for (size_t f = j * 5; f < (j + 1) * 5 && f < result->num_frames; ++f) + if (row[f] > maximum) + maximum = row[f]; + CHECK(track->values[j] == maximum); + } + } + return 0; +} + +int main(int argc, char **argv) { + const char *weights = argc > 1 ? argv[1] : "models/framesed_weights.bin"; + /* A partial final frame/window exercises padding and duration clipping. */ + const size_t count = 30u * FRAMESED_SAMPLE_RATE + 137; + const double duration = (double)count / FRAMESED_SAMPLE_RATE; + int16_t *samples = malloc(count * sizeof(*samples)); + CHECK(samples); + for (size_t i = 0; i < count; ++i) { + const double t = (double)i / FRAMESED_SAMPLE_RATE; + double value = (sin(2 * M_PI * 261.6 * t) + sin(2 * M_PI * 329.6 * t) + + sin(2 * M_PI * 392.0 * t)) / 3; + value *= 0.65 + 0.35 * sin(2 * M_PI * 4.0 * t); + samples[i] = (int16_t)(value * 0.10 * 32767.0); + } + framesed_t *engine = framesed_create(weights); + CHECK(engine); + framesed_options options = { + .threads = 1, .track_step = 0.2, .keep_planes = 1, .all_classes = 1, + }; + framesed_result one = {0}; + CHECK(framesed_detect_s16(engine, samples, 10u * FRAMESED_SAMPLE_RATE, &options, &one) == 0); + CHECK(check_result(&one, 10.0) == 0); + framesed_logits logits = {0}; + CHECK(framesed_infer_s16(engine, samples, 10u * FRAMESED_SAMPLE_RATE, &logits) == 0); + CHECK(one.num_frames == logits.frames); + for (size_t i = 0; i < FRAMESED_CLASS_COUNT * logits.frames; ++i) { + const float expected = 1.0f / (1.0f + expf(-logits.logits[i])); + CHECK(fabsf(one.planes[i] - expected) < 2e-7f); + } + framesed_logits_free(&logits); + framesed_result_free(&one); + + CHECK(framesed_detect_s16(engine, samples, count, &options, &one) == 0); + CHECK(check_result(&one, duration) == 0); + options.threads = 4; + framesed_result many = {0}; + CHECK(framesed_detect_s16(engine, samples, count, &options, &many) == 0); + CHECK(check_result(&many, duration) == 0); + CHECK(one.num_events == many.num_events && one.num_tracks == many.num_tracks); + CHECK(one.num_frames == many.num_frames && one.num_groups == many.num_groups); + CHECK(memcmp(one.planes, many.planes, + one.num_frames * one.num_groups * sizeof(float)) == 0); + for (size_t i = 0; i < one.num_events; ++i) { + const framesed_event *a = &one.events[i], *b = &many.events[i]; + CHECK(a->class_group == b->class_group && a->label == b->label); + CHECK(a->start == b->start && a->end == b->end && a->score == b->score); + } + for (size_t i = 0; i < one.num_tracks; ++i) { + const framesed_track *a = &one.tracks[i], *b = &many.tracks[i]; + CHECK(a->class_group == b->class_group && a->label == b->label && a->count == b->count); + CHECK(memcmp(a->values, b->values, a->count * sizeof(float)) == 0); + } + printf("framesed classes: 447 score planes match native logits; 1/4 workers agree " + "(%zu events, %zu tracks)\n", one.num_events, one.num_tracks); + framesed_result_free(&one); + framesed_result_free(&many); + options.keep_planes = 0; + options.track_step = 0; + CHECK(framesed_detect_s16(engine, samples, 10u * FRAMESED_SAMPLE_RATE, &options, &one) == 0); + CHECK(one.num_events > 0 && !one.planes && !one.tracks); + CHECK(one.num_groups == 0 && one.num_frames == 0 && one.num_tracks == 0); + framesed_result_free(&one); + options.keep_planes = 1; + CHECK(framesed_detect_s16(engine, samples, 17, &options, &one) == 0); + CHECK(one.num_groups == FRAMESED_CLASS_COUNT && one.num_frames == 1 && one.planes); + CHECK(one.num_events == 0); + framesed_result_free(&one); + CHECK(framesed_detect_s16(engine, NULL, 0, &options, &one) == 0); + CHECK(!one.planes && !one.events && !one.tracks); + framesed_result_free(&one); + framesed_destroy(engine); + free(samples); + return 0; +} diff --git a/tests/test_cli.py b/tests/test_cli.py new file mode 100644 index 0000000..858f439 --- /dev/null +++ b/tests/test_cli.py @@ -0,0 +1,76 @@ +"""CLI vocabulary, caption/class output, and argument contracts.""" + +import csv +import math +from pathlib import Path +import struct +import subprocess +import sys +import tempfile +import wave + + +WEIGHTS = sys.argv[1] if len(sys.argv) > 1 else "models/framesed_weights.bin" + + +def run(*args, code=0): + result = subprocess.run( + ["build/framesed", *map(str, args)], capture_output=True, text=True + ) + if result.returncode != code: + raise AssertionError(f"{args}: exit {result.returncode}\n{result.stderr}") + return result.stdout + + +def main(): + assert "--all-classes" in run("--help") + with Path("framesed/classes.csv").open(newline="", encoding="utf-8") as source: + rows = list(csv.DictReader(source)) + expected = "".join( + f"{row['index']}\t{row['mid']}\t{row['display_name']}\n" for row in rows + ) + assert len(rows) == 447 + assert run("--list-classes") == expected + assert run("--all-classes", code=2) == "" + run("--threads", "0", code=2) + run("--track-step", "nan", code=2) + run("--unknown-option", code=2) + + with tempfile.TemporaryDirectory(prefix="framesed-cli-") as directory: + path = Path(directory) / "music.wav" + samples = [] + for i in range(10 * 16000): + t = i / 16000 + value = sum(math.sin(2 * math.pi * f * t) for f in (261.6, 329.6, 392.0)) / 3 + value *= 0.65 + 0.35 * math.sin(2 * math.pi * 4 * t) + samples.append(int(value * 0.10 * 32767)) + with wave.open(str(path), "wb") as output: + output.setparams((1, 2, 16000, 0, "NONE", "not compressed")) + output.writeframes(struct.pack(f"<{len(samples)}h", *samples)) + caption = run("--weights", WEIGHTS, path) + assert any(line.endswith(" 0 music") for line in caption.splitlines()) + classes = run("--all-classes", "--track-step", "0.2", "--weights", WEIGHTS, path) + events = 0 + tracks = 0 + music = False + for line in classes.splitlines(): + kind = line.split(" ", 1)[0] + if kind == "event": + _, start, end, score, index, label = line.split(" ", 5) + assert 0 <= float(start) < float(end) <= 10.0 + events += 1 + music |= int(index) == 253 + else: + assert kind == "track" + _, time, score, index, label = line.split(" ", 4) + assert 0 <= float(time) < 10.0 + tracks += 1 + assert 0 <= float(score) <= 1 + assert 0 <= int(index) < 447 + assert label == rows[int(index)]["display_name"] + assert events and tracks and music + print("framesed CLI: 447 labels, caption/class detection, tracks, arguments passed") + + +if __name__ == "__main__": + main() diff --git a/tests/test_framesed.c b/tests/test_framesed.c new file mode 100644 index 0000000..5294a79 --- /dev/null +++ b/tests/test_framesed.c @@ -0,0 +1,280 @@ +#include <math.h> +#include <stdint.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "framesed.h" + +typedef struct { + char magic[12]; + uint32_t version; + uint32_t samples; + uint32_t frames; + uint32_t classes; + uint32_t tracks; +} reference_header; + +static int16_t *make_waveform(size_t count) { + int16_t *samples = malloc(count * sizeof(*samples)); + if (!samples) + return NULL; + uint32_t state = 1; + for (size_t index = 0; index < count; ++index) { + state = state * 1664525u + 1013904223u; + const int noise = (int)((state >> 16) & 0xffffu) - 32768; + const int saw = (int)(index % 1600) - 800; + int value = noise / 8 + saw * 20; + if (value < -32768) + value = -32768; + if (value > 32767) + value = 32767; + samples[index] = (int16_t)value; + } + return samples; +} + +/* A major triad under a 4 Hz tremolo, at the moderate level the model scores + * highest for music. This gives the event and score-track gates below + * something to detect without shipping a recording. */ +static int16_t *make_music_waveform(size_t count) { + int16_t *samples = malloc(count * sizeof(*samples)); + if (!samples) + return NULL; + static const double partials[] = {261.6, 329.6, 392.0}; + for (size_t index = 0; index < count; ++index) { + const double t = (double)index / FRAMESED_SAMPLE_RATE; + double value = 0.0; + for (size_t p = 0; p < sizeof partials / sizeof partials[0]; ++p) + value += sin(2.0 * M_PI * partials[p] * t); + value /= (double)(sizeof partials / sizeof partials[0]); + value *= 0.65 + 0.35 * sin(2.0 * M_PI * 4.0 * t); + samples[index] = (int16_t)(value * 0.10 * 32767.0); + } + return samples; +} + +static void *read_file(const char *path, size_t *size) { + FILE *file = fopen(path, "rb"); + if (!file) + return NULL; + if (fseek(file, 0, SEEK_END) != 0) { + fclose(file); + return NULL; + } + const long length = ftell(file); + if (length <= 0 || fseek(file, 0, SEEK_SET) != 0) { + fclose(file); + return NULL; + } + void *data = malloc((size_t)length); + if (!data || fread(data, 1, (size_t)length, file) != (size_t)length) { + free(data); + fclose(file); + return NULL; + } + fclose(file); + *size = (size_t)length; + return data; +} + +int main(int argc, char **argv) { + const char *weights = argc > 1 ? argv[1] : "models/framesed_weights.bin"; + const char *reference_path = argc > 2 ? argv[2] : "testdata/framesed_reference.bin"; + size_t reference_size = 0; + unsigned char *reference = read_file(reference_path, &reference_size); + if (!reference || reference_size < sizeof(reference_header)) { + fprintf(stderr, "cannot read FRAMESED reference\n"); + free(reference); + return 1; + } + const reference_header *header = (const reference_header *)reference; + if (memcmp(header->magic, "FRAMESEDR", 9) != 0 || header->version != 1 || + header->classes != FRAMESED_CLASS_COUNT) { + fprintf(stderr, "invalid FRAMESED reference\n"); + free(reference); + return 1; + } + const size_t expected_size = + sizeof(*header) + (size_t)header->tracks * sizeof(uint32_t) + + ((size_t)header->classes + (size_t)header->tracks * header->frames) * sizeof(float); + if (reference_size != expected_size) { + fprintf(stderr, "truncated FRAMESED reference\n"); + free(reference); + return 1; + } + + int16_t *samples = make_waveform(header->samples); + framesed_t *engine = framesed_create(weights); + framesed_logits result; + if (!samples || !engine || framesed_infer_s16(engine, samples, header->samples, &result) != 0) { + fprintf(stderr, "FRAMESED inference failed\n"); + free(samples); + framesed_destroy(engine); + free(reference); + return 1; + } + if (result.frames != header->frames) { + fprintf(stderr, "FRAMESED frame count differs: %zu != %u\n", result.frames, header->frames); + framesed_logits_free(&result); + framesed_destroy(engine); + free(samples); + free(reference); + return 1; + } + + const uint32_t *track_classes = (const uint32_t *)(reference + sizeof(*header)); + const float *expected_global = (const float *)(track_classes + header->tracks); + const float *expected_tracks = expected_global + header->classes; + float max_global = 0.0f; + float max_track = 0.0f; + for (uint32_t class_index = 0; class_index < header->classes; ++class_index) { + const float *row = result.logits + (size_t)class_index * result.frames; + float sum = 0.0f; + for (size_t frame = 0; frame < result.frames; ++frame) + sum += row[frame]; + const float error = fabsf(sum / result.frames - expected_global[class_index]); + if (error > max_global) + max_global = error; + } + for (uint32_t track = 0; track < header->tracks; ++track) { + const float *actual = result.logits + (size_t)track_classes[track] * result.frames; + const float *expected = expected_tracks + (size_t)track * result.frames; + for (size_t frame = 0; frame < result.frames; ++frame) { + const float error = fabsf(actual[frame] - expected[frame]); + if (error > max_track) + max_track = error; + } + } + + printf("framesed parity: global %.6g, tracks %.6g (%s)\n", max_global, max_track, framesed_backend(engine)); + int failed = max_global > 2e-3f || max_track > 4e-3f; + framesed_logits_free(&result); + framesed_result events = {0}; + const framesed_options serial = {.threads = 1, .track_step = 0.2}; + if (framesed_detect_s16(engine, samples, header->samples, &serial, &events) != 0) { + fprintf(stderr, "FRAMESED event detection failed\n"); + failed = 1; + } + for (size_t i = 0; i < events.num_events; ++i) { + const framesed_event *event = &events.events[i]; + if (!event->label || event->start < 0.0 || event->end <= event->start || + event->end > (double)header->samples / FRAMESED_SAMPLE_RATE + 1e-9 || event->score < 0.0f || + event->score > 1.0f || (i && events.events[i - 1].start > event->start)) { + fprintf(stderr, "invalid FRAMESED event result\n"); + failed = 1; + break; + } + } + printf("framesed policy: %zu deterministic event%s\n", events.num_events, + events.num_events == 1 ? "" : "s"); + + /* Model windows write disjoint frame ranges chosen by window index, so the + * worker count must not reach the output. */ + framesed_result threaded = {0}; + const framesed_options parallel = {.threads = 4, .track_step = 0.2}; + if (framesed_detect_s16(engine, samples, header->samples, ¶llel, &threaded) != 0) { + fprintf(stderr, "threaded FRAMESED event detection failed\n"); + failed = 1; + } else if (threaded.num_events != events.num_events) { + fprintf(stderr, "FRAMESED event count depends on the worker count: %zu vs %zu\n", + threaded.num_events, events.num_events); + failed = 1; + } else { + for (size_t i = 0; i < events.num_events; ++i) { + const framesed_event *one = &events.events[i]; + const framesed_event *many = &threaded.events[i]; + if (one->start != many->start || one->end != many->end || one->score != many->score || + one->label != many->label) { + fprintf(stderr, "FRAMESED event %zu depends on the worker count\n", i); + failed = 1; + break; + } + } + } + printf("framesed threads: 1 and 4 workers agree exactly\n"); + + const double expected_step = 0.2; + const size_t expected_count = + (size_t)((double)header->samples / FRAMESED_SAMPLE_RATE / expected_step + 0.9999); + for (size_t i = 0; i < events.num_tracks; ++i) { + const framesed_track *track = &events.tracks[i]; + if (track->count != expected_count || !track->label) { + fprintf(stderr, "score track %zu has %zu values, expected %zu\n", i, track->count, + expected_count); + failed = 1; + break; + } + for (size_t v = 0; v < track->count; ++v) + if (!(track->values[v] >= 0.0f && track->values[v] <= 1.0f)) { + fprintf(stderr, "score track %s leaves [0, 1]\n", track->label); + failed = 1; + break; + } + } + if (events.num_tracks && events.track_step != expected_step) { + fprintf(stderr, "score track step is %g, expected %g\n", events.track_step, expected_step); + failed = 1; + } + for (size_t i = 0; i < events.num_tracks; ++i) { + const framesed_track *one = &events.tracks[i]; + const framesed_track *many = &threaded.tracks[i]; + if (i >= threaded.num_tracks || one->count != many->count || + memcmp(one->values, many->values, one->count * sizeof(float))) { + fprintf(stderr, "score track %zu depends on the worker count\n", i); + failed = 1; + break; + } + } + printf("framesed scores: %zu track%s of %zu values at %.2f s\n", events.num_tracks, + events.num_tracks == 1 ? "" : "s", events.num_tracks ? events.tracks[0].count : 0, + events.track_step); + framesed_result_free(&threaded); + + /* A signal the model does recognise, so that events and score tracks are + * exercised rather than merely being empty and well formed. */ + const framesed_options music_options = {.threads = 2, .track_step = 1.0, .keep_planes = 1}; + const size_t music_samples = 30u * FRAMESED_SAMPLE_RATE; + int16_t *music = make_music_waveform(music_samples); + framesed_result music_result = {0}; + if (!music || framesed_detect_s16(engine, music, music_samples, &music_options, &music_result) != 0) { + fprintf(stderr, "music FRAMESED detection failed\n"); + failed = 1; + } else { + size_t music_events = 0; + double covered = 0.0; + for (size_t i = 0; i < music_result.num_events; ++i) + if (!strcmp(music_result.events[i].label, "music")) { + ++music_events; + covered += music_result.events[i].end - music_result.events[i].start; + } + const framesed_track *music_track = NULL; + for (size_t i = 0; i < music_result.num_tracks; ++i) + if (!strcmp(music_result.tracks[i].label, "music")) + music_track = &music_result.tracks[i]; + float peak = 0.0f; + if (music_track) + for (size_t v = 0; v < music_track->count; ++v) + if (music_track->values[v] > peak) + peak = music_track->values[v]; + if (!music_events || covered < 20.0 || !music_track || peak < 0.5f || + music_result.track_step != 1.0 || music_track->count != 30) { + fprintf(stderr, + "music gate: %zu events over %.1f s, track %s peak %.3f, " + "step %g, %zu values\n", + music_events, covered, music_track ? "present" : "missing", (double)peak, + music_result.track_step, music_track ? music_track->count : 0); + failed = 1; + } else { + printf("framesed music: %zu event%s over %.1f s, track peak %.2f at 1.00 s\n", music_events, + music_events == 1 ? "" : "s", covered, (double)peak); + } + } + framesed_result_free(&music_result); + free(music); + framesed_result_free(&events); + framesed_destroy(engine); + free(samples); + free(reference); + return failed; +} diff --git a/tests/test_policy.c b/tests/test_policy.c new file mode 100644 index 0000000..b8731bd --- /dev/null +++ b/tests/test_policy.c @@ -0,0 +1,140 @@ +/* Synthetic scores isolate policy contracts from model recognition accuracy. */ +#include "../framesed/framesed.c" + +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \ + return 1; \ + } \ +} while (0) + +static int test_metadata(void) { + CHECK(framesed_group_count() == 59); + CHECK(!strcmp(framesed_group_label(0), "music")); + CHECK(framesed_group_label(framesed_group_count()) == NULL); + CHECK(framesed_class_label(FRAMESED_CLASS_COUNT) == NULL); + CHECK(framesed_class_mid(SIZE_MAX) == NULL); + CHECK(!strcmp(framesed_class_label(0), "Accelerating, revving, vroom")); + CHECK(!strcmp(framesed_class_label(253), "Music")); + CHECK(!strcmp(framesed_class_mid(253), "/m/04rlf")); + CHECK(!strcmp(framesed_class_label(446), "Zipper (clothing)")); + for (size_t i = 0; i < FRAMESED_CLASS_COUNT; ++i) { + CHECK(framesed_class_label(i) && *framesed_class_label(i)); + CHECK(framesed_class_mid(i) && *framesed_class_mid(i)); + for (size_t j = 0; j < i; ++j) { + CHECK(strcmp(framesed_class_label(i), framesed_class_label(j))); + CHECK(strcmp(framesed_class_mid(i), framesed_class_mid(j))); + } + } + return 0; +} + +static int test_probabilities(void) { + enum { FRAMES = 2 }; + float logits[FRAMESED_CLASS_COUNT * FRAMES]; + float scores[FRAMESED_CLASS_COUNT * FRAMES]; + for (size_t i = 0; i < FRAMESED_CLASS_COUNT * FRAMES; ++i) + logits[i] = ((float)i - 447.0f) / 100.0f; + const framesed_logits input = {.logits = logits, .frames = FRAMES}; + CHECK(group_probabilities(&input, scores, 1) == 0); + for (size_t i = 0; i < FRAMESED_CLASS_COUNT * FRAMES; ++i) + CHECK(scores[i] == sigmoid(logits[i])); + CHECK(group_probabilities(&input, scores, 0) == 0); + for (size_t group = 0; group < FRAMESED_GROUP_COUNT; ++group) { + const framesed_group_definition *d = &framesed_groups[group]; + CHECK(d->class_count > 0); + CHECK((size_t)d->class_offset + d->class_count <= + sizeof(framesed_group_classes) / sizeof(framesed_group_classes[0])); + for (size_t frame = 0; frame < FRAMES; ++frame) { + float maximum = 0.0f; + for (size_t member = 0; member < d->class_count; ++member) { + const size_t index = framesed_group_classes[d->class_offset + member]; + CHECK(index < FRAMESED_CLASS_COUNT); + const float p = sigmoid(logits[index * FRAMES + frame]); + if (p > maximum) + maximum = p; + } + CHECK(scores[group * FRAMES + frame] == maximum); + } + } + return 0; +} + +static int test_all_classes(void) { + enum { FRAMES = 12 }; + float scores[FRAMESED_CLASS_COUNT * FRAMES]; + for (size_t i = 0; i < FRAMESED_CLASS_COUNT * FRAMES; ++i) + scores[i] = 0.75f; + framesed_result result = {0}; + CHECK(extract_events(scores, FRAMES, 0.48, &result, 1) == 0); + CHECK(result.num_events == FRAMESED_CLASS_COUNT); + CHECK(build_tracks(scores, FRAMES, 0.2, &result, 1) == 0); + CHECK(result.num_tracks == FRAMESED_CLASS_COUNT); + for (size_t i = 0; i < FRAMESED_CLASS_COUNT; ++i) { + const framesed_event *event = &result.events[i]; + CHECK(event->class_group == (int)i); + CHECK(event->label == framesed_class_label(i)); + CHECK(event->start == 0.0 && event->end == 0.48 && event->score == 0.75f); + const framesed_track *track = &result.tracks[i]; + CHECK(track->class_group == (int)i && track->label == framesed_class_label(i)); + CHECK(track->count == 3); + for (size_t frame = 0; frame < track->count; ++frame) + CHECK(track->values[frame] == 0.75f); + } + framesed_result_free(&result); + return 0; +} + +static int test_timing(void) { + enum { FRAMES = 30, CLASS = FRAMESED_CLASS_COUNT - 1 }; + float scores[FRAMESED_CLASS_COUNT * FRAMES] = {0}; + float *row = scores + CLASS * FRAMES; + row[1] = 0.75f; + for (size_t i = 2; i <= 8; ++i) + row[i] = 0.40f; + row[3] = 0.0f; /* A brief dropout must not split the event. */ + for (size_t i = 20; i < 23; ++i) + row[i] = 0.9f; /* Below the minimum duration. */ + for (size_t i = 0; i < FRAMES; ++i) + scores[10 * FRAMES + i] = 0.4f; /* Track without an event. */ + framesed_result result = {0}; + CHECK(extract_events(scores, FRAMES, 1.2, &result, 1) == 0); + CHECK(result.num_events == 1); + CHECK(result.events[0].class_group == CLASS); + CHECK(result.events[0].start == FRAMESED_FRAME_STEP); + CHECK(result.events[0].end == 9 * FRAMESED_FRAME_STEP); + CHECK(result.events[0].score == 0.75f); + CHECK(build_tracks(scores, FRAMES, 0.2, &result, 1) == 0); + CHECK(result.num_tracks == 2 && result.track_step == 0.2); + CHECK(result.tracks[0].class_group == 10 && result.tracks[1].class_group == CLASS); + CHECK(result.tracks[1].count == 6); + CHECK(result.tracks[1].values[0] == 0.75f); + CHECK(result.tracks[1].values[4] == 0.9f); + framesed_result_free(&result); + return 0; +} + +static int test_caption_suppression(void) { + enum { FRAMES = 30 }; + float scores[FRAMESED_GROUP_COUNT * FRAMES] = {0}; + for (size_t i = 0; i < FRAMES; ++i) { + scores[1 * FRAMES + i] = 0.75f; /* applause */ + scores[3 * FRAMES + i] = 0.75f; /* crowd */ + scores[58 * FRAMES + i] = 0.75f; /* impact */ + } + framesed_result result = {0}; + CHECK(build_tracks(scores, FRAMES, 0.2, &result, 0) == 0); + CHECK(result.num_tracks == 3); + CHECK(extract_events(scores, FRAMES, 1.2, &result, 0) == 0); + CHECK(result.num_events == 1 && result.events[0].class_group == 1); + framesed_result_free(&result); + return 0; +} + +int main(void) { + if (test_metadata() || test_probabilities() || test_all_classes() || + test_timing() || test_caption_suppression()) + return 1; + puts("framesed policy: metadata, 447 classes, timing, tracks, caption suppression passed"); + return 0; +} diff --git a/tools/framesed.c b/tools/framesed.c new file mode 100644 index 0000000..f5b519f --- /dev/null +++ b/tools/framesed.c @@ -0,0 +1,138 @@ +#include <errno.h> +#include <limits.h> +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "framesed.h" +#include "wav.h" + +static void usage(FILE *output, const char *program) { + fprintf(output, + "usage: %s [--all-classes] [--threads N] [--track-step SECONDS] " + "--weights FILE INPUT.wav\n" + " %s --list-classes\n", + program, program); +} + +static int parse_threads(const char *text, int *value) { + char *end = NULL; + errno = 0; + long parsed = strtol(text, &end, 10); + if (errno || !end || *end || parsed < 1 || parsed > INT_MAX) + return -1; + *value = (int)parsed; + return 0; +} + +static int parse_track_step(const char *text, double *value) { + char *end = NULL; + errno = 0; + double parsed = strtod(text, &end); + if (errno || !end || *end || !isfinite(parsed) || parsed < FRAMESED_FRAME_STEP) + return -1; + *value = parsed; + return 0; +} + +int main(int argc, char **argv) { + const char *input_path = NULL; + const char *weights_path = NULL; + int threads = 1; + double track_step = 0.0; + int all_classes = 0; + int list_classes = 0; + + for (int i = 1; i < argc; i++) { + const char *argument = argv[i]; + if (!strcmp(argument, "-h") || !strcmp(argument, "--help")) { + usage(stdout, argv[0]); + return EXIT_SUCCESS; + } + if (!strcmp(argument, "--all-classes")) { + all_classes = 1; + } else if (!strcmp(argument, "--list-classes")) { + list_classes = 1; + } else if (!strcmp(argument, "--threads")) { + if (++i >= argc || parse_threads(argv[i], &threads) != 0) { + fprintf(stderr, "%s: invalid thread count\n", argv[0]); + return 2; + } + } else if (!strcmp(argument, "--track-step")) { + if (++i >= argc || parse_track_step(argv[i], &track_step) != 0) { + fprintf(stderr, "%s: invalid track step\n", argv[0]); + return 2; + } + } else if (!strcmp(argument, "--weights")) { + if (++i >= argc) { + fprintf(stderr, "%s: --weights needs a file\n", argv[0]); + return 2; + } + weights_path = argv[i]; + } else if (argument[0] == '-') { + fprintf(stderr, "%s: unknown option %s\n", argv[0], argument); + return 2; + } else if (!input_path) { + input_path = argument; + } else { + fprintf(stderr, "%s: unexpected argument %s\n", argv[0], argument); + return 2; + } + } + if (list_classes) { + for (size_t i = 0; i < FRAMESED_CLASS_COUNT; ++i) + printf("%zu\t%s\t%s\n", i, framesed_class_mid(i), framesed_class_label(i)); + return EXIT_SUCCESS; + } + if (!input_path || !weights_path) { + usage(stderr, argv[0]); + return 2; + } + + wav_t wav; + const char *error = NULL; + if (wav_read_mono_s16(input_path, &wav, &error) != 0) { + fprintf(stderr, "%s: %s\n", input_path, error ? error : "could not read input"); + return EXIT_FAILURE; + } + if (wav.sample_rate != FRAMESED_SAMPLE_RATE) { + fprintf(stderr, "%s: expected %d Hz input\n", input_path, FRAMESED_SAMPLE_RATE); + wav_free(&wav); + return EXIT_FAILURE; + } + + framesed_t *engine = framesed_create(weights_path); + if (!engine) { + fprintf(stderr, "%s: could not load weights\n", weights_path); + wav_free(&wav); + return EXIT_FAILURE; + } + framesed_options options = { + .threads = threads, .track_step = track_step, .all_classes = all_classes, + }; + framesed_result result; + if (framesed_detect_s16(engine, wav.samples, wav.count, &options, &result) != 0) { + fprintf(stderr, "%s: sound-event detection failed\n", input_path); + framesed_destroy(engine); + wav_free(&wav); + return EXIT_FAILURE; + } + + for (size_t i = 0; i < result.num_events; i++) { + const framesed_event *event = &result.events[i]; + printf("event %.3f %.3f %.6f %d %s\n", event->start, event->end, event->score, + event->class_group, event->label); + } + for (size_t i = 0; i < result.num_tracks; i++) { + const framesed_track *track = &result.tracks[i]; + for (size_t j = 0; j < track->count; j++) + printf("track %.3f %.6f %d %s\n", j * result.track_step, track->values[j], + track->class_group, track->label); + } + + framesed_result_free(&result); + framesed_destroy(engine); + wav_free(&wav); + return EXIT_SUCCESS; +} @@ -0,0 +1,138 @@ +#include "wav.h" + +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +static int rd_u32(FILE *f, uint32_t *v) { + unsigned char b[4]; + if (fread(b, 1, 4, f) != 4) + return -1; + *v = (uint32_t)b[0] | (uint32_t)b[1] << 8 | (uint32_t)b[2] << 16 | (uint32_t)b[3] << 24; + return 0; +} + +static int rd_u16(FILE *f, uint16_t *v) { + unsigned char b[2]; + if (fread(b, 1, 2, f) != 2) + return -1; + *v = (uint16_t)((uint32_t)b[0] | (uint32_t)b[1] << 8); + return 0; +} + +#define FAIL(msg) \ + do { \ + if (err) \ + *err = (msg); \ + goto fail; \ + } while (0) + +int wav_read_mono_s16(const char *path, wav_t *out, const char **err) { + memset(out, 0, sizeof *out); + FILE *f = fopen(path, "rb"); + if (!f) { + if (err) + *err = "cannot open file"; + return -1; + } + + char id[4]; + uint32_t riff_size = 0; + if (fread(id, 1, 4, f) != 4 || memcmp(id, "RIFF", 4) != 0) + FAIL("not a RIFF file"); + if (rd_u32(f, &riff_size) != 0) + FAIL("truncated header"); + if (fread(id, 1, 4, f) != 4 || memcmp(id, "WAVE", 4) != 0) + FAIL("not a WAVE file"); + + uint16_t fmt = 0, channels = 0, bits = 0; + uint32_t rate = 0; + int have_fmt = 0; + + for (;;) { + uint32_t size; + if (fread(id, 1, 4, f) != 4 || rd_u32(f, &size) != 0) + break; /* end of file, or a truncated trailing chunk */ + + if (memcmp(id, "fmt ", 4) == 0) { + uint16_t block_align; + uint32_t byte_rate; + if (size < 16) + FAIL("short fmt chunk"); + if (rd_u16(f, &fmt) || rd_u16(f, &channels) || rd_u32(f, &rate) || + rd_u32(f, &byte_rate) || rd_u16(f, &block_align) || rd_u16(f, &bits)) + FAIL("truncated fmt chunk"); + if (fseek(f, (long)size - 16, SEEK_CUR) != 0) + FAIL("bad fmt chunk size"); + have_fmt = 1; + continue; + } + + if (memcmp(id, "data", 4) == 0) { + if (!have_fmt) + FAIL("data chunk before fmt chunk"); + /* Format 1 is PCM; 0xFFFE is extensible, whose subformat this + * reader does not inspect, so it is accepted only at 16 bits. */ + if ((fmt != 1 && fmt != 0xFFFE) || bits != 16) + FAIL("only 16-bit PCM is supported"); + if (channels == 0) + FAIL("zero channels"); + + const size_t frames = size / (2u * channels); + if (frames == 0) + FAIL("empty data chunk"); + int16_t *mono = malloc(frames * sizeof(int16_t)); + if (!mono) + FAIL("out of memory"); + + const size_t step = channels; + int16_t buf[1024]; + size_t got = 0; + const size_t batch_frames = sizeof buf / sizeof buf[0] / step; + if (batch_frames == 0) { + free(mono); + FAIL("too many channels"); + } + while (got < frames) { + const size_t take = (frames - got) < batch_frames ? frames - got : batch_frames; + const size_t want = take * step; + const size_t n = fread(buf, sizeof(int16_t), want, f); + if (n < step) + break; + for (size_t i = 0; i + step <= n; i += step) { + int64_t sum = 0; + for (size_t c = 0; c < step; ++c) + sum += buf[i + c]; + mono[got++] = (int16_t)lrint((double)sum / channels); + } + if (n < want) + break; + } + if (got == 0) { + free(mono); + FAIL("no samples read"); + } + out->samples = mono; + out->count = got; + out->sample_rate = (int)rate; + out->channels = (int)channels; + fclose(f); + return 0; + } + + /* Skip unknown chunks; RIFF chunks are word aligned. */ + if (fseek(f, (long)size + (size & 1u), SEEK_CUR) != 0) + break; + } + FAIL("no data chunk"); + +fail: + fclose(f); + return -1; +} + +void wav_free(wav_t *w) { + free(w->samples); + memset(w, 0, sizeof *w); +} @@ -0,0 +1,23 @@ +/* Minimal RIFF/WAVE reader for the command line tool: uncompressed 16-bit PCM + * only. Multichannel input is downmixed by averaging every channel. The + * library itself never touches files. */ + +#ifndef FRAMESED_WAV_H +#define FRAMESED_WAV_H + +#include <stddef.h> +#include <stdint.h> + +typedef struct { + int16_t *samples; /* mono, averaged across source channels */ + size_t count; + int sample_rate; + int channels; +} wav_t; + +/* Returns 0 on success. On failure returns non-zero and, when err is + * non-NULL, writes a short reason there. */ +int wav_read_mono_s16(const char *path, wav_t *out, const char **err); +void wav_free(wav_t *w); + +#endif |
