aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
Diffstat (limited to '')
-rw-r--r--.gitignore7
-rw-r--r--LICENSE20
-rw-r--r--Makefile125
-rw-r--r--README.md119
-rw-r--r--devtools/export_framesed_classes.py46
-rw-r--r--devtools/export_framesed_weights.py118
-rw-r--r--framesed/README.md19
-rw-r--r--framesed/classes.csv448
-rw-r--r--framesed/framesed.c1102
-rw-r--r--framesed/framesed.h100
-rw-r--r--framesed/framesed_classes.inc448
-rw-r--r--framesed/framesed_policy.inc85
-rw-r--r--model_blob.c111
-rw-r--r--model_blob.h36
-rw-r--r--testdata/framesed_reference.binbin0 -> 9852 bytes
-rw-r--r--tests/test_classes.c124
-rw-r--r--tests/test_cli.py76
-rw-r--r--tests/test_framesed.c280
-rw-r--r--tests/test_policy.c140
-rw-r--r--tools/framesed.c138
-rw-r--r--wav.c138
-rw-r--r--wav.h23
22 files changed, 3703 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..ec13c96
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,7 @@
+/build/
+/libframesed.a
+/models/*.bin
+/.venv/
+__pycache__/
+*.pyc
+.DS_Store
diff --git a/LICENSE b/LICENSE
new file mode 100644
index 0000000..425371a
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,20 @@
+MIT License
+
+Copyright (c) 2026 The framesed authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of
+this software and associated documentation files (the "Software"), to deal in
+the Software without restriction, including without limitation the rights to
+use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of
+the Software, and to permit persons to whom the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS
+FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR
+COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER
+IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
diff --git a/Makefile b/Makefile
new file mode 100644
index 0000000..d451190
--- /dev/null
+++ b/Makefile
@@ -0,0 +1,125 @@
+CC ?= cc
+AR ?= ar
+PYTHON ?= python3
+MODE ?= release
+ARCH_FLAGS ?=
+TARGET ?= libframesed.a
+OBJDIR := build/obj
+BUILD_CONFIG := build/.build-config
+
+CPPFLAGS += -I. -Iframesed
+WARNING_FLAGS ?= -Wall -Wextra -Werror=implicit-function-declaration
+ifeq ($(MODE),release)
+OPT_FLAGS ?= -O3 -DNDEBUG
+else ifeq ($(MODE),debug)
+OPT_FLAGS ?= -O0 -g3 -fno-omit-frame-pointer
+else ifeq ($(MODE),profile)
+OPT_FLAGS ?= -O2 -g -fno-omit-frame-pointer
+else
+$(error MODE must be release, debug, or profile)
+endif
+CFLAGS ?= $(OPT_FLAGS) $(WARNING_FLAGS) -std=gnu11 $(ARCH_FLAGS)
+ARFLAGS := rcs
+
+ifeq ($(shell uname -s),Darwin)
+AUDIO_BLAS ?= accelerate
+else
+AUDIO_BLAS ?= none
+endif
+ifeq ($(AUDIO_BLAS),accelerate)
+CPPFLAGS += -DFRAMESED_ACCELERATE -DACCELERATE_NEW_LAPACK
+AUDIO_LDLIBS := -framework Accelerate
+else ifeq ($(AUDIO_BLAS),openblas)
+CPPFLAGS += -DFRAMESED_OPENBLAS $(shell pkg-config --cflags openblas 2>/dev/null)
+AUDIO_LDLIBS := $(shell pkg-config --libs openblas 2>/dev/null || echo -lopenblas)
+else ifneq ($(AUDIO_BLAS),none)
+$(error AUDIO_BLAS must be none, accelerate, or openblas)
+endif
+BASE_LDLIBS = -lm -lpthread $(AUDIO_LDLIBS) $(LDLIBS)
+
+LIB_OBJECTS := $(OBJDIR)/framesed/framesed.o $(OBJDIR)/model_blob.o
+CLI_OBJECTS := $(OBJDIR)/tools/framesed.o $(OBJDIR)/wav.o
+TEST_OBJECTS := $(addprefix $(OBJDIR)/tests/,test_framesed.o test_classes.o test_policy.o)
+OBJECTS := $(LIB_OBJECTS) $(CLI_OBJECTS) $(TEST_OBJECTS)
+FRAMESED_TEST := build/tests/test_framesed
+CLASS_TEST := build/tests/test_classes
+POLICY_TEST := build/tests/test_policy
+FRAMESED_WEIGHTS ?= models/framesed_weights.bin
+FRAMESED_REFERENCE ?= testdata/framesed_reference.bin
+SANITIZER_CFLAGS ?= -O1 -g -Wall -Wextra -std=gnu11 -fsanitize=address,undefined \
+ -fno-omit-frame-pointer -fno-sanitize-recover=all
+SANITIZER_LDFLAGS ?= -fsanitize=address,undefined
+
+.PHONY: all framesed tools cli libframesed release debug profile test test-framesed check \
+ check-sanitize test-classes test-policy test-cli check-classes regenerate-classes clean FORCE
+
+all: $(TARGET) build/framesed
+framesed tools cli: build/framesed
+libframesed: $(TARGET)
+
+release debug profile:
+ $(MAKE) MODE=$@ all
+
+$(TARGET): $(LIB_OBJECTS)
+ $(RM) $@
+ $(AR) $(ARFLAGS) $@ $^
+
+build/framesed: $(CLI_OBJECTS) $(TARGET)
+ $(CC) $(CFLAGS) $(CLI_OBJECTS) $(TARGET) $(LDFLAGS) $(BASE_LDLIBS) -o $@
+
+$(FRAMESED_TEST) $(CLASS_TEST): build/tests/%: $(OBJDIR)/tests/%.o $(TARGET)
+ @mkdir -p $(dir $@)
+ $(CC) $(CFLAGS) $< $(TARGET) $(LDFLAGS) $(BASE_LDLIBS) -o $@
+
+# The policy test includes framesed.c to exercise private score-to-event contracts.
+$(POLICY_TEST): $(OBJDIR)/tests/test_policy.o $(OBJDIR)/model_blob.o
+ @mkdir -p $(dir $@)
+ $(CC) $(CFLAGS) $^ $(LDFLAGS) $(BASE_LDLIBS) -o $@
+
+test check: test-framesed test-classes test-policy test-cli check-classes
+
+test-framesed: $(FRAMESED_TEST) $(FRAMESED_WEIGHTS) $(FRAMESED_REFERENCE)
+ $(FRAMESED_TEST) "$(FRAMESED_WEIGHTS)" "$(FRAMESED_REFERENCE)"
+
+test-classes: $(CLASS_TEST) $(FRAMESED_WEIGHTS)
+ $(CLASS_TEST) "$(FRAMESED_WEIGHTS)"
+
+test-policy: $(POLICY_TEST)
+ $(POLICY_TEST)
+
+test-cli: build/framesed $(FRAMESED_WEIGHTS)
+ $(PYTHON) tests/test_cli.py "$(FRAMESED_WEIGHTS)"
+
+regenerate-classes:
+ $(PYTHON) devtools/export_framesed_classes.py framesed/classes.csv framesed/framesed_classes.inc
+
+check-classes:
+ $(PYTHON) devtools/export_framesed_classes.py framesed/classes.csv framesed/framesed_classes.inc --check
+
+check-sanitize:
+ $(MAKE) CFLAGS='$(SANITIZER_CFLAGS)' LDFLAGS='$(SANITIZER_LDFLAGS)' all test
+
+$(OBJECTS): $(BUILD_CONFIG)
+
+$(OBJDIR)/%.o: %.c
+ @mkdir -p $(dir $@)
+ $(CC) $(CPPFLAGS) $(CFLAGS) -MMD -MP -c $< -o $@
+
+# Rebuild on configuration changes, but leave unchanged builds incremental.
+$(BUILD_CONFIG): FORCE | build
+ @printf '%s\n' 'MODE=$(MODE)' 'AUDIO_BLAS=$(AUDIO_BLAS)' \
+ 'CC=$(CC)' 'AR=$(AR)' 'ARFLAGS=$(ARFLAGS)' \
+ 'CPPFLAGS=$(CPPFLAGS)' 'CFLAGS=$(CFLAGS)' \
+ 'LDFLAGS=$(LDFLAGS)' 'BASE_LDLIBS=$(BASE_LDLIBS)' > $@.new
+ @cmp -s $@.new $@ || cp $@.new $@
+ @rm -f $@.new
+
+FORCE:
+
+build:
+ mkdir -p $@
+
+clean:
+ rm -rf build $(TARGET)
+
+-include $(OBJECTS:.o=.d)
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..00ae9f4
--- /dev/null
+++ b/README.md
@@ -0,0 +1,119 @@
+# framesed
+
+Frame-level sound event detection using `PretrainedSED/frame_mn10`.
+
+## Build and run
+
+```sh
+make # libframesed.a and build/framesed
+make test # model parity, policies, threading, CLI, class metadata
+build/framesed --weights models/framesed_weights.bin audio.wav
+```
+
+A C11 compiler, Make, and POSIX threads are required. Tests and metadata generation also require Python 3 (standard library only). The public API is `framesed/framesed.h`.
+
+## CLI
+
+```text
+build/framesed [--all-classes] [--threads N] [--track-step SECONDS] --weights FILE INPUT.wav
+build/framesed --list-classes
+```
+
+Input is 16-bit PCM WAV at 16 kHz; multiple channels are averaged to mono.
+The CLI does not resample. `--threads` defaults to 1 and must be positive.
+`--track-step` enables score tracks and must be at least 0.04 seconds; the
+returned step rounds down to a multiple of 0.04 seconds.
+
+Default detection uses 59 caption groups from `framesed/framesed_policy.inc`.
+`--all-classes` detects each of the 447 model classes independently, without
+caption grouping or suppression. This includes speech and background/context
+classes, not just sound effects.
+
+```sh
+build/framesed --all-classes --weights models/framesed_weights.bin audio.wav
+build/framesed --list-classes # tab-separated index, AudioSet ID, label; no weights needed
+```
+
+Class-mode gates are uniform: start score 0.50, release score 0.35, minimum
+duration 0.24 s, allowed gap 0.32 s. They are not individually calibrated;
+recognition quality varies by class. Tracks are returned for classes reaching
+0.35, even when no event survives the duration gate.
+
+Output is one record per line, with times in seconds:
+
+```text
+event START END SCORE GROUP LABEL
+track TIME SCORE GROUP LABEL
+```
+
+`GROUP` is a caption-group index by default, or a model class index (0–446)
+with `--all-classes`. Labels may contain spaces and commas.
+
+## API
+
+Set `framesed_options.all_classes = 1` for individual-class detection; zero or
+NULL options select caption groups. `framesed_class_label(index)` and
+`framesed_class_mid(index)` return model metadata, or NULL for an invalid index.
+`framesed_group_label(index)` always refers to caption groups.
+
+With `keep_planes = 1`, class-mode results contain 447 probability rows in
+model order, at 0.04 s resolution. `framesed_infer_s16()` returns raw logits for one
+10-second window. Free detection results with `framesed_result_free()` and logits
+with `framesed_logits_free()`.
+
+The vocabulary is in [`framesed/classes.csv`](framesed/classes.csv). Its indices follow
+PretrainedSED's [`as_strong_train_classes`](https://github.com/fschmid56/PretrainedSED/blob/1aa47e482f7e89904cba2338999345025d8b4e36/data_util/audioset_classes.py);
+AudioSet IDs come from its `hf_dataset_gen/metadata` label tables. Class order
+must match the weights; editing labels does not teach the model new sounds.
+`make regenerate-classes` generates the compiled table; `make check-classes`
+checks that it matches the CSV. Neither command requires PyTorch.
+
+## Recreating the weights
+
+`models/framesed_weights.bin` is an inference export of PretrainedSED's
+[`frame_mn10_strong_1.pt`](https://github.com/fschmid56/PretrainedSED/releases/download/v0.0.1/frame_mn10_strong_1.pt),
+not a separately trained model. The exporter folds batch normalization into
+convolution weights, includes the Hann window and Kaldi mel filterbank, and
+keeps the 447-class strong head (the weak head is omitted).
+
+From the project root, in a Python environment with `torch`, `torchaudio`, and
+`numpy` installed (use matching torch/torchaudio versions):
+
+```sh
+python3 devtools/export_framesed_weights.py
+make test
+```
+
+The script downloads and verifies the upstream checkpoint before writing
+`models/framesed_weights.bin`. For offline use or a different output path:
+
+```sh
+python3 devtools/export_framesed_weights.py \
+ --checkpoint /path/to/frame_mn10_strong_1.pt --output /tmp/framesed_weights.bin
+```
+
+The exporter follows upstream revision `1aa47e482f7e89904cba2338999345025d8b4e36`.
+
+`devtools/export_framesed_classes.py` exports class metadata.
+
+## Build options
+
+- `make debug` / `make profile` / `make release`
+- `make AUDIO_BLAS=none all test` for portable CPU inference
+- `make AUDIO_BLAS=openblas all test` for OpenBLAS (requires OpenBLAS;
+ discovered through `pkg-config`)
+- `make check-sanitize` for AddressSanitizer and UndefinedBehaviorSanitizer
+- `make clean` removes build outputs, not weights or test data
+
+macOS defaults to Accelerate; other systems default to portable CPU.
+Configuration and header changes automatically trigger rebuilds.
+`FRAMESED_WEIGHTS` and `FRAMESED_REFERENCE` override test fixture paths.
+
+Link `libframesed.a` with the math and pthread libraries, plus Accelerate or OpenBLAS
+when enabled. The WAV reader is used only by the CLI, not the library.
+
+## License
+
+MIT License.
+
+The model weights are `PretrainedSED/frame_mn10` by Floarian Schmid; see the [model repository](https://github.com/fschmid56/PretrainedSED) for details.
diff --git a/devtools/export_framesed_classes.py b/devtools/export_framesed_classes.py
new file mode 100644
index 0000000..e02752a
--- /dev/null
+++ b/devtools/export_framesed_classes.py
@@ -0,0 +1,46 @@
+"""Generate C class metadata from the model-ordered index,mid,display_name CSV."""
+
+import argparse
+import csv
+import json
+from pathlib import Path
+
+
+def render(path: Path) -> str:
+ with path.open(newline="", encoding="utf-8") as source:
+ rows = list(csv.DictReader(source))
+ if len(rows) != 447:
+ raise ValueError("frame_mn10 requires exactly 447 classes")
+ mids = set()
+ labels = set()
+ lines = ["/* Model output order; regenerate with make regenerate-classes. */"]
+ for index, row in enumerate(rows):
+ mid, label = row["mid"], row["display_name"]
+ if int(row["index"]) != index:
+ raise ValueError("class indices must be contiguous and ordered from zero")
+ if not mid or not label or mid in mids or label in labels:
+ raise ValueError("class IDs and labels must be nonempty and unique")
+ if any(ord(c) < 32 for c in mid + label):
+ raise ValueError("class metadata must not contain control characters")
+ mids.add(mid)
+ labels.add(label)
+ lines.append(f" {{{json.dumps(mid)}, {json.dumps(label)}}},")
+ return "\n".join(lines) + "\n"
+
+
+def main() -> None:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("labels", type=Path)
+ parser.add_argument("output", type=Path)
+ parser.add_argument("--check", action="store_true")
+ args = parser.parse_args()
+ content = render(args.labels)
+ if args.check:
+ if args.output.read_text(encoding="utf-8") != content:
+ parser.exit(1, f"{args.output}: regenerate with make regenerate-classes\n")
+ else:
+ args.output.write_text(content, encoding="utf-8")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/devtools/export_framesed_weights.py b/devtools/export_framesed_weights.py
new file mode 100644
index 0000000..cdce3a9
--- /dev/null
+++ b/devtools/export_framesed_weights.py
@@ -0,0 +1,118 @@
+"""Export PretrainedSED frame_mn10_strong_1.pt to the C library's weight blob.
+
+Requires torch, torchaudio, and numpy; no upstream checkout is needed.
+Architecture and frontend follow PretrainedSED revision
+1aa47e482f7e89904cba2338999345025d8b4e36 (models/frame_mn and
+models/frame_passt/preprocess.py). Only the strong (frame-level) head is used.
+"""
+
+import argparse
+import hashlib
+from pathlib import Path
+import struct
+import tempfile
+import urllib.request
+
+URL = ("https://github.com/fschmid56/PretrainedSED/releases/download/"
+ "v0.0.1/frame_mn10_strong_1.pt")
+CHECKPOINT_SHA256 = "4a0fe320d5369987b772394c51881fb20a602967e6842f72f3e5c8181065ece7"
+
+
+def export_tensors(checkpoint):
+ import torch
+ import torch.nn.functional as F
+ from torchaudio.compliance.kaldi import get_mel_banks
+
+ # weights_only avoids arbitrary pickle code execution. Pin the checkpoint
+ # digest as well: other MobileNet variants are not supported by the C code.
+ digest = hashlib.sha256(checkpoint.read_bytes()).hexdigest()
+ if digest != CHECKPOINT_SHA256:
+ raise ValueError(f"unexpected checkpoint SHA-256: {digest}")
+ state = torch.load(checkpoint, map_location="cpu", weights_only=True)
+ tensors = {}
+
+ def add(name, tensor):
+ if not torch.isfinite(tensor).all():
+ raise ValueError(f"nonfinite tensor: {name}")
+ tensors[name] = tensor.detach().cpu().numpy().astype("<f4").tobytes()
+
+ def conv(name, prefix):
+ # Upstream BatchNorm2d uses eps=0.001. Convolutions have no bias.
+ weight = state[prefix + ".0.weight"]
+ bn = prefix + ".1."
+ scale = state[bn + "weight"] / torch.sqrt(state[bn + "running_var"] + 0.001)
+ add(name + "_w", weight * scale[:, None, None, None])
+ add(name + "_b", state[bn + "bias"] - state[bn + "running_mean"] * scale)
+
+ add("window", torch.hann_window(400, periodic=False))
+ # Evaluation fmax is sr/2 - fmax_aug_range/2 = 7000 Hz, not 8000.
+ mel, _ = get_mel_banks(128, 512, 16000, 0.0, 7000.0,
+ vtln_low=100.0, vtln_high=-500.0, vtln_warp_factor=1.0)
+ add("mel", F.pad(mel, (0, 1)))
+ base = "model.frame_mn.features."
+ conv("c0", base + "0")
+ se_blocks = {4, 5, 6, 11, 12, 13, 14, 15}
+ for index in range(1, 16):
+ prefix = base + f"{index}.block."
+ layer = 0
+ if index != 1:
+ conv(f"b{index}e", prefix + str(layer))
+ layer += 1
+ conv(f"b{index}d", prefix + str(layer))
+ layer += 1
+ if index in se_blocks:
+ se = prefix + f"{layer}.conc_se_layers.0."
+ for fc in (1, 2):
+ add(f"b{index}s{fc}w", state[se + f"fc{fc}.weight"])
+ add(f"b{index}s{fc}b", state[se + f"fc{fc}.bias"])
+ layer += 1
+ conv(f"b{index}p", prefix + str(layer))
+ conv("cf", base + "16")
+ if tuple(state["strong_head.weight"].shape) != (447, 960):
+ raise ValueError("expected a 447-class, 960-channel strong head")
+ add("out_w", state["strong_head.weight"])
+ add("out_b", state["strong_head.bias"])
+ return tensors
+
+
+def fnv1a(data):
+ digest = 0x811C9DC5
+ for byte in data:
+ digest = ((digest ^ byte) * 0x01000193) & 0xFFFFFFFF
+ return digest
+
+
+def pack_blob(tensors):
+ # model_blob.c: 16-byte header, 36-byte entries, 64-byte aligned payloads.
+ blob = bytearray(16 + 36 * len(tensors))
+ for index, (name, payload) in enumerate(tensors.items()):
+ blob.extend(bytes((-len(blob)) % 64))
+ offset = len(blob)
+ blob.extend(payload)
+ struct.pack_into("<24sIII", blob, 16 + 36 * index,
+ name.encode("ascii"), offset, len(payload), fnv1a(payload))
+ blob.extend(bytes((-len(blob)) % 64))
+ struct.pack_into("<4sIII", blob, 0, b"SDC1", 1, len(tensors), len(blob))
+ return blob
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--checkpoint", type=Path,
+ help="local upstream checkpoint; otherwise download to a temporary directory")
+ parser.add_argument("--output", type=Path, default=Path("models/framesed_weights.bin"))
+ args = parser.parse_args()
+ with tempfile.TemporaryDirectory(prefix="framesed-export-") as directory:
+ checkpoint = args.checkpoint
+ if checkpoint is None:
+ checkpoint = Path(directory) / "frame_mn10_strong_1.pt"
+ print(f"Downloading {URL}")
+ urllib.request.urlretrieve(URL, checkpoint)
+ blob = pack_blob(export_tensors(checkpoint))
+ args.output.parent.mkdir(parents=True, exist_ok=True)
+ args.output.write_bytes(blob)
+ print(f"Wrote {args.output}: {len(blob)} bytes, SHA-256 {hashlib.sha256(blob).hexdigest()}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/framesed/README.md b/framesed/README.md
new file mode 100644
index 0000000..00d60ca
--- /dev/null
+++ b/framesed/README.md
@@ -0,0 +1,19 @@
+# FRAMESED sound events
+
+This directory contains the standalone 16 kHz sound-event model and the
+deterministic policy that turns class scores into 59 caption groups. Detection
+can instead return all 447 model classes independently. Both modes support
+unsuppressed score tracks and per-frame probability planes.
+
+Build and run from the project root:
+
+```sh
+make build/framesed
+build/framesed --weights models/framesed_weights.bin audio.wav
+build/framesed --all-classes --weights models/framesed_weights.bin audio.wav
+build/framesed --list-classes
+```
+
+The public API is `framesed.h`. Input is mono signed 16-bit PCM at 16 kHz; callers
+own decoding and resampling. See the [project reference](../README.md) for
+class-mode thresholds, metadata, and output formats.
diff --git a/framesed/classes.csv b/framesed/classes.csv
new file mode 100644
index 0000000..c518714
--- /dev/null
+++ b/framesed/classes.csv
@@ -0,0 +1,448 @@
+index,mid,display_name
+0,/m/07q2z82,"Accelerating, revving, vroom"
+1,/m/0gvgw0,Air brake
+2,/m/025wky1,Air conditioning
+3,/m/05x_td,"Air horn, truck horn"
+4,/m/0k5j,Aircraft
+5,/m/014yck,Aircraft engine
+6,/m/07pp_mv,Alarm
+7,/m/046dlr,Alarm clock
+8,/t/dd00144,Alert
+9,/m/012n7d,Ambulance (siren)
+10,/m/0jbk,Animal
+11,/m/028ght,Applause
+12,/m/0dl83,Arrow
+13,/m/0_1c,Artillery fire
+14,/t/dd00142,Audio logo
+15,/m/0261r1,Babbling
+16,/t/dd00002,"Baby cry, infant cry"
+17,/t/dd00001,Baby laughter
+18,/m/093_4n,Background noise
+19,/m/07pws3f,Bang
+20,/m/05tny_,Bark
+21,/m/018w8,Basketball bounce
+22,/m/01j2bj,Bathroom sounds
+23,/m/03dnzn,Bathtub (filling or washing)
+24,/m/04gy_2,Battle cry
+25,/m/01h3n,"Bee, wasp, etc."
+26,/m/02fs_r,"Beep, bleep"
+27,/m/0395lw,Bell
+28,/m/07q4ntr,Bellow
+29,/m/07sq110,Belly laugh
+30,/m/0gy1t2s,Bicycle bell
+31,/m/0199g,"Bicycle, tricycle"
+32,/m/015p6,Bird
+33,/m/05_wcq,"Bird flight, flapping wings"
+34,/m/020bb7,"Bird vocalization, bird call, bird song"
+35,/m/07pdhp0,Biting
+36,/m/07q0h5t,Bleat
+37,/m/02pjr4,"Blender, food processor"
+38,/m/019jd,"Boat, Water vehicle"
+39,/m/0dv3j,Boiling
+40,/t/dd00121,Boing
+41,/m/04v5dt,Booing
+42,/m/07qqyl4,Boom
+43,/m/07pqn27,Bouncing
+44,/m/07rc7d9,Bow-wow
+45,/m/07pc8lb,Breaking
+46,/m/0lyf6,Breathing
+47,/t/dd00138,Brief tone
+48,/m/03q5_w,"Burping, eructation"
+49,/m/07qsvvw,"Burst, pop"
+50,/m/01bjv,Bus
+51,/m/01z47d,Busy signal
+52,/m/07pjwq1,Buzz
+53,/m/030rvx,Buzzer
+54,/m/07szfh9,Cacophony
+55,/m/0dv5r,Camera
+56,/m/01z5f,"Canidae, wild dogs, wolves"
+57,/m/073cg4,Cap gun
+58,/m/0k4j,Car
+59,/m/02mfyn,Car alarm
+60,/t/dd00134,Car passing by
+61,/m/08dckq,"Carbon monoxide detector, CO detector"
+62,/m/018p4k,Cart
+63,/m/024dl,Cash register
+64,/m/01yrx,Cat
+65,/m/07r81j2,Caterwaul
+66,/m/01xq0k1,"Cattle, bovinae"
+67,/m/07r5c2p,Caw
+68,/m/01sb50,"Cellphone buzz, vibrating alert"
+69,/m/0269r2s,Chain
+70,/m/01j4z9,Chainsaw
+71,/m/0150b9,Change ringing (campanology)
+72,/t/dd00123,"Channel, environment and background"
+73,/m/02bk07,Chant
+74,/m/053hz1,Cheering
+75,/m/03cczk,"Chewing, mastication"
+76,/m/09b5t,"Chicken, rooster"
+77,/t/dd00005,Child singing
+78,/m/0ytgt,"Child speech, kid speaking"
+79,/t/dd00013,Children playing
+80,/t/dd00135,Children shouting
+81,/m/0f8s22,Chime
+82,/m/02021,Chipmunk
+83,/m/0hdsk,Chirp tone
+84,/m/07pggtn,"Chirp, tweet"
+85,/m/0l14jd,Choir
+86,/m/07pczhz,Chop
+87,/m/07pn_8q,Chopping (food)
+88,/m/07m2kt,Chorus effect
+89,/m/07rgt08,"Chuckle, chortle"
+90,/m/03w41f,Church bell
+91,/m/0dgbq,Civil defense siren
+92,/m/07rv4dm,Clang
+93,/m/0l15bq,Clapping
+94,/m/07qmpdm,Clatter
+95,/m/07rwm0c,Clickety-clack
+96,/m/07qc9xj,Clicking
+97,/m/07rv9rh,Clip-clop
+98,/m/01x3z,Clock
+99,/m/07st89h,Cluck
+100,/t/dd00108,Clunk
+101,/m/0242l,Coin (dropping)
+102,/m/01m2v,Computer keyboard
+103,/m/01h8n0,Conversation
+104,/m/07r_25d,Coo
+105,/m/01b_21,Cough
+106,/m/0239kh,Cowbell
+107,/m/07qs1cx,Crack
+108,/m/07pzfmf,Crackle
+109,/m/07qh7jl,Creak
+110,/m/09xqv,Cricket
+111,/m/07st88b,Croak
+112,/m/0hgq8df,Crockery breaking and smashing
+113,/m/04s8yn,Crow
+114,/m/03qtwd,Crowd
+115,/m/07qn5dc,"Crowing, cock-a-doodle-doo"
+116,/t/dd00112,"Crumpling, crinkling"
+117,/m/07s12q4,Crunch
+118,/m/07plct2,Crushing
+119,/m/0463cq4,"Crying, sobbing"
+120,/m/0642b4,Cupboard open or close
+121,/m/023pjk,"Cutlery, silverware"
+122,/t/dd00110,Deformable shell
+123,/m/08j51y,"Dental drill, dentist's drill"
+124,/m/015jpf,Dial tone
+125,/m/0160x5,Digestive
+126,/m/07phxs1,Ding
+127,/m/07r67yg,Ding-dong
+128,/m/04brg2,"Dishes, pots, and pans"
+129,/m/0g12c5,Distortion
+130,/m/0bt9lr,Dog
+131,/m/068hy,"Domestic animals, pets"
+132,/t/dd00147,"Dong, bong"
+133,/m/0ffhf,"Donkey, ass"
+134,/m/02dgv,Door
+135,/m/03wwcy,Doorbell
+136,/m/0fqfqc,Drawer open or close
+137,/m/01d380,Drill
+138,/m/07r5v4s,Drip
+139,/m/04179zz,Duck call (hunting tool)
+140,/m/09ddx,"Ducks, geese, waterfowl"
+141,/m/01jnbd,Echo
+142,/m/02rr_,Effects unit
+143,/m/0c1tlg,"Electric rotor drone, quadcopter"
+144,/m/02g901,"Electric shaver, electric razor"
+145,/m/04fgwm,Electric toothbrush
+146,/m/0b_fwt,Electronic tuner
+147,/m/03j1ly,Emergency vehicle
+148,/m/02mk9,Engine
+149,/m/01h82_,Engine knocking
+150,/t/dd00130,Engine starting
+151,/m/06_y0by,Environmental noise
+152,/m/098_xr,Error signal
+153,/m/07pxg6y,Eruption
+154,/m/014zdl,Explosion
+155,/m/02_nn,Fart
+156,/t/dd00004,Female singing
+157,/m/02zsn,"Female speech, woman speaking"
+158,/m/02p01q,Filing (rasp)
+159,/m/07p7b8y,Fill (with liquid)
+160,/m/025_jnm,Finger snapping
+161,/m/02_41,Fire
+162,/m/0c3f7m,Fire alarm
+163,/m/012ndj,"Fire engine, fire truck (siren)"
+164,/g/122z_qxw,Firecracker
+165,/m/0g6b5,Fireworks
+166,/m/0cmf2,"Fixed-wing aircraft, airplane"
+167,/t/dd00118,Fizz
+168,/m/07pyf11,Flap
+169,/m/0h2mp,"Fly, housefly"
+170,/m/04fq5q,Foghorn
+171,/m/025rv6n,Fowl
+172,/m/09ld4,Frog
+173,/m/0dxrf,Frying (food)
+174,/m/02z32qm,Fusillade
+175,/m/0939n_,Gargling
+176,/m/07s0dtb,Gasp
+177,/m/0l7xg,Gears
+178,/t/dd00099,Generic impact sounds
+179,/m/07r660_,Giggle
+180,/m/039jq,Glass
+181,/m/07q7njn,"Glass chink, clink"
+182,/m/07rn7sz,Glass shatter
+183,/m/03fwl,Goat
+184,/m/07svc2k,Gobble
+185,/m/07pt6mm,Grind
+186,/m/07s2xch,Groan
+187,/m/0ghcn6,Growling
+188,/m/07r4k75,Grunt
+189,/m/01dwxx,"Gull, seagull"
+190,/m/032s66,"Gunshot, gunfire"
+191,/m/07swgks,"Gurgling, bubbling"
+192,/t/dd00088,Gush
+193,/m/03wvsk,Hair dryer
+194,/m/03l9g,Hammer
+195,/m/0k65p,Hands
+196,/m/01jg02,"Heart sounds, heartbeat"
+197,/t/dd00067,Heavy engine (low frequency)
+198,/m/09ct_,Helicopter
+199,/m/02p3nc,Hiccup
+200,/m/07rjwbb,Hiss
+201,/m/07qwf61,Honk
+202,/m/07r_80w,Hoot
+203,/m/03k3r,Horse
+204,/m/07qf0zm,Howl
+205,/m/07q8f3b,Howl (wind)
+206,/m/07qfr4h,"Hubbub, speech noise, speech babble"
+207,/m/07rcgpl,Hum
+208,/t/dd00012,Human group actions
+209,/m/0bpl036,Human locomotion
+210,/m/0dgw9r,Human sounds
+211,/m/09l8g,Human voice
+212,/m/02fxyj,Humming
+213,/m/03cl9h,"Ice cream truck, ice cream van"
+214,/m/07pb8fc,Idling
+215,/m/03vt0,Insect
+216,/t/dd00126,"Inside, large room or hall"
+217,/t/dd00127,"Inside, public space"
+218,/t/dd00125,"Inside, small room"
+219,/m/03p19w,Jackhammer
+220,/m/04229,Jet engine
+221,/m/027m70_,Jingle bell
+222,/m/07qfgpx,"Jingle, tinkle"
+223,/g/11b630rrvh,Kettle whistle
+224,/m/056r_1,Keypress tone
+225,/m/03v3yw,Keys jangling
+226,/m/0d4wf,Kitchen and dining room sounds
+227,/m/04ctx,Knife
+228,/m/07r4wb8,Knock
+229,/m/01j3sz,Laughter
+230,/m/01yg9g,Lawn mower
+231,/t/dd00065,Light engine (high frequency)
+232,/m/04k94,Liquid
+233,/m/0ch8v,"Livestock, farm animals, working animals"
+234,/m/02ll1_,Lock
+235,/m/04zjc,Machine gun
+236,/m/06xkwv,Mains hum
+237,/t/dd00003,Male singing
+238,/m/05zppz,"Male speech, man speaking"
+239,/m/01c194,Mantra
+240,/m/06cyt0,Mechanical bell
+241,/m/02x984l,Mechanical fan
+242,/t/dd00077,Mechanisms
+243,/t/dd00066,Medium engine (mid frequency)
+244,/m/07qrkrw,Meow
+245,/m/0hg7b,Microphone
+246,/m/0fx9l,Microwave oven
+247,/m/07rpkh9,Moo
+248,/m/09f96,Mosquito
+249,/m/012f08,Motor vehicle (road)
+250,/m/02rlv9,"Motorboat, speedboat"
+251,/m/04_sv,Motorcycle
+252,/m/04rmv,Mouse
+253,/m/04rlf,Music
+254,/m/02qldy,"Narration, monologue"
+255,/m/07q5rw0,"Neigh, whinny"
+256,/m/096m7z,Noise
+257,/t/dd00061,Non-motorized land vehicle
+258,/m/05kq4,Ocean
+259,/t/dd00018,Oink
+260,/t/dd00122,Other sourceless
+261,/t/dd00128,"Outside, urban or manmade"
+262,/m/09d5_,Owl
+263,/m/05mxj0q,"Packing tape, duct tape"
+264,/m/07pyy8b,Pant
+265,/t/dd00141,Pant (dog)
+266,/m/0641k,Paper rustling
+267,/m/07r4gkf,Patter
+268,/m/068zj,Pig
+269,/m/0h0rv,"Pigeon, dove"
+270,/m/07qwdck,Ping
+271,/m/07qyrcz,Plop
+272,/m/04qvtq,Police car (siren)
+273,/m/07prgkl,Pour
+274,/m/0md09,"Power saw, circular saw, table saw"
+275,/m/0_ksk,Power tool
+276,/m/04gxbd,"Power windows, electric windows"
+277,/m/01m4t,Printer
+278,/m/02l6bg,"Propeller, airscrew"
+279,/m/07q34h3,Puff
+280,/m/05zc1,Pulleys
+281,/m/07pt_g0,Pulse
+282,/m/07ptzwd,Pump (liquid)
+283,/m/02yds9,Purr
+284,/m/07qdb04,Quack
+285,/m/0ltv,"Race car, auto racing"
+286,/m/06bz3,Radio
+287,/m/06d_3,Rail transport
+288,/m/01g50p,"Railroad car, train wagon"
+289,/m/06mb1,Rain
+290,/t/dd00038,Rain on surface
+291,/m/07r10fb,Raindrop
+292,/m/06bxc,Rapping
+293,/m/02bm9n,"Ratchet, pawl"
+294,/m/07qn4z3,Rattle
+295,/m/040b_t,Refrigerator
+296,/m/09hlz4,Respiratory sounds
+297,/m/01b9nn,Reverberation
+298,/m/02rhddq,Reversing beeps
+299,/m/0bcdqg,"Ringing tone, ringback tone"
+300,/m/01hnzm,Ringtone
+301,/m/04cvmfc,Roar
+302,/m/0cdnk,"Roaring cats (lions, tigers)"
+303,/m/06hps,"Rodents, rats, mice"
+304,/m/07s8j8t,Roll
+305,/m/056ks2,"Rowboat, canoe, kayak"
+306,/m/07pdjhy,Rub
+307,/m/07phhsh,Rumble
+308,/m/06h7j,Run
+309,/m/07qwyj0,Rustle
+310,/m/0hsrw,"Sailboat, sailing ship"
+311,/m/023vsd,Sanding
+312,/m/01b82r,Sawing
+313,/m/01lsmm,Scissors
+314,/m/07qv4k0,Scrape
+315,/m/07qb_dv,Scratch
+316,/m/03qc9zr,Screaming
+317,/m/07q8k13,Screech
+318,/m/0llzx,Sewing machine
+319,/m/07bgp,Sheep
+320,/m/06q74,Ship
+321,/m/07p6fty,Shout
+322,/m/02f9f_,Shower
+323,/m/07qv_x_,Shuffle
+324,/m/05rj2,Shuffling cards
+325,/m/07plz5l,Sigh
+326,/m/01v_m0,Sine wave
+327,/m/015lz1,Singing
+328,/m/07bjf,Single-lens reflex camera
+329,/m/0130jx,Sink (filling or washing)
+330,/m/03kmc9,Siren
+331,/m/07p9k1k,Sizzle
+332,/m/06_fw,Skateboard
+333,/m/07rjzl8,Slam
+334,/m/07ryjzk,"Slap, smack"
+335,/m/02y_763,Sliding door
+336,/m/07p6mqd,Slosh
+337,/m/07pqmly,"Slurp, drinking straw"
+338,/m/07pjjrj,"Smash, crash"
+339,/m/01y3hg,"Smoke detector, smoke alarm"
+340,/m/078jl,Snake
+341,/m/07pc8l3,Snap
+342,/m/01hsr_,Sneeze
+343,/m/07s04w4,Snicker
+344,/m/07ppn3j,Sniff
+345,/m/01d3sd,Snoring
+346,/m/07q0yl5,Snort
+347,/t/dd00139,Snort (horse)
+348,/m/0790c,Sonar
+349,/m/0193bn,Sonic boom
+350,/m/0c1dj,Sound effect
+351,/t/dd00091,Sound equipment
+352,/m/07bm98,Sound reproduction
+353,/m/09x0r,Speech
+354,/m/0brhx,Speech synthesizer
+355,/m/07rrlb6,"Splash, splatter"
+356,/m/07pl1bw,Splinter
+357,/m/07qlf79,Spray
+358,/m/07sx8x_,Squawk
+359,/m/07q6cd_,Squeak
+360,/m/07s02z0,Squeal
+361,/m/07qlwh6,Squish
+362,/m/01lynh,Stairs
+363,/m/07rgkc5,Static
+364,/m/06wzb,Steam
+365,/m/06hck5,Steam whistle
+366,/m/07ptfmf,Stir
+367,/m/01g90h,Stomach rumble
+368,/m/07sk0jz,"Stomp, stamp"
+369,/m/0j6m2,"Stream, river"
+370,/m/0195fx,"Subway, metro, underground"
+371,/t/dd00109,Surface contact
+372,/m/07s13rg,Sweeping
+373,/t/dd00006,Synthetic singing
+374,/m/07qcpgn,Tap
+375,/m/0fw86,Tap dance
+376,/m/09d1b1,Tape hiss
+377,/m/07qcx4z,Tearing
+378,/m/07cx4,Telephone
+379,/m/07pp8cl,Telephone bell ringing
+380,/m/02c8p,"Telephone dialing, DTMF"
+381,/m/07c52,Television
+382,/m/0dl9sf8,Throat clearing
+383,/m/07qnq_y,"Thump, thud"
+384,/m/0ngt1,Thunder
+385,/m/0jb2l,Thunderstorm
+386,/m/07rrh0c,Thunk
+387,/m/07qjznt,Tick
+388,/m/07qjznl,Tick-tock
+389,/m/0h9mv,"Tire squeal, skidding"
+390,/m/01jt3m,Toilet flush
+391,/m/07k1x,Tools
+392,/m/012xff,Toothbrush
+393,/m/0btp2,"Traffic noise, roadway noise"
+394,/m/07jdr,Train
+395,/m/0284vy3,Train horn
+396,/t/dd00048,Train wheels squealing
+397,/m/04zmvq,Train whistle
+398,/m/07pqc89,"Trickle, dribble"
+399,/m/07r04,Truck
+400,/m/07n_g,Tuning fork
+401,/m/01rd7k,Turkey
+402,/m/0c2wf,Typewriter
+403,/m/0316dw,Typing
+404,/t/dd00143,Unknown sound
+405,/m/0d31p,Vacuum cleaner
+406,/m/07yv9,Vehicle
+407,/m/0912c9,"Vehicle horn, car horn, honking, toot"
+408,/m/0zmy2j9,"Velcro, hook and loop fastener"
+409,/m/0bzvm2,Video game sound
+410,/m/07qw_06,"Wail, moan"
+411,/m/07pbtc8,"Walk, footsteps"
+412,/m/0174k2,Washing machine
+413,/m/0838f,Water
+414,/m/02jz0l,"Water tap, faucet"
+415,/m/0j2kx,Waterfall
+416,/m/034srq,"Waves, surf"
+417,/m/07rdhzs,"Whack, thwack"
+418,/m/032n05,Whale vocalization
+419,/m/07mzm6,Wheeze
+420,/m/07qz6j3,Whimper
+421,/t/dd00136,Whimper (dog)
+422,/m/07rbp7_,Whip
+423,/m/07s34ls,Whir
+424,/m/02rtxlg,Whispering
+425,/m/0l156k,Whistle
+426,/m/01w250,Whistling
+427,/m/0chx_,"White noise, pink noise"
+428,/m/07rwj3x,Whoop
+429,/m/07rqsjt,"Whoosh, swoosh, swish"
+430,/m/01280g,Wild animals
+431,/m/0fjy1,Wildfire
+432,/m/03m9d0z,Wind
+433,/m/026fgl,Wind chime
+434,/t/dd00092,Wind noise (microphone)
+435,/m/02417f,"Windscreen wiper, windshield wiper"
+436,/m/07pk7mg,Wobble
+437,/m/079vc8,Wolf-whistling
+438,/m/083vt,Wood
+439,/m/081rb,Writing
+440,/m/01hhp3,Yak
+441,/m/01j423,Yawn
+442,/m/07sr1lc,Yell
+443,/m/07r_k2n,Yip
+444,/m/01swy6,Yodeling
+445,/m/07p78v5,Zing
+446,/m/01s0vc,Zipper (clothing)
diff --git a/framesed/framesed.c b/framesed/framesed.c
new file mode 100644
index 0000000..36eb87f
--- /dev/null
+++ b/framesed/framesed.c
@@ -0,0 +1,1102 @@
+#include "framesed.h"
+
+#include <math.h>
+#include <pthread.h>
+#include <stdatomic.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#if defined(FRAMESED_ACCELERATE)
+# include <Accelerate/Accelerate.h>
+#elif defined(FRAMESED_OPENBLAS)
+# include <cblas.h>
+#endif
+
+#include "model_blob.h"
+
+#define MEL_BINS 128
+#define FFT_SIZE 512
+#define FFT_BINS 257
+#define FFT_BITS 9
+#define WINDOW_SIZE 400
+#define WINDOW_OFFSET 56
+#define HOP_SIZE 160
+#define MODEL_SECONDS 10u
+#define MODEL_SAMPLES (MODEL_SECONDS * FRAMESED_SAMPLE_RATE)
+#define MODEL_FRAMES 250u
+#define FINAL_CHANNELS 960
+#define FRAMESED_FRAME_SAMPLES 640u
+#define INFERENCE_HOP_SECONDS 5u
+#define INFERENCE_HOP_SAMPLES (INFERENCE_HOP_SECONDS * FRAMESED_SAMPLE_RATE)
+#define INFERENCE_HOP_FRAMES 125u
+#define INTERNAL_KEEP_START 63u
+#define INTERNAL_KEEP_END (INTERNAL_KEEP_START + INFERENCE_HOP_FRAMES)
+
+typedef enum {
+ ACT_NONE,
+ ACT_RELU,
+ ACT_HSWISH,
+} activation;
+
+typedef struct {
+ const float *weight;
+ const float *bias;
+ int input_channels;
+ int output_channels;
+ int kernel;
+ int stride_y;
+ int stride_x;
+} conv;
+
+typedef struct {
+ int input_channels;
+ int expanded_channels;
+ int output_channels;
+ int kernel;
+ int stride_y;
+ int stride_x;
+ int squeeze_channels;
+ activation act;
+ int residual;
+ conv expand;
+ conv depthwise;
+ conv project;
+ const float *se1_weight;
+ const float *se1_bias;
+ const float *se2_weight;
+ const float *se2_bias;
+} block;
+
+typedef struct {
+ float *data;
+ int channels;
+ int height;
+ int width;
+} tensor;
+
+typedef struct {
+ int start;
+ int count;
+ const float *weights;
+} mel_band;
+
+struct framesed {
+ framesed_model_blob blob;
+ void *mapping;
+ size_t mapping_size;
+ const float *window;
+ const float *mel;
+ mel_band bands[MEL_BINS];
+ conv initial;
+ block blocks[15];
+ conv final;
+ conv output;
+ float twiddle_real[FFT_SIZE / 2];
+ float twiddle_imag[FFT_SIZE / 2];
+ uint16_t reverse[FFT_SIZE];
+ char backend[24];
+};
+
+typedef struct {
+ int input;
+ int expanded;
+ int output;
+ int kernel;
+ int stride_y;
+ int stride_x;
+ int squeeze;
+ activation act;
+} block_config;
+
+typedef struct {
+ const char *label;
+ float threshold;
+ float release;
+ float minimum;
+ float gap;
+ uint16_t class_offset;
+ uint16_t class_count;
+ int16_t parent;
+ uint64_t suppresses;
+} framesed_group_definition;
+
+#include "framesed_policy.inc"
+
+_Static_assert(FRAMESED_GROUP_COUNT <= 64, "FRAMESED suppression mask is too small");
+_Static_assert(FRAMESED_GROUP_COUNT <= FRAMESED_CLASS_COUNT, "FRAMESED track presence array is too small");
+
+static const struct {
+ const char *mid;
+ const char *label;
+} framesed_classes[] = {
+#include "framesed_classes.inc"
+};
+
+_Static_assert(sizeof(framesed_classes) / sizeof(framesed_classes[0]) == FRAMESED_CLASS_COUNT,
+ "FRAMESED labels must match the model output count");
+
+static size_t detection_group_count(int all_classes) {
+ return all_classes ? FRAMESED_CLASS_COUNT : FRAMESED_GROUP_COUNT;
+}
+
+static framesed_group_definition detection_group(size_t group, int all_classes) {
+ if (!all_classes)
+ return framesed_groups[group];
+ return (framesed_group_definition){
+ .label = framesed_classes[group].label,
+ .threshold = 0.50f,
+ .release = 0.35f,
+ .minimum = 0.24f,
+ .gap = 0.32f,
+ .class_offset = (uint16_t)group,
+ .class_count = 1,
+ .parent = -1,
+ };
+}
+
+static const block_config block_configs[15] = {
+ {16, 16, 16, 3, 1, 1, 0, ACT_RELU}, {16, 64, 24, 3, 2, 2, 0, ACT_RELU},
+ {24, 72, 24, 3, 2, 1, 0, ACT_RELU}, {24, 72, 40, 5, 1, 1, 24, ACT_RELU},
+ {40, 120, 40, 5, 1, 1, 32, ACT_RELU}, {40, 120, 40, 5, 2, 1, 32, ACT_RELU},
+ {40, 240, 80, 3, 1, 1, 0, ACT_HSWISH}, {80, 200, 80, 3, 2, 1, 0, ACT_HSWISH},
+ {80, 184, 80, 3, 1, 1, 0, ACT_HSWISH}, {80, 184, 80, 3, 1, 1, 0, ACT_HSWISH},
+ {80, 480, 112, 3, 2, 1, 120, ACT_HSWISH}, {112, 672, 112, 3, 1, 1, 168, ACT_HSWISH},
+ {112, 672, 160, 5, 1, 1, 168, ACT_HSWISH}, {160, 960, 160, 5, 1, 1, 240, ACT_HSWISH},
+ {160, 960, 160, 5, 2, 1, 240, ACT_HSWISH},
+};
+
+static int checked_product(size_t a, size_t b, size_t *result) {
+ if (a && b > SIZE_MAX / a)
+ return -1;
+ *result = a * b;
+ return 0;
+}
+
+static int tensor_allocate(tensor *value, int channels, int height, int width) {
+ if (channels <= 0 || height <= 0 || width <= 0)
+ return -1;
+ size_t count;
+ if (checked_product((size_t)channels, (size_t)height, &count) != 0 ||
+ checked_product(count, (size_t)width, &count) != 0 || count > SIZE_MAX / sizeof(float))
+ return -1;
+ void *data = NULL;
+ if (posix_memalign(&data, 64, count * sizeof(float)) != 0)
+ return -1;
+ value->data = data;
+ value->channels = channels;
+ value->height = height;
+ value->width = width;
+ return 0;
+}
+
+static void tensor_release(tensor *value) {
+ free(value->data);
+ memset(value, 0, sizeof(*value));
+}
+
+static float activate(float value, activation act) {
+ if (act == ACT_RELU)
+ return value > 0.0f ? value : 0.0f;
+ if (act == ACT_HSWISH) {
+ float gate = value + 3.0f;
+ if (gate < 0.0f)
+ gate = 0.0f;
+ if (gate > 6.0f)
+ gate = 6.0f;
+ return value * gate * (1.0f / 6.0f);
+ }
+ return value;
+}
+
+static void matrix_multiply(
+ const float *weight, const float *input, float *output, int rows, int columns, int inner) {
+#if defined(FRAMESED_ACCELERATE) || defined(FRAMESED_OPENBLAS)
+ cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, columns, inner, 1.0f, weight, inner,
+ input, columns, 0.0f, output, columns);
+#else
+ for (int row = 0; row < rows; ++row) {
+ const float *w = weight + (size_t)row * inner;
+ float *out = output + (size_t)row * columns;
+ memset(out, 0, (size_t)columns * sizeof(float));
+ for (int k = 0; k < inner; ++k) {
+ const float scale = w[k];
+ const float *in = input + (size_t)k * columns;
+ for (int column = 0; column < columns; ++column)
+ out[column] += scale * in[column];
+ }
+ }
+#endif
+}
+
+static int pointwise(const conv *layer, const tensor *input, activation act, tensor *output) {
+ if (layer->kernel != 1 || layer->stride_y != 1 || layer->stride_x != 1 ||
+ layer->input_channels != input->channels ||
+ tensor_allocate(output, layer->output_channels, input->height, input->width) != 0)
+ return -1;
+ const int spatial = input->height * input->width;
+ matrix_multiply(layer->weight, input->data, output->data, layer->output_channels, spatial,
+ layer->input_channels);
+ for (int channel = 0; channel < output->channels; ++channel) {
+ float *row = output->data + (size_t)channel * spatial;
+ const float bias = layer->bias[channel];
+ for (int index = 0; index < spatial; ++index)
+ row[index] = activate(row[index] + bias, act);
+ }
+ return 0;
+}
+
+static int
+spatial_conv(const conv *layer, const tensor *input, activation act, int depthwise, tensor *output) {
+ const int padding = layer->kernel / 2;
+ const int out_height = (input->height + 2 * padding - layer->kernel) / layer->stride_y + 1;
+ const int out_width = (input->width + 2 * padding - layer->kernel) / layer->stride_x + 1;
+ if (layer->input_channels != input->channels ||
+ (depthwise && layer->output_channels != input->channels) ||
+ tensor_allocate(output, layer->output_channels, out_height, out_width) != 0)
+ return -1;
+
+ for (int output_channel = 0; output_channel < output->channels; ++output_channel) {
+ const int first_input = depthwise ? output_channel : 0;
+ const int last_input = depthwise ? output_channel + 1 : input->channels;
+ const float *kernel =
+ layer->weight +
+ (size_t)output_channel * (depthwise ? layer->kernel * layer->kernel
+ : input->channels * layer->kernel * layer->kernel);
+ float *destination = output->data + (size_t)output_channel * out_height * out_width;
+ for (int y = 0; y < out_height; ++y) {
+ for (int x = 0; x < out_width; ++x) {
+ float sum = layer->bias[output_channel];
+ for (int input_channel = first_input; input_channel < last_input; ++input_channel) {
+ const float *source =
+ input->data + (size_t)input_channel * input->height * input->width;
+ const int kernel_channel =
+ depthwise ? 0 : input_channel * layer->kernel * layer->kernel;
+ for (int ky = 0; ky < layer->kernel; ++ky) {
+ const int sy = y * layer->stride_y + ky - padding;
+ if (sy < 0 || sy >= input->height)
+ continue;
+ for (int kx = 0; kx < layer->kernel; ++kx) {
+ const int sx = x * layer->stride_x + kx - padding;
+ if (sx < 0 || sx >= input->width)
+ continue;
+ sum += source[(size_t)sy * input->width + sx] *
+ kernel[kernel_channel + ky * layer->kernel + kx];
+ }
+ }
+ }
+ destination[(size_t)y * out_width + x] = activate(sum, act);
+ }
+ }
+ }
+ return 0;
+}
+
+static int squeeze_excitation(const block *layer, tensor *input) {
+ if (!layer->squeeze_channels)
+ return 0;
+ const int spatial = input->height * input->width;
+ float *mean = malloc((size_t)input->channels * sizeof(float));
+ float *hidden = malloc((size_t)layer->squeeze_channels * sizeof(float));
+ float *scale = malloc((size_t)input->channels * sizeof(float));
+ if (!mean || !hidden || !scale) {
+ free(mean);
+ free(hidden);
+ free(scale);
+ return -1;
+ }
+ for (int channel = 0; channel < input->channels; ++channel) {
+ const float *row = input->data + (size_t)channel * spatial;
+ float sum = 0.0f;
+ for (int index = 0; index < spatial; ++index)
+ sum += row[index];
+ mean[channel] = sum / spatial;
+ }
+ for (int row = 0; row < layer->squeeze_channels; ++row) {
+ const float *weight = layer->se1_weight + (size_t)row * input->channels;
+ float sum = layer->se1_bias[row];
+ for (int column = 0; column < input->channels; ++column)
+ sum += weight[column] * mean[column];
+ hidden[row] = sum > 0.0f ? sum : 0.0f;
+ }
+ for (int row = 0; row < input->channels; ++row) {
+ const float *weight = layer->se2_weight + (size_t)row * layer->squeeze_channels;
+ float sum = layer->se2_bias[row];
+ for (int column = 0; column < layer->squeeze_channels; ++column)
+ sum += weight[column] * hidden[column];
+ scale[row] = 1.0f / (1.0f + expf(-sum));
+ }
+ for (int channel = 0; channel < input->channels; ++channel) {
+ float *row = input->data + (size_t)channel * spatial;
+ for (int index = 0; index < spatial; ++index)
+ row[index] *= scale[channel];
+ }
+ free(mean);
+ free(hidden);
+ free(scale);
+ return 0;
+}
+
+static int run_block(const block *layer, tensor *input, tensor *output) {
+ tensor expanded = {0};
+ tensor depth = {0};
+ tensor projected = {0};
+ const tensor *depth_input = input;
+ if (layer->expanded_channels != layer->input_channels) {
+ if (pointwise(&layer->expand, input, layer->act, &expanded) != 0)
+ goto failed;
+ depth_input = &expanded;
+ }
+ if (spatial_conv(&layer->depthwise, depth_input, layer->act, 1, &depth) != 0 ||
+ squeeze_excitation(layer, &depth) != 0 ||
+ pointwise(&layer->project, &depth, ACT_NONE, &projected) != 0)
+ goto failed;
+
+ if (layer->residual) {
+ const size_t count = (size_t)input->channels * input->height * input->width;
+ for (size_t index = 0; index < count; ++index)
+ projected.data[index] += input->data[index];
+ }
+ tensor_release(&expanded);
+ tensor_release(&depth);
+ *output = projected;
+ return 0;
+
+failed:
+ tensor_release(&expanded);
+ tensor_release(&depth);
+ tensor_release(&projected);
+ return -1;
+}
+
+static void fft(framesed_t *engine, float *real, float *imag) {
+ for (int length = 2; length <= FFT_SIZE; length <<= 1) {
+ const int half = length / 2;
+ const int stride = FFT_SIZE / length;
+ for (int base = 0; base < FFT_SIZE; base += length) {
+ for (int j = 0; j < half; ++j) {
+ const float wr = engine->twiddle_real[j * stride];
+ const float wi = engine->twiddle_imag[j * stride];
+ const int a = base + j;
+ const int b = a + half;
+ const float tr = real[b] * wr - imag[b] * wi;
+ const float ti = real[b] * wi + imag[b] * wr;
+ real[b] = real[a] - tr;
+ imag[b] = imag[a] - ti;
+ real[a] += tr;
+ imag[a] += ti;
+ }
+ }
+ }
+}
+
+static size_t reflect_index(int64_t index, size_t length) {
+ if (length <= 1)
+ return 0;
+ const int64_t period = (int64_t)(2 * length - 2);
+ index %= period;
+ if (index < 0)
+ index += period;
+ if ((size_t)index >= length)
+ index = period - index;
+ return (size_t)index;
+}
+
+static float preemphasized(const int16_t *samples, size_t index) {
+ return ((float)samples[index + 1] - 0.97f * (float)samples[index]) * (1.0f / 32768.0f);
+}
+
+static int frontend(framesed_t *engine, const int16_t *samples, size_t num_samples, tensor *features) {
+ if (num_samples < 2)
+ return -1;
+ const size_t emphasized = num_samples - 1;
+ const size_t frames_size = 1 + emphasized / HOP_SIZE;
+ if (frames_size > INT32_MAX || tensor_allocate(features, 1, MEL_BINS, (int)frames_size) != 0)
+ return -1;
+ const int frames = (int)frames_size;
+
+ float real[FFT_SIZE];
+ float imag[FFT_SIZE];
+ float power[FFT_BINS];
+ for (int frame = 0; frame < frames; ++frame) {
+ memset(real, 0, sizeof(real));
+ memset(imag, 0, sizeof(imag));
+ const int64_t center = (int64_t)frame * HOP_SIZE;
+ for (int index = 0; index < WINDOW_SIZE; ++index) {
+ const size_t source = reflect_index(center - WINDOW_SIZE / 2 + index, emphasized);
+ const int fft_index = WINDOW_OFFSET + index;
+ real[engine->reverse[fft_index]] = preemphasized(samples, source) * engine->window[index];
+ }
+ fft(engine, real, imag);
+ for (int bin = 0; bin < FFT_BINS; ++bin)
+ power[bin] = real[bin] * real[bin] + imag[bin] * imag[bin];
+
+ for (int mel = 0; mel < MEL_BINS; ++mel) {
+ const mel_band *band = &engine->bands[mel];
+ float sum = 0.0f;
+ for (int bin = 0; bin < band->count; ++bin)
+ sum += power[band->start + bin] * band->weights[bin];
+ if (sum < 1e-7f)
+ sum = 1e-7f;
+ features->data[(size_t)mel * frames + frame] = (logf(sum) + 4.5f) * 0.2f;
+ }
+ }
+ return 0;
+}
+
+static int load_conv(framesed_model_blob *blob,
+ conv *layer,
+ const char *name,
+ int input,
+ int output,
+ int kernel,
+ int stride_y,
+ int stride_x) {
+ char weight_name[24];
+ char bias_name[24];
+ if (snprintf(weight_name, sizeof(weight_name), "%s_w", name) >= (int)sizeof(weight_name) ||
+ snprintf(bias_name, sizeof(bias_name), "%s_b", name) >= (int)sizeof(bias_name))
+ return -1;
+ layer->weight = framesed_model_blob_take(blob, weight_name, (size_t)output * input * kernel * kernel);
+ layer->bias = framesed_model_blob_take(blob, bias_name, (size_t)output);
+ layer->input_channels = input;
+ layer->output_channels = output;
+ layer->kernel = kernel;
+ layer->stride_y = stride_y;
+ layer->stride_x = stride_x;
+ return layer->weight && layer->bias ? 0 : -1;
+}
+
+static int load_block(framesed_model_blob *blob, block *layer, int index, const block_config *config) {
+ char name[24];
+ memset(layer, 0, sizeof(*layer));
+ layer->input_channels = config->input;
+ layer->expanded_channels = config->expanded;
+ layer->output_channels = config->output;
+ layer->kernel = config->kernel;
+ layer->stride_y = config->stride_y;
+ layer->stride_x = config->stride_x;
+ layer->squeeze_channels = config->squeeze;
+ layer->act = config->act;
+ layer->residual =
+ config->stride_y == 1 && config->stride_x == 1 && config->input == config->output;
+
+ if (config->expanded != config->input) {
+ snprintf(name, sizeof(name), "b%de", index);
+ if (load_conv(blob, &layer->expand, name, config->input, config->expanded, 1, 1, 1) != 0)
+ return -1;
+ }
+ snprintf(name, sizeof(name), "b%dd", index);
+ if (load_conv(blob, &layer->depthwise, name, 1, config->expanded, config->kernel,
+ config->stride_y, config->stride_x) != 0)
+ return -1;
+ layer->depthwise.input_channels = config->expanded;
+
+ if (config->squeeze) {
+ snprintf(name, sizeof(name), "b%ds1w", index);
+ layer->se1_weight =
+ framesed_model_blob_take(blob, name, (size_t)config->squeeze * config->expanded);
+ snprintf(name, sizeof(name), "b%ds1b", index);
+ layer->se1_bias = framesed_model_blob_take(blob, name, config->squeeze);
+ snprintf(name, sizeof(name), "b%ds2w", index);
+ layer->se2_weight =
+ framesed_model_blob_take(blob, name, (size_t)config->expanded * config->squeeze);
+ snprintf(name, sizeof(name), "b%ds2b", index);
+ layer->se2_bias = framesed_model_blob_take(blob, name, config->expanded);
+ if (!layer->se1_weight || !layer->se1_bias || !layer->se2_weight || !layer->se2_bias)
+ return -1;
+ }
+ snprintf(name, sizeof(name), "b%dp", index);
+ return load_conv(blob, &layer->project, name, config->expanded, config->output, 1, 1, 1);
+}
+
+framesed_t *framesed_create(const char *weights_path) {
+ if (!weights_path)
+ return NULL;
+ framesed_t *engine = calloc(1, sizeof(*engine));
+ if (!engine)
+ return NULL;
+ if (framesed_model_blob_map(weights_path, 1, &engine->blob, &engine->mapping, &engine->mapping_size) !=
+ 0)
+ goto failed;
+
+ engine->window = framesed_model_blob_take(&engine->blob, "window", WINDOW_SIZE);
+ engine->mel = framesed_model_blob_take(&engine->blob, "mel", MEL_BINS * FFT_BINS);
+ if (!engine->window || !engine->mel ||
+ load_conv(&engine->blob, &engine->initial, "c0", 1, 16, 3, 2, 2) != 0)
+ goto failed;
+ for (int index = 0; index < 15; ++index)
+ if (load_block(&engine->blob, &engine->blocks[index], index + 1, &block_configs[index]) != 0)
+ goto failed;
+ if (load_conv(&engine->blob, &engine->final, "cf", 160, FINAL_CHANNELS, 1, 1, 1) != 0 ||
+ load_conv(&engine->blob, &engine->output, "out", FINAL_CHANNELS, FRAMESED_CLASS_COUNT, 1, 1, 1) !=
+ 0 ||
+ engine->blob.failed)
+ goto failed;
+
+ for (int mel = 0; mel < MEL_BINS; ++mel) {
+ const float *row = engine->mel + (size_t)mel * FFT_BINS;
+ int start = 0;
+ while (start < FFT_BINS && row[start] == 0.0f)
+ ++start;
+ int end = FFT_BINS;
+ while (end > start && row[end - 1] == 0.0f)
+ --end;
+ engine->bands[mel].start = start;
+ engine->bands[mel].count = end - start;
+ engine->bands[mel].weights = row + start;
+ }
+ for (int index = 0; index < FFT_SIZE / 2; ++index) {
+ const double angle = -2.0 * M_PI * index / FFT_SIZE;
+ engine->twiddle_real[index] = (float)cos(angle);
+ engine->twiddle_imag[index] = (float)sin(angle);
+ }
+ for (int index = 0; index < FFT_SIZE; ++index) {
+ unsigned reversed = 0;
+ for (int bit = 0; bit < FFT_BITS; ++bit)
+ reversed |= ((unsigned)(index >> bit) & 1u) << (FFT_BITS - 1 - bit);
+ engine->reverse[index] = (uint16_t)reversed;
+ }
+#if defined(FRAMESED_ACCELERATE)
+ strcpy(engine->backend, "accelerate");
+#elif defined(FRAMESED_OPENBLAS)
+ strcpy(engine->backend, "openblas");
+#else
+ strcpy(engine->backend, "scalar");
+#endif
+ return engine;
+
+failed:
+ framesed_destroy(engine);
+ return NULL;
+}
+
+void framesed_destroy(framesed_t *engine) {
+ if (!engine)
+ return;
+ framesed_model_blob_unmap(engine->mapping, engine->mapping_size);
+ free(engine);
+}
+
+int framesed_infer_s16(framesed_t *engine, const int16_t *samples, size_t num_samples, framesed_logits *output) {
+ if (!engine || !samples || !output || num_samples != MODEL_SAMPLES)
+ return -1;
+ memset(output, 0, sizeof(*output));
+ tensor current = {0};
+ tensor next = {0};
+ if (frontend(engine, samples, num_samples, &current) != 0 ||
+ spatial_conv(&engine->initial, &current, ACT_HSWISH, 0, &next) != 0)
+ goto failed;
+ tensor_release(&current);
+ current = next;
+ memset(&next, 0, sizeof(next));
+
+ for (int index = 0; index < 15; ++index) {
+ if (run_block(&engine->blocks[index], &current, &next) != 0)
+ goto failed;
+ tensor_release(&current);
+ current = next;
+ memset(&next, 0, sizeof(next));
+ }
+ if (pointwise(&engine->final, &current, ACT_HSWISH, &next) != 0)
+ goto failed;
+ tensor_release(&current);
+ current = next;
+ memset(&next, 0, sizeof(next));
+
+ if (current.height != 1)
+ goto failed;
+ const int frames = current.width;
+ size_t logit_count;
+ if (checked_product(FRAMESED_CLASS_COUNT, (size_t)frames, &logit_count) != 0 ||
+ logit_count > SIZE_MAX / sizeof(float))
+ goto failed;
+ float *logits = malloc(logit_count * sizeof(float));
+ if (!logits)
+ goto failed;
+ matrix_multiply(engine->output.weight, current.data, logits, FRAMESED_CLASS_COUNT, frames,
+ FINAL_CHANNELS);
+ for (int class_index = 0; class_index < FRAMESED_CLASS_COUNT; ++class_index) {
+ float *row = logits + (size_t)class_index * frames;
+ const float bias = engine->output.bias[class_index];
+ for (int frame = 0; frame < frames; ++frame)
+ row[frame] += bias;
+ }
+ tensor_release(&current);
+ output->logits = logits;
+ output->frames = (size_t)frames;
+ return 0;
+
+failed:
+ tensor_release(&current);
+ tensor_release(&next);
+ return -1;
+}
+
+void framesed_logits_free(framesed_logits *output) {
+ if (!output)
+ return;
+ free(output->logits);
+ memset(output, 0, sizeof(*output));
+}
+
+static size_t output_frames(size_t samples) {
+ return samples ? (samples + FRAMESED_FRAME_SAMPLES - 1) / FRAMESED_FRAME_SAMPLES : 0;
+}
+
+static float sigmoid(float value) {
+ if (value >= 0.0f)
+ return 1.0f / (1.0f + expf(-value));
+ const float exponential = expf(value);
+ return exponential / (1.0f + exponential);
+}
+
+static int group_probabilities(const framesed_logits *logits, float *scores, int all_classes) {
+ if (!logits || !logits->logits || !scores || !logits->frames)
+ return -1;
+ const size_t frames = logits->frames;
+ for (size_t group = 0; group < detection_group_count(all_classes); ++group) {
+ float *destination = scores + group * frames;
+ for (size_t frame = 0; frame < frames; ++frame)
+ destination[frame] = 0.0f;
+ const framesed_group_definition definition = detection_group(group, all_classes);
+ for (uint16_t member = 0; member < definition.class_count; ++member) {
+ const size_t class_index =
+ all_classes ? group : framesed_group_classes[definition.class_offset + member];
+ const float *source = logits->logits + (size_t)class_index * frames;
+ for (size_t frame = 0; frame < frames; ++frame) {
+ const float probability = sigmoid(source[frame]);
+ if (probability > destination[frame])
+ destination[frame] = probability;
+ }
+ }
+ }
+ return 0;
+}
+
+static int append_event(framesed_result *result,
+ size_t *capacity,
+ size_t group,
+ const char *label,
+ size_t start_frame,
+ size_t end_frame,
+ float score,
+ double duration) {
+ if (result->num_events == *capacity) {
+ size_t grown = *capacity ? *capacity * 2 : 32;
+ if (grown < *capacity || grown > SIZE_MAX / sizeof(*result->events))
+ return -1;
+ framesed_event *events = realloc(result->events, grown * sizeof(*result->events));
+ if (!events)
+ return -1;
+ result->events = events;
+ *capacity = grown;
+ }
+ double start = start_frame * FRAMESED_FRAME_STEP;
+ double end = end_frame * FRAMESED_FRAME_STEP;
+ if (end > duration)
+ end = duration;
+ result->events[result->num_events++] = (framesed_event){
+ .start = start,
+ .end = end,
+ .score = score,
+ .label = label,
+ .class_group = (int)group,
+ };
+ return 0;
+}
+
+static int compare_event(const void *left, const void *right) {
+ const framesed_event *a = left;
+ const framesed_event *b = right;
+ if (a->start != b->start)
+ return (a->start > b->start) - (a->start < b->start);
+ if (a->end != b->end)
+ return (a->end > b->end) - (a->end < b->end);
+ if (a->score != b->score)
+ return (a->score < b->score) - (a->score > b->score);
+ return a->class_group - b->class_group;
+}
+
+/* Caption suppression uses a 64-bit group mask; model classes never enter
+ * this path because their outputs are independent, including parent classes. */
+static void suppress_groups(float *scores, size_t frames) {
+ for (size_t child = 0; child < FRAMESED_GROUP_COUNT; ++child) {
+ const float *child_scores = scores + child * frames;
+ uint64_t suppressed = framesed_groups[child].suppresses;
+ int parent = framesed_groups[child].parent;
+ while (parent >= 0) {
+ suppressed |= UINT64_C(1) << parent;
+ float *parent_scores = scores + (size_t)parent * frames;
+ for (size_t frame = 0; frame < frames; ++frame)
+ if (child_scores[frame] >= framesed_groups[child].release)
+ parent_scores[frame] = 0.0f;
+ parent = framesed_groups[parent].parent;
+ }
+ for (size_t group = 0; group < FRAMESED_GROUP_COUNT; ++group) {
+ if (!(suppressed & (UINT64_C(1) << group)))
+ continue;
+ float *row = scores + group * frames;
+ for (size_t frame = 0; frame < frames; ++frame)
+ if (child_scores[frame] >= framesed_groups[child].release)
+ row[frame] = 0.0f;
+ }
+ }
+}
+
+static int extract_events(float *scores, size_t frames, double duration, framesed_result *result,
+ int all_classes) {
+ if (!all_classes)
+ suppress_groups(scores, frames);
+ size_t capacity = 0;
+ for (size_t group = 0; group < detection_group_count(all_classes); ++group) {
+ const framesed_group_definition definition = detection_group(group, all_classes);
+ const float *row = scores + group * frames;
+ size_t frame = 0;
+ while (frame < frames) {
+ while (frame < frames && row[frame] < definition.threshold)
+ ++frame;
+ if (frame == frames)
+ break;
+ const size_t start = frame;
+ size_t last_active = frame;
+ float maximum = row[frame];
+ ++frame;
+ while (frame < frames) {
+ if (row[frame] >= definition.release) {
+ last_active = frame;
+ if (row[frame] > maximum)
+ maximum = row[frame];
+ } else if ((frame - last_active) * FRAMESED_FRAME_STEP > definition.gap) {
+ break;
+ }
+ ++frame;
+ }
+ const size_t end = last_active + 1;
+ if ((end - start) * FRAMESED_FRAME_STEP >= definition.minimum &&
+ append_event(result, &capacity, group, definition.label, start, end,
+ maximum, duration) != 0)
+ return -1;
+ }
+ }
+ if (result->num_events > 1)
+ qsort(result->events, result->num_events, sizeof(*result->events), compare_event);
+ return 0;
+}
+
+/* Number of model windows the recording is covered by. Windows hop by
+ * INFERENCE_HOP_SAMPLES, and the last one is the first whose remaining
+ * samples fit inside a single window. */
+static size_t chunk_count_for(size_t num_samples) {
+ if (num_samples <= MODEL_SAMPLES)
+ return 1;
+ const size_t beyond = num_samples - MODEL_SAMPLES;
+ return 1 + (beyond + INFERENCE_HOP_SAMPLES - 1) / INFERENCE_HOP_SAMPLES;
+}
+
+/* Runs one model window and writes its kept frames into the recording-wide
+ * score plane. The destination range follows from the window index alone, so
+ * windows never write the same frame and the result does not depend on the
+ * order in which they run. */
+static int run_chunk(framesed_t *engine,
+ const int16_t *samples,
+ size_t num_samples,
+ size_t chunk_index,
+ size_t frames,
+ float *scores,
+ int16_t *padded,
+ int all_classes) {
+ const size_t chunk_start = chunk_index * INFERENCE_HOP_SAMPLES;
+ if (chunk_start >= num_samples)
+ return -1;
+ size_t chunk_count = num_samples - chunk_start;
+ if (chunk_count > MODEL_SAMPLES)
+ chunk_count = MODEL_SAMPLES;
+ const int first = chunk_index == 0;
+ const int last = chunk_start + chunk_count == num_samples;
+ const int16_t *chunk = samples + chunk_start;
+ if (chunk_count < MODEL_SAMPLES) {
+ memcpy(padded, chunk, chunk_count * sizeof(*padded));
+ memset(padded + chunk_count, 0, (MODEL_SAMPLES - chunk_count) * sizeof(*padded));
+ chunk = padded;
+ }
+
+ framesed_logits logits = {0};
+ if (framesed_infer_s16(engine, chunk, MODEL_SAMPLES, &logits) != 0 || logits.frames != MODEL_FRAMES) {
+ framesed_logits_free(&logits);
+ return -1;
+ }
+ size_t local_count;
+ if (checked_product(detection_group_count(all_classes), logits.frames, &local_count) != 0 ||
+ local_count > SIZE_MAX / sizeof(float)) {
+ framesed_logits_free(&logits);
+ return -1;
+ }
+ float *local_scores = malloc(local_count * sizeof(float));
+ if (!local_scores || group_probabilities(&logits, local_scores, all_classes) != 0) {
+ free(local_scores);
+ framesed_logits_free(&logits);
+ return -1;
+ }
+ const size_t actual_frames = output_frames(chunk_count);
+ const size_t keep_start = first ? 0 : INTERNAL_KEEP_START;
+ const size_t keep_end = last ? actual_frames : INTERNAL_KEEP_END;
+ if (keep_start > keep_end || keep_end > logits.frames) {
+ free(local_scores);
+ framesed_logits_free(&logits);
+ return -1;
+ }
+ const size_t global_offset = chunk_start / FRAMESED_FRAME_SAMPLES;
+ for (size_t group = 0; group < detection_group_count(all_classes); ++group) {
+ const float *source = local_scores + group * logits.frames;
+ float *destination = scores + group * frames;
+ for (size_t frame = keep_start; frame < keep_end; ++frame) {
+ const size_t global_frame = global_offset + frame;
+ if (global_frame < frames)
+ destination[global_frame] = source[frame];
+ }
+ }
+ free(local_scores);
+ framesed_logits_free(&logits);
+ return 0;
+}
+
+typedef struct {
+ framesed_t *engine;
+ const int16_t *samples;
+ size_t num_samples;
+ size_t num_chunks;
+ size_t frames;
+ float *scores;
+ int all_classes;
+ atomic_size_t next_chunk;
+ atomic_int failed;
+ void (*progress)(void *context, size_t done, size_t total);
+ void *progress_context;
+ pthread_mutex_t progress_lock;
+ size_t finished; /* guarded by progress_lock */
+} detect_pool;
+
+/* Claims windows until the recording is covered. The engine holds only
+ * read-only weights and every window allocates its own working tensors, so
+ * one engine serves every worker. */
+static void *detect_worker(void *opaque) {
+ detect_pool *pool = opaque;
+ int16_t *padded = calloc(MODEL_SAMPLES, sizeof(*padded));
+ if (!padded) {
+ atomic_store(&pool->failed, 1);
+ return NULL;
+ }
+ for (;;) {
+ const size_t index = atomic_fetch_add(&pool->next_chunk, 1);
+ if (index >= pool->num_chunks || atomic_load(&pool->failed))
+ break;
+ if (run_chunk(pool->engine, pool->samples, pool->num_samples, index, pool->frames,
+ pool->scores, padded, pool->all_classes) != 0) {
+ atomic_store(&pool->failed, 1);
+ break;
+ }
+ if (pool->progress) {
+ pthread_mutex_lock(&pool->progress_lock);
+ pool->progress(pool->progress_context, ++pool->finished, pool->num_chunks);
+ pthread_mutex_unlock(&pool->progress_lock);
+ }
+ }
+ free(padded);
+ return NULL;
+}
+
+/* Reduces a group's frame scores to one value per output step by taking the
+ * maximum, which is how an event's reported score is defined and keeps a
+ * short peak from disappearing into a longer step. */
+static int build_tracks(const float *scores, size_t frames, double track_step, framesed_result *result,
+ int all_classes) {
+ /* The step snaps down to whole model frames, so the returned resolution is
+ * never coarser than the caller asked for. */
+ const size_t per_step = (size_t)(track_step / FRAMESED_FRAME_STEP);
+ const size_t stride = per_step ? per_step : 1;
+ const size_t count = (frames + stride - 1) / stride;
+ if (!count)
+ return 0;
+
+ /* A group earns a track when its score reaches the release threshold
+ * anywhere in the recording. That covers every group that produced an
+ * event and also the sustained near misses, which are the cases where an
+ * event boundary alone does not say how present a sound is. */
+ unsigned char present[FRAMESED_CLASS_COUNT] = {0};
+ const size_t num_groups = detection_group_count(all_classes);
+ size_t num_tracks = 0;
+ for (size_t group = 0; group < num_groups; ++group) {
+ const framesed_group_definition definition = detection_group(group, all_classes);
+ const float *row = scores + group * frames;
+ for (size_t frame = 0; frame < frames; ++frame)
+ if (row[frame] >= definition.release) {
+ present[group] = 1;
+ ++num_tracks;
+ break;
+ }
+ }
+ if (!num_tracks)
+ return 0;
+
+ framesed_track *tracks = calloc(num_tracks, sizeof(*tracks));
+ if (!tracks)
+ return -1;
+ size_t index = 0;
+ for (size_t group = 0; group < num_groups; ++group) {
+ if (!present[group])
+ continue;
+ float *values = malloc(count * sizeof(*values));
+ if (!values) {
+ for (size_t i = 0; i < index; ++i)
+ free(tracks[i].values);
+ free(tracks);
+ return -1;
+ }
+ const float *row = scores + group * frames;
+ for (size_t step = 0; step < count; ++step) {
+ const size_t start = step * stride;
+ size_t end = start + stride;
+ if (end > frames)
+ end = frames;
+ float maximum = row[start];
+ for (size_t frame = start + 1; frame < end; ++frame)
+ if (row[frame] > maximum)
+ maximum = row[frame];
+ values[step] = maximum;
+ }
+ tracks[index++] = (framesed_track){
+ .label = detection_group(group, all_classes).label,
+ .class_group = (int)group,
+ .values = values,
+ .count = count,
+ };
+ }
+ result->tracks = tracks;
+ result->num_tracks = num_tracks;
+ result->track_step = (double)stride * FRAMESED_FRAME_STEP;
+ return 0;
+}
+
+const char *framesed_group_label(size_t group) {
+ return group < FRAMESED_GROUP_COUNT ? framesed_groups[group].label : NULL;
+}
+
+size_t framesed_group_count(void) { return FRAMESED_GROUP_COUNT; }
+
+const char *framesed_class_label(size_t class_index) {
+ return class_index < FRAMESED_CLASS_COUNT ? framesed_classes[class_index].label : NULL;
+}
+
+const char *framesed_class_mid(size_t class_index) {
+ return class_index < FRAMESED_CLASS_COUNT ? framesed_classes[class_index].mid : NULL;
+}
+
+int framesed_detect_s16(framesed_t *engine,
+ const int16_t *samples,
+ size_t num_samples,
+ const framesed_options *options,
+ framesed_result *output) {
+ const framesed_options defaults = {.threads = 1};
+ if (!options)
+ options = &defaults;
+ const int threads = options->threads > 0 ? options->threads : 1;
+ const double track_step = options->track_step;
+ if (!engine || (!samples && num_samples) || !output)
+ return -1;
+ memset(output, 0, sizeof(*output));
+ if (num_samples < 2)
+ return 0;
+
+ const size_t frames = output_frames(num_samples);
+ const int all_classes = options->all_classes;
+ const size_t num_groups = detection_group_count(all_classes);
+ size_t score_count;
+ if (checked_product(num_groups, frames, &score_count) != 0 ||
+ score_count > SIZE_MAX / sizeof(float))
+ return -1;
+ float *scores = calloc(score_count, sizeof(float));
+ if (!scores)
+ return -1;
+
+ detect_pool pool = {
+ .engine = engine,
+ .samples = samples,
+ .num_samples = num_samples,
+ .num_chunks = chunk_count_for(num_samples),
+ .frames = frames,
+ .scores = scores,
+ .all_classes = all_classes,
+ };
+ atomic_init(&pool.next_chunk, 0);
+ atomic_init(&pool.failed, 0);
+ pool.progress = options->progress;
+ pool.progress_context = options->progress_context;
+ pthread_mutex_init(&pool.progress_lock, NULL);
+
+ size_t workers = (size_t)threads;
+ if (workers > pool.num_chunks)
+ workers = pool.num_chunks;
+ pthread_t *pool_threads = workers > 1 ? calloc(workers - 1, sizeof(*pool_threads)) : NULL;
+ size_t started = 0;
+ if (pool_threads)
+ for (size_t i = 0; i + 1 < workers; ++i)
+ if (pthread_create(&pool_threads[i], NULL, detect_worker, &pool) == 0)
+ ++started;
+ detect_worker(&pool);
+ for (size_t i = 0; i < started; ++i)
+ pthread_join(pool_threads[i], NULL);
+ free(pool_threads);
+ pthread_mutex_destroy(&pool.progress_lock);
+
+ if (atomic_load(&pool.failed)) {
+ free(scores);
+ return -1;
+ }
+
+ /* Caption suppression must not modify the evidence returned as tracks or
+ * planes. All-class extraction leaves these scores unsuppressed. */
+ float *unsuppressed = NULL;
+ if (track_step > 0.0 || options->keep_planes) {
+ unsuppressed = malloc(score_count * sizeof(float));
+ if (!unsuppressed) {
+ free(scores);
+ return -1;
+ }
+ memcpy(unsuppressed, scores, score_count * sizeof(float));
+ }
+
+ const double duration = (double)num_samples / FRAMESED_SAMPLE_RATE;
+ if (extract_events(scores, frames, duration, output, all_classes) != 0 ||
+ (unsuppressed && track_step > 0.0 &&
+ build_tracks(unsuppressed, frames, track_step, output, all_classes) != 0)) {
+ framesed_result_free(output);
+ free(unsuppressed);
+ free(scores);
+ return -1;
+ }
+ free(scores);
+
+ if (options->keep_planes) {
+ output->planes = unsuppressed;
+ output->num_groups = num_groups;
+ output->num_frames = frames;
+ } else {
+ free(unsuppressed);
+ }
+ return 0;
+}
+
+void framesed_result_free(framesed_result *output) {
+ if (!output)
+ return;
+ free(output->events);
+ for (size_t i = 0; i < output->num_tracks; ++i)
+ free(output->tracks[i].values);
+ free(output->tracks);
+ free(output->planes);
+ memset(output, 0, sizeof(*output));
+}
+
+const char *framesed_backend(const framesed_t *engine) { return engine ? engine->backend : "unavailable"; }
diff --git a/framesed/framesed.h b/framesed/framesed.h
new file mode 100644
index 0000000..cc77607
--- /dev/null
+++ b/framesed/framesed.h
@@ -0,0 +1,100 @@
+#ifndef OFFLINE_AUDIO_FRAMESED_H
+#define OFFLINE_AUDIO_FRAMESED_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+#define FRAMESED_SAMPLE_RATE 16000
+#define FRAMESED_CLASS_COUNT 447
+#define FRAMESED_FRAME_STEP 0.04
+
+typedef struct framesed framesed_t;
+
+typedef struct {
+ float *logits; /* [FRAMESED_CLASS_COUNT][frames] */
+ size_t frames;
+} framesed_logits;
+
+typedef struct {
+ double start;
+ double end;
+ float score;
+ const char *label;
+ int class_group; /* caption group index, or model class index in all_classes mode */
+} framesed_event;
+
+/* A per-group or per-class detection score, downsampled from the model's 40 ms
+ * frames by taking the maximum in each output step. These are the scores
+ * before caption suppression, so a continuous background sound keeps its
+ * score while a more specific event is being captioned. */
+typedef struct {
+ const char *label;
+ int class_group; /* same index namespace as framesed_event.class_group */
+ float *values;
+ size_t count;
+} framesed_track;
+
+typedef struct {
+ framesed_event *events;
+ size_t num_events;
+ framesed_track *tracks;
+ size_t num_tracks;
+ double track_step;
+ /* Present only with keep_planes: unsuppressed scores [num_groups][num_frames]
+ * at FRAMESED_FRAME_STEP seconds, in framesed_group_label order, or framesed_class_label
+ * order with all_classes. In that mode num_groups is FRAMESED_CLASS_COUNT. */
+ float *planes;
+ size_t num_groups;
+ size_t num_frames;
+} framesed_result;
+
+/* A zeroed struct selects caption detection, serial execution, and no tracks. */
+typedef struct {
+ int threads; /* model windows in flight; 0 or 1 is serial */
+ double track_step; /* seconds per score-track value; 0 returns no tracks */
+ int keep_planes; /* retain the unsuppressed group/class score plane */
+ /* Called once per finished model window from a worker thread, serialized
+ * by the engine. NULL disables it. */
+ void (*progress)(void *context, size_t done, size_t total);
+ void *progress_context;
+ /* Nonzero returns each model class independently, without caption grouping
+ * or suppression. Uniform gates: start 0.50, release 0.35, minimum 0.24 s,
+ * gap 0.32 s. These are not per-class calibrated thresholds. */
+ int all_classes;
+} framesed_options;
+
+/* Caption-group label (regardless of detection mode), or NULL out of range. */
+const char *framesed_group_label(size_t group);
+size_t framesed_group_count(void);
+
+/* Model output metadata, indexed 0..FRAMESED_CLASS_COUNT-1. Returned strings have
+ * static lifetime; invalid indices return NULL. IDs are AudioSet ontology IDs. */
+const char *framesed_class_label(size_t class_index);
+const char *framesed_class_mid(size_t class_index);
+
+/* Loads the PretrainedSED frame_mn10 weight blob. */
+framesed_t *framesed_create(const char *weights_path);
+void framesed_destroy(framesed_t *engine);
+
+/* Runs one 10-second model window and returns its native frame scores. */
+int framesed_infer_s16(framesed_t *engine, const int16_t *samples, size_t num_samples, framesed_logits *output);
+void framesed_logits_free(framesed_logits *output);
+
+/* Detects caption-ready sound events, or individual model classes when
+ * all_classes is set, with bounded inference chunks. Events may overlap.
+ * Model windows run independently on up to `options->threads` workers; the result
+ * does not depend on the worker count. `options` may be NULL.
+ *
+ * A positive `track_step` in seconds also returns a score track for every
+ * group (or class) reaching its release threshold. The returned step is rounded
+ * down to whole model frames, with a minimum of FRAMESED_FRAME_STEP. */
+int framesed_detect_s16(framesed_t *engine,
+ const int16_t *samples,
+ size_t num_samples,
+ const framesed_options *options,
+ framesed_result *output);
+void framesed_result_free(framesed_result *output);
+
+const char *framesed_backend(const framesed_t *engine);
+
+#endif
diff --git a/framesed/framesed_classes.inc b/framesed/framesed_classes.inc
new file mode 100644
index 0000000..a407651
--- /dev/null
+++ b/framesed/framesed_classes.inc
@@ -0,0 +1,448 @@
+/* Model output order; regenerate with make regenerate-classes. */
+ {"/m/07q2z82", "Accelerating, revving, vroom"},
+ {"/m/0gvgw0", "Air brake"},
+ {"/m/025wky1", "Air conditioning"},
+ {"/m/05x_td", "Air horn, truck horn"},
+ {"/m/0k5j", "Aircraft"},
+ {"/m/014yck", "Aircraft engine"},
+ {"/m/07pp_mv", "Alarm"},
+ {"/m/046dlr", "Alarm clock"},
+ {"/t/dd00144", "Alert"},
+ {"/m/012n7d", "Ambulance (siren)"},
+ {"/m/0jbk", "Animal"},
+ {"/m/028ght", "Applause"},
+ {"/m/0dl83", "Arrow"},
+ {"/m/0_1c", "Artillery fire"},
+ {"/t/dd00142", "Audio logo"},
+ {"/m/0261r1", "Babbling"},
+ {"/t/dd00002", "Baby cry, infant cry"},
+ {"/t/dd00001", "Baby laughter"},
+ {"/m/093_4n", "Background noise"},
+ {"/m/07pws3f", "Bang"},
+ {"/m/05tny_", "Bark"},
+ {"/m/018w8", "Basketball bounce"},
+ {"/m/01j2bj", "Bathroom sounds"},
+ {"/m/03dnzn", "Bathtub (filling or washing)"},
+ {"/m/04gy_2", "Battle cry"},
+ {"/m/01h3n", "Bee, wasp, etc."},
+ {"/m/02fs_r", "Beep, bleep"},
+ {"/m/0395lw", "Bell"},
+ {"/m/07q4ntr", "Bellow"},
+ {"/m/07sq110", "Belly laugh"},
+ {"/m/0gy1t2s", "Bicycle bell"},
+ {"/m/0199g", "Bicycle, tricycle"},
+ {"/m/015p6", "Bird"},
+ {"/m/05_wcq", "Bird flight, flapping wings"},
+ {"/m/020bb7", "Bird vocalization, bird call, bird song"},
+ {"/m/07pdhp0", "Biting"},
+ {"/m/07q0h5t", "Bleat"},
+ {"/m/02pjr4", "Blender, food processor"},
+ {"/m/019jd", "Boat, Water vehicle"},
+ {"/m/0dv3j", "Boiling"},
+ {"/t/dd00121", "Boing"},
+ {"/m/04v5dt", "Booing"},
+ {"/m/07qqyl4", "Boom"},
+ {"/m/07pqn27", "Bouncing"},
+ {"/m/07rc7d9", "Bow-wow"},
+ {"/m/07pc8lb", "Breaking"},
+ {"/m/0lyf6", "Breathing"},
+ {"/t/dd00138", "Brief tone"},
+ {"/m/03q5_w", "Burping, eructation"},
+ {"/m/07qsvvw", "Burst, pop"},
+ {"/m/01bjv", "Bus"},
+ {"/m/01z47d", "Busy signal"},
+ {"/m/07pjwq1", "Buzz"},
+ {"/m/030rvx", "Buzzer"},
+ {"/m/07szfh9", "Cacophony"},
+ {"/m/0dv5r", "Camera"},
+ {"/m/01z5f", "Canidae, wild dogs, wolves"},
+ {"/m/073cg4", "Cap gun"},
+ {"/m/0k4j", "Car"},
+ {"/m/02mfyn", "Car alarm"},
+ {"/t/dd00134", "Car passing by"},
+ {"/m/08dckq", "Carbon monoxide detector, CO detector"},
+ {"/m/018p4k", "Cart"},
+ {"/m/024dl", "Cash register"},
+ {"/m/01yrx", "Cat"},
+ {"/m/07r81j2", "Caterwaul"},
+ {"/m/01xq0k1", "Cattle, bovinae"},
+ {"/m/07r5c2p", "Caw"},
+ {"/m/01sb50", "Cellphone buzz, vibrating alert"},
+ {"/m/0269r2s", "Chain"},
+ {"/m/01j4z9", "Chainsaw"},
+ {"/m/0150b9", "Change ringing (campanology)"},
+ {"/t/dd00123", "Channel, environment and background"},
+ {"/m/02bk07", "Chant"},
+ {"/m/053hz1", "Cheering"},
+ {"/m/03cczk", "Chewing, mastication"},
+ {"/m/09b5t", "Chicken, rooster"},
+ {"/t/dd00005", "Child singing"},
+ {"/m/0ytgt", "Child speech, kid speaking"},
+ {"/t/dd00013", "Children playing"},
+ {"/t/dd00135", "Children shouting"},
+ {"/m/0f8s22", "Chime"},
+ {"/m/02021", "Chipmunk"},
+ {"/m/0hdsk", "Chirp tone"},
+ {"/m/07pggtn", "Chirp, tweet"},
+ {"/m/0l14jd", "Choir"},
+ {"/m/07pczhz", "Chop"},
+ {"/m/07pn_8q", "Chopping (food)"},
+ {"/m/07m2kt", "Chorus effect"},
+ {"/m/07rgt08", "Chuckle, chortle"},
+ {"/m/03w41f", "Church bell"},
+ {"/m/0dgbq", "Civil defense siren"},
+ {"/m/07rv4dm", "Clang"},
+ {"/m/0l15bq", "Clapping"},
+ {"/m/07qmpdm", "Clatter"},
+ {"/m/07rwm0c", "Clickety-clack"},
+ {"/m/07qc9xj", "Clicking"},
+ {"/m/07rv9rh", "Clip-clop"},
+ {"/m/01x3z", "Clock"},
+ {"/m/07st89h", "Cluck"},
+ {"/t/dd00108", "Clunk"},
+ {"/m/0242l", "Coin (dropping)"},
+ {"/m/01m2v", "Computer keyboard"},
+ {"/m/01h8n0", "Conversation"},
+ {"/m/07r_25d", "Coo"},
+ {"/m/01b_21", "Cough"},
+ {"/m/0239kh", "Cowbell"},
+ {"/m/07qs1cx", "Crack"},
+ {"/m/07pzfmf", "Crackle"},
+ {"/m/07qh7jl", "Creak"},
+ {"/m/09xqv", "Cricket"},
+ {"/m/07st88b", "Croak"},
+ {"/m/0hgq8df", "Crockery breaking and smashing"},
+ {"/m/04s8yn", "Crow"},
+ {"/m/03qtwd", "Crowd"},
+ {"/m/07qn5dc", "Crowing, cock-a-doodle-doo"},
+ {"/t/dd00112", "Crumpling, crinkling"},
+ {"/m/07s12q4", "Crunch"},
+ {"/m/07plct2", "Crushing"},
+ {"/m/0463cq4", "Crying, sobbing"},
+ {"/m/0642b4", "Cupboard open or close"},
+ {"/m/023pjk", "Cutlery, silverware"},
+ {"/t/dd00110", "Deformable shell"},
+ {"/m/08j51y", "Dental drill, dentist's drill"},
+ {"/m/015jpf", "Dial tone"},
+ {"/m/0160x5", "Digestive"},
+ {"/m/07phxs1", "Ding"},
+ {"/m/07r67yg", "Ding-dong"},
+ {"/m/04brg2", "Dishes, pots, and pans"},
+ {"/m/0g12c5", "Distortion"},
+ {"/m/0bt9lr", "Dog"},
+ {"/m/068hy", "Domestic animals, pets"},
+ {"/t/dd00147", "Dong, bong"},
+ {"/m/0ffhf", "Donkey, ass"},
+ {"/m/02dgv", "Door"},
+ {"/m/03wwcy", "Doorbell"},
+ {"/m/0fqfqc", "Drawer open or close"},
+ {"/m/01d380", "Drill"},
+ {"/m/07r5v4s", "Drip"},
+ {"/m/04179zz", "Duck call (hunting tool)"},
+ {"/m/09ddx", "Ducks, geese, waterfowl"},
+ {"/m/01jnbd", "Echo"},
+ {"/m/02rr_", "Effects unit"},
+ {"/m/0c1tlg", "Electric rotor drone, quadcopter"},
+ {"/m/02g901", "Electric shaver, electric razor"},
+ {"/m/04fgwm", "Electric toothbrush"},
+ {"/m/0b_fwt", "Electronic tuner"},
+ {"/m/03j1ly", "Emergency vehicle"},
+ {"/m/02mk9", "Engine"},
+ {"/m/01h82_", "Engine knocking"},
+ {"/t/dd00130", "Engine starting"},
+ {"/m/06_y0by", "Environmental noise"},
+ {"/m/098_xr", "Error signal"},
+ {"/m/07pxg6y", "Eruption"},
+ {"/m/014zdl", "Explosion"},
+ {"/m/02_nn", "Fart"},
+ {"/t/dd00004", "Female singing"},
+ {"/m/02zsn", "Female speech, woman speaking"},
+ {"/m/02p01q", "Filing (rasp)"},
+ {"/m/07p7b8y", "Fill (with liquid)"},
+ {"/m/025_jnm", "Finger snapping"},
+ {"/m/02_41", "Fire"},
+ {"/m/0c3f7m", "Fire alarm"},
+ {"/m/012ndj", "Fire engine, fire truck (siren)"},
+ {"/g/122z_qxw", "Firecracker"},
+ {"/m/0g6b5", "Fireworks"},
+ {"/m/0cmf2", "Fixed-wing aircraft, airplane"},
+ {"/t/dd00118", "Fizz"},
+ {"/m/07pyf11", "Flap"},
+ {"/m/0h2mp", "Fly, housefly"},
+ {"/m/04fq5q", "Foghorn"},
+ {"/m/025rv6n", "Fowl"},
+ {"/m/09ld4", "Frog"},
+ {"/m/0dxrf", "Frying (food)"},
+ {"/m/02z32qm", "Fusillade"},
+ {"/m/0939n_", "Gargling"},
+ {"/m/07s0dtb", "Gasp"},
+ {"/m/0l7xg", "Gears"},
+ {"/t/dd00099", "Generic impact sounds"},
+ {"/m/07r660_", "Giggle"},
+ {"/m/039jq", "Glass"},
+ {"/m/07q7njn", "Glass chink, clink"},
+ {"/m/07rn7sz", "Glass shatter"},
+ {"/m/03fwl", "Goat"},
+ {"/m/07svc2k", "Gobble"},
+ {"/m/07pt6mm", "Grind"},
+ {"/m/07s2xch", "Groan"},
+ {"/m/0ghcn6", "Growling"},
+ {"/m/07r4k75", "Grunt"},
+ {"/m/01dwxx", "Gull, seagull"},
+ {"/m/032s66", "Gunshot, gunfire"},
+ {"/m/07swgks", "Gurgling, bubbling"},
+ {"/t/dd00088", "Gush"},
+ {"/m/03wvsk", "Hair dryer"},
+ {"/m/03l9g", "Hammer"},
+ {"/m/0k65p", "Hands"},
+ {"/m/01jg02", "Heart sounds, heartbeat"},
+ {"/t/dd00067", "Heavy engine (low frequency)"},
+ {"/m/09ct_", "Helicopter"},
+ {"/m/02p3nc", "Hiccup"},
+ {"/m/07rjwbb", "Hiss"},
+ {"/m/07qwf61", "Honk"},
+ {"/m/07r_80w", "Hoot"},
+ {"/m/03k3r", "Horse"},
+ {"/m/07qf0zm", "Howl"},
+ {"/m/07q8f3b", "Howl (wind)"},
+ {"/m/07qfr4h", "Hubbub, speech noise, speech babble"},
+ {"/m/07rcgpl", "Hum"},
+ {"/t/dd00012", "Human group actions"},
+ {"/m/0bpl036", "Human locomotion"},
+ {"/m/0dgw9r", "Human sounds"},
+ {"/m/09l8g", "Human voice"},
+ {"/m/02fxyj", "Humming"},
+ {"/m/03cl9h", "Ice cream truck, ice cream van"},
+ {"/m/07pb8fc", "Idling"},
+ {"/m/03vt0", "Insect"},
+ {"/t/dd00126", "Inside, large room or hall"},
+ {"/t/dd00127", "Inside, public space"},
+ {"/t/dd00125", "Inside, small room"},
+ {"/m/03p19w", "Jackhammer"},
+ {"/m/04229", "Jet engine"},
+ {"/m/027m70_", "Jingle bell"},
+ {"/m/07qfgpx", "Jingle, tinkle"},
+ {"/g/11b630rrvh", "Kettle whistle"},
+ {"/m/056r_1", "Keypress tone"},
+ {"/m/03v3yw", "Keys jangling"},
+ {"/m/0d4wf", "Kitchen and dining room sounds"},
+ {"/m/04ctx", "Knife"},
+ {"/m/07r4wb8", "Knock"},
+ {"/m/01j3sz", "Laughter"},
+ {"/m/01yg9g", "Lawn mower"},
+ {"/t/dd00065", "Light engine (high frequency)"},
+ {"/m/04k94", "Liquid"},
+ {"/m/0ch8v", "Livestock, farm animals, working animals"},
+ {"/m/02ll1_", "Lock"},
+ {"/m/04zjc", "Machine gun"},
+ {"/m/06xkwv", "Mains hum"},
+ {"/t/dd00003", "Male singing"},
+ {"/m/05zppz", "Male speech, man speaking"},
+ {"/m/01c194", "Mantra"},
+ {"/m/06cyt0", "Mechanical bell"},
+ {"/m/02x984l", "Mechanical fan"},
+ {"/t/dd00077", "Mechanisms"},
+ {"/t/dd00066", "Medium engine (mid frequency)"},
+ {"/m/07qrkrw", "Meow"},
+ {"/m/0hg7b", "Microphone"},
+ {"/m/0fx9l", "Microwave oven"},
+ {"/m/07rpkh9", "Moo"},
+ {"/m/09f96", "Mosquito"},
+ {"/m/012f08", "Motor vehicle (road)"},
+ {"/m/02rlv9", "Motorboat, speedboat"},
+ {"/m/04_sv", "Motorcycle"},
+ {"/m/04rmv", "Mouse"},
+ {"/m/04rlf", "Music"},
+ {"/m/02qldy", "Narration, monologue"},
+ {"/m/07q5rw0", "Neigh, whinny"},
+ {"/m/096m7z", "Noise"},
+ {"/t/dd00061", "Non-motorized land vehicle"},
+ {"/m/05kq4", "Ocean"},
+ {"/t/dd00018", "Oink"},
+ {"/t/dd00122", "Other sourceless"},
+ {"/t/dd00128", "Outside, urban or manmade"},
+ {"/m/09d5_", "Owl"},
+ {"/m/05mxj0q", "Packing tape, duct tape"},
+ {"/m/07pyy8b", "Pant"},
+ {"/t/dd00141", "Pant (dog)"},
+ {"/m/0641k", "Paper rustling"},
+ {"/m/07r4gkf", "Patter"},
+ {"/m/068zj", "Pig"},
+ {"/m/0h0rv", "Pigeon, dove"},
+ {"/m/07qwdck", "Ping"},
+ {"/m/07qyrcz", "Plop"},
+ {"/m/04qvtq", "Police car (siren)"},
+ {"/m/07prgkl", "Pour"},
+ {"/m/0md09", "Power saw, circular saw, table saw"},
+ {"/m/0_ksk", "Power tool"},
+ {"/m/04gxbd", "Power windows, electric windows"},
+ {"/m/01m4t", "Printer"},
+ {"/m/02l6bg", "Propeller, airscrew"},
+ {"/m/07q34h3", "Puff"},
+ {"/m/05zc1", "Pulleys"},
+ {"/m/07pt_g0", "Pulse"},
+ {"/m/07ptzwd", "Pump (liquid)"},
+ {"/m/02yds9", "Purr"},
+ {"/m/07qdb04", "Quack"},
+ {"/m/0ltv", "Race car, auto racing"},
+ {"/m/06bz3", "Radio"},
+ {"/m/06d_3", "Rail transport"},
+ {"/m/01g50p", "Railroad car, train wagon"},
+ {"/m/06mb1", "Rain"},
+ {"/t/dd00038", "Rain on surface"},
+ {"/m/07r10fb", "Raindrop"},
+ {"/m/06bxc", "Rapping"},
+ {"/m/02bm9n", "Ratchet, pawl"},
+ {"/m/07qn4z3", "Rattle"},
+ {"/m/040b_t", "Refrigerator"},
+ {"/m/09hlz4", "Respiratory sounds"},
+ {"/m/01b9nn", "Reverberation"},
+ {"/m/02rhddq", "Reversing beeps"},
+ {"/m/0bcdqg", "Ringing tone, ringback tone"},
+ {"/m/01hnzm", "Ringtone"},
+ {"/m/04cvmfc", "Roar"},
+ {"/m/0cdnk", "Roaring cats (lions, tigers)"},
+ {"/m/06hps", "Rodents, rats, mice"},
+ {"/m/07s8j8t", "Roll"},
+ {"/m/056ks2", "Rowboat, canoe, kayak"},
+ {"/m/07pdjhy", "Rub"},
+ {"/m/07phhsh", "Rumble"},
+ {"/m/06h7j", "Run"},
+ {"/m/07qwyj0", "Rustle"},
+ {"/m/0hsrw", "Sailboat, sailing ship"},
+ {"/m/023vsd", "Sanding"},
+ {"/m/01b82r", "Sawing"},
+ {"/m/01lsmm", "Scissors"},
+ {"/m/07qv4k0", "Scrape"},
+ {"/m/07qb_dv", "Scratch"},
+ {"/m/03qc9zr", "Screaming"},
+ {"/m/07q8k13", "Screech"},
+ {"/m/0llzx", "Sewing machine"},
+ {"/m/07bgp", "Sheep"},
+ {"/m/06q74", "Ship"},
+ {"/m/07p6fty", "Shout"},
+ {"/m/02f9f_", "Shower"},
+ {"/m/07qv_x_", "Shuffle"},
+ {"/m/05rj2", "Shuffling cards"},
+ {"/m/07plz5l", "Sigh"},
+ {"/m/01v_m0", "Sine wave"},
+ {"/m/015lz1", "Singing"},
+ {"/m/07bjf", "Single-lens reflex camera"},
+ {"/m/0130jx", "Sink (filling or washing)"},
+ {"/m/03kmc9", "Siren"},
+ {"/m/07p9k1k", "Sizzle"},
+ {"/m/06_fw", "Skateboard"},
+ {"/m/07rjzl8", "Slam"},
+ {"/m/07ryjzk", "Slap, smack"},
+ {"/m/02y_763", "Sliding door"},
+ {"/m/07p6mqd", "Slosh"},
+ {"/m/07pqmly", "Slurp, drinking straw"},
+ {"/m/07pjjrj", "Smash, crash"},
+ {"/m/01y3hg", "Smoke detector, smoke alarm"},
+ {"/m/078jl", "Snake"},
+ {"/m/07pc8l3", "Snap"},
+ {"/m/01hsr_", "Sneeze"},
+ {"/m/07s04w4", "Snicker"},
+ {"/m/07ppn3j", "Sniff"},
+ {"/m/01d3sd", "Snoring"},
+ {"/m/07q0yl5", "Snort"},
+ {"/t/dd00139", "Snort (horse)"},
+ {"/m/0790c", "Sonar"},
+ {"/m/0193bn", "Sonic boom"},
+ {"/m/0c1dj", "Sound effect"},
+ {"/t/dd00091", "Sound equipment"},
+ {"/m/07bm98", "Sound reproduction"},
+ {"/m/09x0r", "Speech"},
+ {"/m/0brhx", "Speech synthesizer"},
+ {"/m/07rrlb6", "Splash, splatter"},
+ {"/m/07pl1bw", "Splinter"},
+ {"/m/07qlf79", "Spray"},
+ {"/m/07sx8x_", "Squawk"},
+ {"/m/07q6cd_", "Squeak"},
+ {"/m/07s02z0", "Squeal"},
+ {"/m/07qlwh6", "Squish"},
+ {"/m/01lynh", "Stairs"},
+ {"/m/07rgkc5", "Static"},
+ {"/m/06wzb", "Steam"},
+ {"/m/06hck5", "Steam whistle"},
+ {"/m/07ptfmf", "Stir"},
+ {"/m/01g90h", "Stomach rumble"},
+ {"/m/07sk0jz", "Stomp, stamp"},
+ {"/m/0j6m2", "Stream, river"},
+ {"/m/0195fx", "Subway, metro, underground"},
+ {"/t/dd00109", "Surface contact"},
+ {"/m/07s13rg", "Sweeping"},
+ {"/t/dd00006", "Synthetic singing"},
+ {"/m/07qcpgn", "Tap"},
+ {"/m/0fw86", "Tap dance"},
+ {"/m/09d1b1", "Tape hiss"},
+ {"/m/07qcx4z", "Tearing"},
+ {"/m/07cx4", "Telephone"},
+ {"/m/07pp8cl", "Telephone bell ringing"},
+ {"/m/02c8p", "Telephone dialing, DTMF"},
+ {"/m/07c52", "Television"},
+ {"/m/0dl9sf8", "Throat clearing"},
+ {"/m/07qnq_y", "Thump, thud"},
+ {"/m/0ngt1", "Thunder"},
+ {"/m/0jb2l", "Thunderstorm"},
+ {"/m/07rrh0c", "Thunk"},
+ {"/m/07qjznt", "Tick"},
+ {"/m/07qjznl", "Tick-tock"},
+ {"/m/0h9mv", "Tire squeal, skidding"},
+ {"/m/01jt3m", "Toilet flush"},
+ {"/m/07k1x", "Tools"},
+ {"/m/012xff", "Toothbrush"},
+ {"/m/0btp2", "Traffic noise, roadway noise"},
+ {"/m/07jdr", "Train"},
+ {"/m/0284vy3", "Train horn"},
+ {"/t/dd00048", "Train wheels squealing"},
+ {"/m/04zmvq", "Train whistle"},
+ {"/m/07pqc89", "Trickle, dribble"},
+ {"/m/07r04", "Truck"},
+ {"/m/07n_g", "Tuning fork"},
+ {"/m/01rd7k", "Turkey"},
+ {"/m/0c2wf", "Typewriter"},
+ {"/m/0316dw", "Typing"},
+ {"/t/dd00143", "Unknown sound"},
+ {"/m/0d31p", "Vacuum cleaner"},
+ {"/m/07yv9", "Vehicle"},
+ {"/m/0912c9", "Vehicle horn, car horn, honking, toot"},
+ {"/m/0zmy2j9", "Velcro, hook and loop fastener"},
+ {"/m/0bzvm2", "Video game sound"},
+ {"/m/07qw_06", "Wail, moan"},
+ {"/m/07pbtc8", "Walk, footsteps"},
+ {"/m/0174k2", "Washing machine"},
+ {"/m/0838f", "Water"},
+ {"/m/02jz0l", "Water tap, faucet"},
+ {"/m/0j2kx", "Waterfall"},
+ {"/m/034srq", "Waves, surf"},
+ {"/m/07rdhzs", "Whack, thwack"},
+ {"/m/032n05", "Whale vocalization"},
+ {"/m/07mzm6", "Wheeze"},
+ {"/m/07qz6j3", "Whimper"},
+ {"/t/dd00136", "Whimper (dog)"},
+ {"/m/07rbp7_", "Whip"},
+ {"/m/07s34ls", "Whir"},
+ {"/m/02rtxlg", "Whispering"},
+ {"/m/0l156k", "Whistle"},
+ {"/m/01w250", "Whistling"},
+ {"/m/0chx_", "White noise, pink noise"},
+ {"/m/07rwj3x", "Whoop"},
+ {"/m/07rqsjt", "Whoosh, swoosh, swish"},
+ {"/m/01280g", "Wild animals"},
+ {"/m/0fjy1", "Wildfire"},
+ {"/m/03m9d0z", "Wind"},
+ {"/m/026fgl", "Wind chime"},
+ {"/t/dd00092", "Wind noise (microphone)"},
+ {"/m/02417f", "Windscreen wiper, windshield wiper"},
+ {"/m/07pk7mg", "Wobble"},
+ {"/m/079vc8", "Wolf-whistling"},
+ {"/m/083vt", "Wood"},
+ {"/m/081rb", "Writing"},
+ {"/m/01hhp3", "Yak"},
+ {"/m/01j423", "Yawn"},
+ {"/m/07sr1lc", "Yell"},
+ {"/m/07r_k2n", "Yip"},
+ {"/m/01swy6", "Yodeling"},
+ {"/m/07p78v5", "Zing"},
+ {"/m/01s0vc", "Zipper (clothing)"},
diff --git a/framesed/framesed_policy.inc b/framesed/framesed_policy.inc
new file mode 100644
index 0000000..fc6d6ee
--- /dev/null
+++ b/framesed/framesed_policy.inc
@@ -0,0 +1,85 @@
+/* Generated by tools/export_framesed_policy.py. */
+static const uint16_t framesed_group_classes[] = {
+ 27, 30, 71, 73, 81, 85, 90, 106, 221, 239, 253, 400, 433, 11, 93, 74,
+ 114, 17, 29, 89, 179, 229, 343, 16, 119, 420, 16, 24, 28, 80, 316, 321,
+ 428, 442, 424, 73, 77, 85, 156, 237, 239, 292, 327, 373, 444, 325, 176, 186,
+ 46, 176, 264, 345, 346, 419, 345, 105, 382, 342, 75, 20, 44, 130, 187, 204,
+ 421, 443, 64, 65, 187, 200, 244, 283, 32, 33, 34, 67, 84, 104, 113, 189,
+ 202, 262, 269, 358, 25, 52, 110, 169, 215, 248, 10, 36, 56, 66, 76, 82,
+ 97, 99, 106, 111, 115, 131, 133, 140, 171, 172, 183, 184, 201, 203, 233, 247,
+ 252, 255, 259, 267, 268, 284, 294, 301, 302, 303, 319, 340, 347, 401, 418, 430,
+ 440, 289, 290, 291, 384, 385, 205, 432, 434, 416, 108, 161, 431, 191, 200, 258,
+ 364, 369, 413, 415, 9, 91, 163, 272, 330, 407, 394, 395, 397, 4, 5, 166,
+ 220, 278, 198, 1, 3, 9, 30, 31, 38, 50, 58, 59, 60, 147, 163, 213,
+ 249, 250, 251, 257, 272, 276, 285, 287, 288, 298, 305, 310, 320, 332, 370, 389,
+ 393, 396, 399, 406, 407, 0, 70, 123, 148, 149, 150, 197, 214, 220, 230, 231,
+ 243, 411, 27, 30, 71, 81, 90, 106, 221, 400, 433, 223, 365, 425, 127, 134,
+ 135, 333, 335, 359, 374, 228, 51, 68, 124, 300, 378, 379, 380, 3, 6, 7,
+ 9, 30, 53, 59, 91, 127, 135, 162, 163, 170, 223, 272, 330, 339, 365, 407,
+ 425, 98, 387, 388, 102, 439, 128, 405, 390, 225, 446, 55, 328, 70, 312, 42,
+ 49, 153, 154, 349, 13, 57, 174, 190, 235, 164, 165, 180, 181, 182, 19,
+};
+
+static const framesed_group_definition framesed_groups[] = {
+ {"music", 0.50f, 0.35f, 0.80f, 0.56f, 0, 13, -1, UINT64_C(0x0)},
+ {"applause", 0.50f, 0.35f, 0.48f, 0.40f, 13, 2, 3, UINT64_C(0x400000000000000)},
+ {"cheering", 0.55f, 0.40f, 0.48f, 0.40f, 15, 1, 3, UINT64_C(0x0)},
+ {"crowd", 0.62f, 0.47f, 0.80f, 0.56f, 16, 1, -1, UINT64_C(0x0)},
+ {"laughter", 0.50f, 0.35f, 0.48f, 0.40f, 17, 6, -1, UINT64_C(0x0)},
+ {"crying", 0.55f, 0.40f, 0.48f, 0.40f, 23, 3, -1, UINT64_C(0x0)},
+ {"baby crying", 0.55f, 0.40f, 0.48f, 0.40f, 26, 1, 5, UINT64_C(0x0)},
+ {"shouting", 0.58f, 0.43f, 0.48f, 0.40f, 27, 7, -1, UINT64_C(0x0)},
+ {"whispering", 0.62f, 0.47f, 0.48f, 0.40f, 34, 1, -1, UINT64_C(0x0)},
+ {"singing", 0.55f, 0.40f, 0.80f, 0.56f, 35, 10, -1, UINT64_C(0x0)},
+ {"sighing", 0.60f, 0.45f, 0.48f, 0.40f, 45, 1, -1, UINT64_C(0x0)},
+ {"gasping", 0.60f, 0.45f, 0.48f, 0.40f, 46, 1, -1, UINT64_C(0x0)},
+ {"groaning", 0.60f, 0.45f, 0.48f, 0.40f, 47, 1, -1, UINT64_C(0x0)},
+ {"breathing", 0.65f, 0.50f, 0.48f, 0.40f, 48, 6, -1, UINT64_C(0x0)},
+ {"snoring", 0.58f, 0.43f, 0.80f, 0.56f, 54, 1, -1, UINT64_C(0x0)},
+ {"coughing", 0.58f, 0.43f, 0.24f, 0.32f, 55, 2, -1, UINT64_C(0x0)},
+ {"sneezing", 0.58f, 0.43f, 0.24f, 0.32f, 57, 1, -1, UINT64_C(0x0)},
+ {"chewing", 0.62f, 0.47f, 0.48f, 0.40f, 58, 1, -1, UINT64_C(0x0)},
+ {"dog", 0.55f, 0.40f, 0.48f, 0.40f, 59, 7, 22, UINT64_C(0x0)},
+ {"cat", 0.55f, 0.40f, 0.48f, 0.40f, 66, 6, 22, UINT64_C(0x0)},
+ {"birds", 0.55f, 0.40f, 0.48f, 0.40f, 72, 12, 22, UINT64_C(0x0)},
+ {"insects", 0.58f, 0.43f, 0.48f, 0.40f, 84, 6, 22, UINT64_C(0x0)},
+ {"animal", 0.65f, 0.50f, 0.80f, 0.56f, 90, 39, -1, UINT64_C(0x0)},
+ {"rain", 0.58f, 0.43f, 0.80f, 0.56f, 129, 3, 28, UINT64_C(0x0)},
+ {"thunder", 0.58f, 0.43f, 0.48f, 0.40f, 132, 2, -1, UINT64_C(0x0)},
+ {"wind", 0.60f, 0.45f, 0.80f, 0.56f, 134, 3, -1, UINT64_C(0x0)},
+ {"waves", 0.58f, 0.43f, 0.80f, 0.56f, 137, 1, 28, UINT64_C(0x0)},
+ {"fire", 0.60f, 0.45f, 0.80f, 0.56f, 138, 3, -1, UINT64_C(0x0)},
+ {"water", 0.65f, 0.50f, 0.80f, 0.56f, 141, 7, -1, UINT64_C(0x0)},
+ {"siren", 0.55f, 0.40f, 0.48f, 0.40f, 148, 5, -1, UINT64_C(0x0)},
+ {"horn", 0.58f, 0.43f, 0.24f, 0.32f, 153, 1, -1, UINT64_C(0x400000000000000)},
+ {"train", 0.58f, 0.43f, 0.80f, 0.56f, 154, 3, 34, UINT64_C(0x0)},
+ {"aircraft", 0.60f, 0.45f, 0.80f, 0.56f, 157, 5, 34, UINT64_C(0x0)},
+ {"helicopter", 0.55f, 0.40f, 0.80f, 0.56f, 162, 1, 32, UINT64_C(0x0)},
+ {"vehicle", 0.65f, 0.50f, 0.80f, 0.56f, 163, 34, -1, UINT64_C(0x0)},
+ {"engine", 0.60f, 0.45f, 0.80f, 0.56f, 197, 12, 34, UINT64_C(0x0)},
+ {"footsteps", 0.58f, 0.43f, 0.48f, 0.40f, 209, 1, -1, UINT64_C(0x0)},
+ {"bell ringing", 0.58f, 0.43f, 0.48f, 0.40f, 210, 9, -1, UINT64_C(0x0)},
+ {"whistle", 0.60f, 0.45f, 0.48f, 0.40f, 219, 3, -1, UINT64_C(0x0)},
+ {"door", 0.60f, 0.45f, 0.48f, 0.40f, 222, 7, -1, UINT64_C(0x0)},
+ {"knocking", 0.58f, 0.43f, 0.24f, 0.32f, 229, 1, 39, UINT64_C(0x400000000000000)},
+ {"telephone", 0.58f, 0.43f, 0.48f, 0.40f, 230, 7, -1, UINT64_C(0x0)},
+ {"alarm", 0.58f, 0.43f, 0.48f, 0.40f, 237, 20, -1, UINT64_C(0x0)},
+ {"clock", 0.62f, 0.47f, 0.48f, 0.40f, 257, 3, -1, UINT64_C(0x0)},
+ {"typing", 0.58f, 0.43f, 0.48f, 0.40f, 260, 1, -1, UINT64_C(0x0)},
+ {"writing", 0.62f, 0.47f, 0.48f, 0.40f, 261, 1, -1, UINT64_C(0x0)},
+ {"dishes", 0.60f, 0.45f, 0.48f, 0.40f, 262, 1, -1, UINT64_C(0x0)},
+ {"vacuum cleaner", 0.58f, 0.43f, 0.80f, 0.56f, 263, 1, -1, UINT64_C(0x0)},
+ {"toilet flush", 0.58f, 0.43f, 0.48f, 0.40f, 264, 1, 28, UINT64_C(0x0)},
+ {"keys jangling", 0.60f, 0.45f, 0.48f, 0.40f, 265, 1, -1, UINT64_C(0x0)},
+ {"zipper", 0.62f, 0.47f, 0.48f, 0.40f, 266, 1, -1, UINT64_C(0x0)},
+ {"camera shutter", 0.60f, 0.45f, 0.24f, 0.32f, 267, 2, -1, UINT64_C(0x0)},
+ {"chainsaw", 0.58f, 0.43f, 0.80f, 0.56f, 269, 1, -1, UINT64_C(0x800000000)},
+ {"sawing", 0.60f, 0.45f, 0.48f, 0.40f, 270, 1, -1, UINT64_C(0x0)},
+ {"explosion", 0.60f, 0.45f, 0.24f, 0.32f, 271, 5, -1, UINT64_C(0x400000000000000)},
+ {"gunfire", 0.60f, 0.45f, 0.24f, 0.32f, 276, 5, 54, UINT64_C(0x400000000000000)},
+ {"fireworks", 0.58f, 0.43f, 0.24f, 0.32f, 281, 2, 54, UINT64_C(0x400000000000000)},
+ {"glass breaking", 0.58f, 0.43f, 0.24f, 0.32f, 283, 3, -1, UINT64_C(0x400000000000000)},
+ {"impact", 0.65f, 0.50f, 0.24f, 0.32f, 286, 1, -1, UINT64_C(0x0)},
+};
+
+#define FRAMESED_GROUP_COUNT (sizeof(framesed_groups) / sizeof(framesed_groups[0]))
diff --git a/model_blob.c b/model_blob.c
new file mode 100644
index 0000000..a13d113
--- /dev/null
+++ b/model_blob.c
@@ -0,0 +1,111 @@
+/* The file contains a header, a fixed-size table of named tensors, and tensor
+ * data at 64-byte boundaries. */
+
+#include "model_blob.h"
+
+#include <fcntl.h>
+#include <stdint.h>
+#include <string.h>
+#include <sys/mman.h>
+#include <sys/stat.h>
+#include <unistd.h>
+
+#define MAGIC "SDC1"
+#define VERSION 1u
+#define NAME_LEN 24
+#define MAX_TENSORS 1024
+
+typedef struct {
+ char magic[4];
+ uint32_t version;
+ uint32_t count;
+ uint32_t total;
+} blob_header;
+
+typedef struct {
+ char name[NAME_LEN];
+ uint32_t offset;
+ uint32_t nbytes;
+ uint32_t digest;
+} blob_entry;
+
+static uint32_t fnv1a(const unsigned char *p, size_t n) {
+ uint32_t h = 0x811C9DC5u;
+ for (size_t i = 0; i < n; ++i)
+ h = (h ^ p[i]) * 0x01000193u;
+ return h;
+}
+
+int framesed_model_blob_parse(const void *data, size_t len, int verify, framesed_model_blob *out) {
+ if (!data || len < sizeof(blob_header))
+ return -1;
+ const blob_header *h = (const blob_header *)data;
+ if (memcmp(h->magic, MAGIC, 4) != 0 || h->version != VERSION)
+ return -1;
+ if (h->count == 0 || h->count > MAX_TENSORS)
+ return -1;
+ const size_t table_bytes = (size_t)h->count * sizeof(blob_entry);
+ if (len < sizeof(blob_header) + table_bytes || len < h->total)
+ return -1;
+
+ out->base = (const unsigned char *)data;
+ out->len = len;
+ out->table = (const unsigned char *)data + sizeof *h;
+ out->count = h->count;
+ out->verify = verify;
+ out->failed = 0;
+ return 0;
+}
+
+const float *framesed_model_blob_take(framesed_model_blob *blob, const char *name, size_t floats) {
+ if (blob->failed)
+ return NULL;
+ if (floats > SIZE_MAX / sizeof(float)) {
+ blob->failed = 1;
+ return NULL;
+ }
+ const blob_entry *table = (const blob_entry *)blob->table;
+ for (unsigned i = 0; i < blob->count; ++i) {
+ const blob_entry *e = &table[i];
+ if (strncmp(e->name, name, NAME_LEN) != 0)
+ continue;
+ if (e->nbytes != floats * sizeof(float) || e->offset % 64 != 0 || e->offset > blob->len ||
+ e->nbytes > blob->len - e->offset)
+ break;
+ const unsigned char *payload = blob->base + e->offset;
+ if (blob->verify && fnv1a(payload, e->nbytes) != e->digest)
+ break;
+ return (const float *)(const void *)payload;
+ }
+ blob->failed = 1;
+ return NULL;
+}
+
+int framesed_model_blob_map(
+ const char *path, int verify, framesed_model_blob *out, void **map, size_t *map_len) {
+ const int fd = open(path, O_RDONLY);
+ if (fd < 0)
+ return -1;
+ struct stat st;
+ if (fstat(fd, &st) != 0 || st.st_size <= 0) {
+ close(fd);
+ return -1;
+ }
+ const size_t len = (size_t)st.st_size;
+ void *addr = mmap(NULL, len, PROT_READ, MAP_PRIVATE, fd, 0);
+ close(fd);
+ if (addr == MAP_FAILED)
+ return -1;
+ if (framesed_model_blob_parse(addr, len, verify, out) != 0) {
+ munmap(addr, len);
+ return -1;
+ }
+ *map = addr;
+ *map_len = len;
+ return 0;
+}
+
+void framesed_model_blob_unmap(void *map, size_t len) {
+ if (map)
+ munmap(map, len);
+}
diff --git a/model_blob.h b/model_blob.h
new file mode 100644
index 0000000..433947d
--- /dev/null
+++ b/model_blob.h
@@ -0,0 +1,36 @@
+/* Access to packed, named float tensors in a read-only model blob. */
+
+#ifndef FRAMESED_MODEL_BLOB_H
+#define FRAMESED_MODEL_BLOB_H
+
+#include <stddef.h>
+
+/* Tensor lookup returns NULL and records a failure, so a loader can request
+ * every tensor in a straight line and check the blob once at the end. */
+typedef struct {
+ const unsigned char *base;
+ size_t len;
+ const void *table;
+ unsigned count;
+ int verify;
+ int failed;
+} framesed_model_blob;
+
+/* Returns 0 when the header and table are well formed. Tensor contents are
+ * only checked as they are taken. */
+int framesed_model_blob_parse(const void *data, size_t len, int verify, framesed_model_blob *out);
+
+/* The tensor called `name`, which must be exactly `floats` long. Returns NULL
+ * and marks the blob failed on any mismatch, so a blob exported from a
+ * different checkpoint or a different layout is rejected at load time rather
+ * than producing plausible wrong numbers. */
+const float *framesed_model_blob_take(framesed_model_blob *blob, const char *name, size_t floats);
+
+/* Maps `path` read-only and parses it. On success the caller must keep the
+ * mapping alive for as long as any pointer taken from it is used. */
+int framesed_model_blob_map(
+ const char *path, int verify, framesed_model_blob *out, void **map, size_t *map_len);
+
+void framesed_model_blob_unmap(void *map, size_t len);
+
+#endif
diff --git a/testdata/framesed_reference.bin b/testdata/framesed_reference.bin
new file mode 100644
index 0000000..b7d7216
--- /dev/null
+++ b/testdata/framesed_reference.bin
Binary files differ
diff --git a/tests/test_classes.c b/tests/test_classes.c
new file mode 100644
index 0000000..6dcf19f
--- /dev/null
+++ b/tests/test_classes.c
@@ -0,0 +1,124 @@
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "framesed.h"
+
+#define CHECK(condition) do { \
+ if (!(condition)) { \
+ fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \
+ return 1; \
+ } \
+} while (0)
+
+static int check_result(const framesed_result *result, double duration) {
+ CHECK(result->planes && result->num_groups == FRAMESED_CLASS_COUNT);
+ CHECK(result->num_frames == (size_t)ceil(duration / FRAMESED_FRAME_STEP));
+ for (size_t i = 0; i < result->num_groups * result->num_frames; ++i)
+ CHECK(result->planes[i] >= 0.0f && result->planes[i] <= 1.0f);
+ int music = 0;
+ for (size_t i = 0; i < result->num_events; ++i) {
+ const framesed_event *event = &result->events[i];
+ CHECK(event->class_group >= 0 && event->class_group < FRAMESED_CLASS_COUNT);
+ CHECK(event->label == framesed_class_label((size_t)event->class_group));
+ CHECK(event->start >= 0.0 && event->end > event->start && event->end <= duration);
+ CHECK(event->score >= 0.50f && event->score <= 1.0f);
+ CHECK(i == 0 || result->events[i - 1].start <= event->start);
+ if (event->class_group == 253)
+ music = 1;
+ }
+ CHECK(music);
+ CHECK(result->num_tracks > 0 && result->track_step == 0.2);
+ for (size_t i = 0; i < result->num_tracks; ++i) {
+ const framesed_track *track = &result->tracks[i];
+ CHECK(track->class_group >= 0 && track->class_group < FRAMESED_CLASS_COUNT);
+ CHECK(track->label == framesed_class_label((size_t)track->class_group));
+ CHECK(track->count == (result->num_frames + 4) / 5);
+ CHECK(i == 0 || result->tracks[i - 1].class_group < track->class_group);
+ const float *row = result->planes + (size_t)track->class_group * result->num_frames;
+ for (size_t j = 0; j < track->count; ++j) {
+ float maximum = 0.0f;
+ for (size_t f = j * 5; f < (j + 1) * 5 && f < result->num_frames; ++f)
+ if (row[f] > maximum)
+ maximum = row[f];
+ CHECK(track->values[j] == maximum);
+ }
+ }
+ return 0;
+}
+
+int main(int argc, char **argv) {
+ const char *weights = argc > 1 ? argv[1] : "models/framesed_weights.bin";
+ /* A partial final frame/window exercises padding and duration clipping. */
+ const size_t count = 30u * FRAMESED_SAMPLE_RATE + 137;
+ const double duration = (double)count / FRAMESED_SAMPLE_RATE;
+ int16_t *samples = malloc(count * sizeof(*samples));
+ CHECK(samples);
+ for (size_t i = 0; i < count; ++i) {
+ const double t = (double)i / FRAMESED_SAMPLE_RATE;
+ double value = (sin(2 * M_PI * 261.6 * t) + sin(2 * M_PI * 329.6 * t) +
+ sin(2 * M_PI * 392.0 * t)) / 3;
+ value *= 0.65 + 0.35 * sin(2 * M_PI * 4.0 * t);
+ samples[i] = (int16_t)(value * 0.10 * 32767.0);
+ }
+ framesed_t *engine = framesed_create(weights);
+ CHECK(engine);
+ framesed_options options = {
+ .threads = 1, .track_step = 0.2, .keep_planes = 1, .all_classes = 1,
+ };
+ framesed_result one = {0};
+ CHECK(framesed_detect_s16(engine, samples, 10u * FRAMESED_SAMPLE_RATE, &options, &one) == 0);
+ CHECK(check_result(&one, 10.0) == 0);
+ framesed_logits logits = {0};
+ CHECK(framesed_infer_s16(engine, samples, 10u * FRAMESED_SAMPLE_RATE, &logits) == 0);
+ CHECK(one.num_frames == logits.frames);
+ for (size_t i = 0; i < FRAMESED_CLASS_COUNT * logits.frames; ++i) {
+ const float expected = 1.0f / (1.0f + expf(-logits.logits[i]));
+ CHECK(fabsf(one.planes[i] - expected) < 2e-7f);
+ }
+ framesed_logits_free(&logits);
+ framesed_result_free(&one);
+
+ CHECK(framesed_detect_s16(engine, samples, count, &options, &one) == 0);
+ CHECK(check_result(&one, duration) == 0);
+ options.threads = 4;
+ framesed_result many = {0};
+ CHECK(framesed_detect_s16(engine, samples, count, &options, &many) == 0);
+ CHECK(check_result(&many, duration) == 0);
+ CHECK(one.num_events == many.num_events && one.num_tracks == many.num_tracks);
+ CHECK(one.num_frames == many.num_frames && one.num_groups == many.num_groups);
+ CHECK(memcmp(one.planes, many.planes,
+ one.num_frames * one.num_groups * sizeof(float)) == 0);
+ for (size_t i = 0; i < one.num_events; ++i) {
+ const framesed_event *a = &one.events[i], *b = &many.events[i];
+ CHECK(a->class_group == b->class_group && a->label == b->label);
+ CHECK(a->start == b->start && a->end == b->end && a->score == b->score);
+ }
+ for (size_t i = 0; i < one.num_tracks; ++i) {
+ const framesed_track *a = &one.tracks[i], *b = &many.tracks[i];
+ CHECK(a->class_group == b->class_group && a->label == b->label && a->count == b->count);
+ CHECK(memcmp(a->values, b->values, a->count * sizeof(float)) == 0);
+ }
+ printf("framesed classes: 447 score planes match native logits; 1/4 workers agree "
+ "(%zu events, %zu tracks)\n", one.num_events, one.num_tracks);
+ framesed_result_free(&one);
+ framesed_result_free(&many);
+ options.keep_planes = 0;
+ options.track_step = 0;
+ CHECK(framesed_detect_s16(engine, samples, 10u * FRAMESED_SAMPLE_RATE, &options, &one) == 0);
+ CHECK(one.num_events > 0 && !one.planes && !one.tracks);
+ CHECK(one.num_groups == 0 && one.num_frames == 0 && one.num_tracks == 0);
+ framesed_result_free(&one);
+ options.keep_planes = 1;
+ CHECK(framesed_detect_s16(engine, samples, 17, &options, &one) == 0);
+ CHECK(one.num_groups == FRAMESED_CLASS_COUNT && one.num_frames == 1 && one.planes);
+ CHECK(one.num_events == 0);
+ framesed_result_free(&one);
+ CHECK(framesed_detect_s16(engine, NULL, 0, &options, &one) == 0);
+ CHECK(!one.planes && !one.events && !one.tracks);
+ framesed_result_free(&one);
+ framesed_destroy(engine);
+ free(samples);
+ return 0;
+}
diff --git a/tests/test_cli.py b/tests/test_cli.py
new file mode 100644
index 0000000..858f439
--- /dev/null
+++ b/tests/test_cli.py
@@ -0,0 +1,76 @@
+"""CLI vocabulary, caption/class output, and argument contracts."""
+
+import csv
+import math
+from pathlib import Path
+import struct
+import subprocess
+import sys
+import tempfile
+import wave
+
+
+WEIGHTS = sys.argv[1] if len(sys.argv) > 1 else "models/framesed_weights.bin"
+
+
+def run(*args, code=0):
+ result = subprocess.run(
+ ["build/framesed", *map(str, args)], capture_output=True, text=True
+ )
+ if result.returncode != code:
+ raise AssertionError(f"{args}: exit {result.returncode}\n{result.stderr}")
+ return result.stdout
+
+
+def main():
+ assert "--all-classes" in run("--help")
+ with Path("framesed/classes.csv").open(newline="", encoding="utf-8") as source:
+ rows = list(csv.DictReader(source))
+ expected = "".join(
+ f"{row['index']}\t{row['mid']}\t{row['display_name']}\n" for row in rows
+ )
+ assert len(rows) == 447
+ assert run("--list-classes") == expected
+ assert run("--all-classes", code=2) == ""
+ run("--threads", "0", code=2)
+ run("--track-step", "nan", code=2)
+ run("--unknown-option", code=2)
+
+ with tempfile.TemporaryDirectory(prefix="framesed-cli-") as directory:
+ path = Path(directory) / "music.wav"
+ samples = []
+ for i in range(10 * 16000):
+ t = i / 16000
+ value = sum(math.sin(2 * math.pi * f * t) for f in (261.6, 329.6, 392.0)) / 3
+ value *= 0.65 + 0.35 * math.sin(2 * math.pi * 4 * t)
+ samples.append(int(value * 0.10 * 32767))
+ with wave.open(str(path), "wb") as output:
+ output.setparams((1, 2, 16000, 0, "NONE", "not compressed"))
+ output.writeframes(struct.pack(f"<{len(samples)}h", *samples))
+ caption = run("--weights", WEIGHTS, path)
+ assert any(line.endswith(" 0 music") for line in caption.splitlines())
+ classes = run("--all-classes", "--track-step", "0.2", "--weights", WEIGHTS, path)
+ events = 0
+ tracks = 0
+ music = False
+ for line in classes.splitlines():
+ kind = line.split(" ", 1)[0]
+ if kind == "event":
+ _, start, end, score, index, label = line.split(" ", 5)
+ assert 0 <= float(start) < float(end) <= 10.0
+ events += 1
+ music |= int(index) == 253
+ else:
+ assert kind == "track"
+ _, time, score, index, label = line.split(" ", 4)
+ assert 0 <= float(time) < 10.0
+ tracks += 1
+ assert 0 <= float(score) <= 1
+ assert 0 <= int(index) < 447
+ assert label == rows[int(index)]["display_name"]
+ assert events and tracks and music
+ print("framesed CLI: 447 labels, caption/class detection, tracks, arguments passed")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/test_framesed.c b/tests/test_framesed.c
new file mode 100644
index 0000000..5294a79
--- /dev/null
+++ b/tests/test_framesed.c
@@ -0,0 +1,280 @@
+#include <math.h>
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "framesed.h"
+
+typedef struct {
+ char magic[12];
+ uint32_t version;
+ uint32_t samples;
+ uint32_t frames;
+ uint32_t classes;
+ uint32_t tracks;
+} reference_header;
+
+static int16_t *make_waveform(size_t count) {
+ int16_t *samples = malloc(count * sizeof(*samples));
+ if (!samples)
+ return NULL;
+ uint32_t state = 1;
+ for (size_t index = 0; index < count; ++index) {
+ state = state * 1664525u + 1013904223u;
+ const int noise = (int)((state >> 16) & 0xffffu) - 32768;
+ const int saw = (int)(index % 1600) - 800;
+ int value = noise / 8 + saw * 20;
+ if (value < -32768)
+ value = -32768;
+ if (value > 32767)
+ value = 32767;
+ samples[index] = (int16_t)value;
+ }
+ return samples;
+}
+
+/* A major triad under a 4 Hz tremolo, at the moderate level the model scores
+ * highest for music. This gives the event and score-track gates below
+ * something to detect without shipping a recording. */
+static int16_t *make_music_waveform(size_t count) {
+ int16_t *samples = malloc(count * sizeof(*samples));
+ if (!samples)
+ return NULL;
+ static const double partials[] = {261.6, 329.6, 392.0};
+ for (size_t index = 0; index < count; ++index) {
+ const double t = (double)index / FRAMESED_SAMPLE_RATE;
+ double value = 0.0;
+ for (size_t p = 0; p < sizeof partials / sizeof partials[0]; ++p)
+ value += sin(2.0 * M_PI * partials[p] * t);
+ value /= (double)(sizeof partials / sizeof partials[0]);
+ value *= 0.65 + 0.35 * sin(2.0 * M_PI * 4.0 * t);
+ samples[index] = (int16_t)(value * 0.10 * 32767.0);
+ }
+ return samples;
+}
+
+static void *read_file(const char *path, size_t *size) {
+ FILE *file = fopen(path, "rb");
+ if (!file)
+ return NULL;
+ if (fseek(file, 0, SEEK_END) != 0) {
+ fclose(file);
+ return NULL;
+ }
+ const long length = ftell(file);
+ if (length <= 0 || fseek(file, 0, SEEK_SET) != 0) {
+ fclose(file);
+ return NULL;
+ }
+ void *data = malloc((size_t)length);
+ if (!data || fread(data, 1, (size_t)length, file) != (size_t)length) {
+ free(data);
+ fclose(file);
+ return NULL;
+ }
+ fclose(file);
+ *size = (size_t)length;
+ return data;
+}
+
+int main(int argc, char **argv) {
+ const char *weights = argc > 1 ? argv[1] : "models/framesed_weights.bin";
+ const char *reference_path = argc > 2 ? argv[2] : "testdata/framesed_reference.bin";
+ size_t reference_size = 0;
+ unsigned char *reference = read_file(reference_path, &reference_size);
+ if (!reference || reference_size < sizeof(reference_header)) {
+ fprintf(stderr, "cannot read FRAMESED reference\n");
+ free(reference);
+ return 1;
+ }
+ const reference_header *header = (const reference_header *)reference;
+ if (memcmp(header->magic, "FRAMESEDR", 9) != 0 || header->version != 1 ||
+ header->classes != FRAMESED_CLASS_COUNT) {
+ fprintf(stderr, "invalid FRAMESED reference\n");
+ free(reference);
+ return 1;
+ }
+ const size_t expected_size =
+ sizeof(*header) + (size_t)header->tracks * sizeof(uint32_t) +
+ ((size_t)header->classes + (size_t)header->tracks * header->frames) * sizeof(float);
+ if (reference_size != expected_size) {
+ fprintf(stderr, "truncated FRAMESED reference\n");
+ free(reference);
+ return 1;
+ }
+
+ int16_t *samples = make_waveform(header->samples);
+ framesed_t *engine = framesed_create(weights);
+ framesed_logits result;
+ if (!samples || !engine || framesed_infer_s16(engine, samples, header->samples, &result) != 0) {
+ fprintf(stderr, "FRAMESED inference failed\n");
+ free(samples);
+ framesed_destroy(engine);
+ free(reference);
+ return 1;
+ }
+ if (result.frames != header->frames) {
+ fprintf(stderr, "FRAMESED frame count differs: %zu != %u\n", result.frames, header->frames);
+ framesed_logits_free(&result);
+ framesed_destroy(engine);
+ free(samples);
+ free(reference);
+ return 1;
+ }
+
+ const uint32_t *track_classes = (const uint32_t *)(reference + sizeof(*header));
+ const float *expected_global = (const float *)(track_classes + header->tracks);
+ const float *expected_tracks = expected_global + header->classes;
+ float max_global = 0.0f;
+ float max_track = 0.0f;
+ for (uint32_t class_index = 0; class_index < header->classes; ++class_index) {
+ const float *row = result.logits + (size_t)class_index * result.frames;
+ float sum = 0.0f;
+ for (size_t frame = 0; frame < result.frames; ++frame)
+ sum += row[frame];
+ const float error = fabsf(sum / result.frames - expected_global[class_index]);
+ if (error > max_global)
+ max_global = error;
+ }
+ for (uint32_t track = 0; track < header->tracks; ++track) {
+ const float *actual = result.logits + (size_t)track_classes[track] * result.frames;
+ const float *expected = expected_tracks + (size_t)track * result.frames;
+ for (size_t frame = 0; frame < result.frames; ++frame) {
+ const float error = fabsf(actual[frame] - expected[frame]);
+ if (error > max_track)
+ max_track = error;
+ }
+ }
+
+ printf("framesed parity: global %.6g, tracks %.6g (%s)\n", max_global, max_track, framesed_backend(engine));
+ int failed = max_global > 2e-3f || max_track > 4e-3f;
+ framesed_logits_free(&result);
+ framesed_result events = {0};
+ const framesed_options serial = {.threads = 1, .track_step = 0.2};
+ if (framesed_detect_s16(engine, samples, header->samples, &serial, &events) != 0) {
+ fprintf(stderr, "FRAMESED event detection failed\n");
+ failed = 1;
+ }
+ for (size_t i = 0; i < events.num_events; ++i) {
+ const framesed_event *event = &events.events[i];
+ if (!event->label || event->start < 0.0 || event->end <= event->start ||
+ event->end > (double)header->samples / FRAMESED_SAMPLE_RATE + 1e-9 || event->score < 0.0f ||
+ event->score > 1.0f || (i && events.events[i - 1].start > event->start)) {
+ fprintf(stderr, "invalid FRAMESED event result\n");
+ failed = 1;
+ break;
+ }
+ }
+ printf("framesed policy: %zu deterministic event%s\n", events.num_events,
+ events.num_events == 1 ? "" : "s");
+
+ /* Model windows write disjoint frame ranges chosen by window index, so the
+ * worker count must not reach the output. */
+ framesed_result threaded = {0};
+ const framesed_options parallel = {.threads = 4, .track_step = 0.2};
+ if (framesed_detect_s16(engine, samples, header->samples, &parallel, &threaded) != 0) {
+ fprintf(stderr, "threaded FRAMESED event detection failed\n");
+ failed = 1;
+ } else if (threaded.num_events != events.num_events) {
+ fprintf(stderr, "FRAMESED event count depends on the worker count: %zu vs %zu\n",
+ threaded.num_events, events.num_events);
+ failed = 1;
+ } else {
+ for (size_t i = 0; i < events.num_events; ++i) {
+ const framesed_event *one = &events.events[i];
+ const framesed_event *many = &threaded.events[i];
+ if (one->start != many->start || one->end != many->end || one->score != many->score ||
+ one->label != many->label) {
+ fprintf(stderr, "FRAMESED event %zu depends on the worker count\n", i);
+ failed = 1;
+ break;
+ }
+ }
+ }
+ printf("framesed threads: 1 and 4 workers agree exactly\n");
+
+ const double expected_step = 0.2;
+ const size_t expected_count =
+ (size_t)((double)header->samples / FRAMESED_SAMPLE_RATE / expected_step + 0.9999);
+ for (size_t i = 0; i < events.num_tracks; ++i) {
+ const framesed_track *track = &events.tracks[i];
+ if (track->count != expected_count || !track->label) {
+ fprintf(stderr, "score track %zu has %zu values, expected %zu\n", i, track->count,
+ expected_count);
+ failed = 1;
+ break;
+ }
+ for (size_t v = 0; v < track->count; ++v)
+ if (!(track->values[v] >= 0.0f && track->values[v] <= 1.0f)) {
+ fprintf(stderr, "score track %s leaves [0, 1]\n", track->label);
+ failed = 1;
+ break;
+ }
+ }
+ if (events.num_tracks && events.track_step != expected_step) {
+ fprintf(stderr, "score track step is %g, expected %g\n", events.track_step, expected_step);
+ failed = 1;
+ }
+ for (size_t i = 0; i < events.num_tracks; ++i) {
+ const framesed_track *one = &events.tracks[i];
+ const framesed_track *many = &threaded.tracks[i];
+ if (i >= threaded.num_tracks || one->count != many->count ||
+ memcmp(one->values, many->values, one->count * sizeof(float))) {
+ fprintf(stderr, "score track %zu depends on the worker count\n", i);
+ failed = 1;
+ break;
+ }
+ }
+ printf("framesed scores: %zu track%s of %zu values at %.2f s\n", events.num_tracks,
+ events.num_tracks == 1 ? "" : "s", events.num_tracks ? events.tracks[0].count : 0,
+ events.track_step);
+ framesed_result_free(&threaded);
+
+ /* A signal the model does recognise, so that events and score tracks are
+ * exercised rather than merely being empty and well formed. */
+ const framesed_options music_options = {.threads = 2, .track_step = 1.0, .keep_planes = 1};
+ const size_t music_samples = 30u * FRAMESED_SAMPLE_RATE;
+ int16_t *music = make_music_waveform(music_samples);
+ framesed_result music_result = {0};
+ if (!music || framesed_detect_s16(engine, music, music_samples, &music_options, &music_result) != 0) {
+ fprintf(stderr, "music FRAMESED detection failed\n");
+ failed = 1;
+ } else {
+ size_t music_events = 0;
+ double covered = 0.0;
+ for (size_t i = 0; i < music_result.num_events; ++i)
+ if (!strcmp(music_result.events[i].label, "music")) {
+ ++music_events;
+ covered += music_result.events[i].end - music_result.events[i].start;
+ }
+ const framesed_track *music_track = NULL;
+ for (size_t i = 0; i < music_result.num_tracks; ++i)
+ if (!strcmp(music_result.tracks[i].label, "music"))
+ music_track = &music_result.tracks[i];
+ float peak = 0.0f;
+ if (music_track)
+ for (size_t v = 0; v < music_track->count; ++v)
+ if (music_track->values[v] > peak)
+ peak = music_track->values[v];
+ if (!music_events || covered < 20.0 || !music_track || peak < 0.5f ||
+ music_result.track_step != 1.0 || music_track->count != 30) {
+ fprintf(stderr,
+ "music gate: %zu events over %.1f s, track %s peak %.3f, "
+ "step %g, %zu values\n",
+ music_events, covered, music_track ? "present" : "missing", (double)peak,
+ music_result.track_step, music_track ? music_track->count : 0);
+ failed = 1;
+ } else {
+ printf("framesed music: %zu event%s over %.1f s, track peak %.2f at 1.00 s\n", music_events,
+ music_events == 1 ? "" : "s", covered, (double)peak);
+ }
+ }
+ framesed_result_free(&music_result);
+ free(music);
+ framesed_result_free(&events);
+ framesed_destroy(engine);
+ free(samples);
+ free(reference);
+ return failed;
+}
diff --git a/tests/test_policy.c b/tests/test_policy.c
new file mode 100644
index 0000000..b8731bd
--- /dev/null
+++ b/tests/test_policy.c
@@ -0,0 +1,140 @@
+/* Synthetic scores isolate policy contracts from model recognition accuracy. */
+#include "../framesed/framesed.c"
+
+#define CHECK(condition) do { \
+ if (!(condition)) { \
+ fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \
+ return 1; \
+ } \
+} while (0)
+
+static int test_metadata(void) {
+ CHECK(framesed_group_count() == 59);
+ CHECK(!strcmp(framesed_group_label(0), "music"));
+ CHECK(framesed_group_label(framesed_group_count()) == NULL);
+ CHECK(framesed_class_label(FRAMESED_CLASS_COUNT) == NULL);
+ CHECK(framesed_class_mid(SIZE_MAX) == NULL);
+ CHECK(!strcmp(framesed_class_label(0), "Accelerating, revving, vroom"));
+ CHECK(!strcmp(framesed_class_label(253), "Music"));
+ CHECK(!strcmp(framesed_class_mid(253), "/m/04rlf"));
+ CHECK(!strcmp(framesed_class_label(446), "Zipper (clothing)"));
+ for (size_t i = 0; i < FRAMESED_CLASS_COUNT; ++i) {
+ CHECK(framesed_class_label(i) && *framesed_class_label(i));
+ CHECK(framesed_class_mid(i) && *framesed_class_mid(i));
+ for (size_t j = 0; j < i; ++j) {
+ CHECK(strcmp(framesed_class_label(i), framesed_class_label(j)));
+ CHECK(strcmp(framesed_class_mid(i), framesed_class_mid(j)));
+ }
+ }
+ return 0;
+}
+
+static int test_probabilities(void) {
+ enum { FRAMES = 2 };
+ float logits[FRAMESED_CLASS_COUNT * FRAMES];
+ float scores[FRAMESED_CLASS_COUNT * FRAMES];
+ for (size_t i = 0; i < FRAMESED_CLASS_COUNT * FRAMES; ++i)
+ logits[i] = ((float)i - 447.0f) / 100.0f;
+ const framesed_logits input = {.logits = logits, .frames = FRAMES};
+ CHECK(group_probabilities(&input, scores, 1) == 0);
+ for (size_t i = 0; i < FRAMESED_CLASS_COUNT * FRAMES; ++i)
+ CHECK(scores[i] == sigmoid(logits[i]));
+ CHECK(group_probabilities(&input, scores, 0) == 0);
+ for (size_t group = 0; group < FRAMESED_GROUP_COUNT; ++group) {
+ const framesed_group_definition *d = &framesed_groups[group];
+ CHECK(d->class_count > 0);
+ CHECK((size_t)d->class_offset + d->class_count <=
+ sizeof(framesed_group_classes) / sizeof(framesed_group_classes[0]));
+ for (size_t frame = 0; frame < FRAMES; ++frame) {
+ float maximum = 0.0f;
+ for (size_t member = 0; member < d->class_count; ++member) {
+ const size_t index = framesed_group_classes[d->class_offset + member];
+ CHECK(index < FRAMESED_CLASS_COUNT);
+ const float p = sigmoid(logits[index * FRAMES + frame]);
+ if (p > maximum)
+ maximum = p;
+ }
+ CHECK(scores[group * FRAMES + frame] == maximum);
+ }
+ }
+ return 0;
+}
+
+static int test_all_classes(void) {
+ enum { FRAMES = 12 };
+ float scores[FRAMESED_CLASS_COUNT * FRAMES];
+ for (size_t i = 0; i < FRAMESED_CLASS_COUNT * FRAMES; ++i)
+ scores[i] = 0.75f;
+ framesed_result result = {0};
+ CHECK(extract_events(scores, FRAMES, 0.48, &result, 1) == 0);
+ CHECK(result.num_events == FRAMESED_CLASS_COUNT);
+ CHECK(build_tracks(scores, FRAMES, 0.2, &result, 1) == 0);
+ CHECK(result.num_tracks == FRAMESED_CLASS_COUNT);
+ for (size_t i = 0; i < FRAMESED_CLASS_COUNT; ++i) {
+ const framesed_event *event = &result.events[i];
+ CHECK(event->class_group == (int)i);
+ CHECK(event->label == framesed_class_label(i));
+ CHECK(event->start == 0.0 && event->end == 0.48 && event->score == 0.75f);
+ const framesed_track *track = &result.tracks[i];
+ CHECK(track->class_group == (int)i && track->label == framesed_class_label(i));
+ CHECK(track->count == 3);
+ for (size_t frame = 0; frame < track->count; ++frame)
+ CHECK(track->values[frame] == 0.75f);
+ }
+ framesed_result_free(&result);
+ return 0;
+}
+
+static int test_timing(void) {
+ enum { FRAMES = 30, CLASS = FRAMESED_CLASS_COUNT - 1 };
+ float scores[FRAMESED_CLASS_COUNT * FRAMES] = {0};
+ float *row = scores + CLASS * FRAMES;
+ row[1] = 0.75f;
+ for (size_t i = 2; i <= 8; ++i)
+ row[i] = 0.40f;
+ row[3] = 0.0f; /* A brief dropout must not split the event. */
+ for (size_t i = 20; i < 23; ++i)
+ row[i] = 0.9f; /* Below the minimum duration. */
+ for (size_t i = 0; i < FRAMES; ++i)
+ scores[10 * FRAMES + i] = 0.4f; /* Track without an event. */
+ framesed_result result = {0};
+ CHECK(extract_events(scores, FRAMES, 1.2, &result, 1) == 0);
+ CHECK(result.num_events == 1);
+ CHECK(result.events[0].class_group == CLASS);
+ CHECK(result.events[0].start == FRAMESED_FRAME_STEP);
+ CHECK(result.events[0].end == 9 * FRAMESED_FRAME_STEP);
+ CHECK(result.events[0].score == 0.75f);
+ CHECK(build_tracks(scores, FRAMES, 0.2, &result, 1) == 0);
+ CHECK(result.num_tracks == 2 && result.track_step == 0.2);
+ CHECK(result.tracks[0].class_group == 10 && result.tracks[1].class_group == CLASS);
+ CHECK(result.tracks[1].count == 6);
+ CHECK(result.tracks[1].values[0] == 0.75f);
+ CHECK(result.tracks[1].values[4] == 0.9f);
+ framesed_result_free(&result);
+ return 0;
+}
+
+static int test_caption_suppression(void) {
+ enum { FRAMES = 30 };
+ float scores[FRAMESED_GROUP_COUNT * FRAMES] = {0};
+ for (size_t i = 0; i < FRAMES; ++i) {
+ scores[1 * FRAMES + i] = 0.75f; /* applause */
+ scores[3 * FRAMES + i] = 0.75f; /* crowd */
+ scores[58 * FRAMES + i] = 0.75f; /* impact */
+ }
+ framesed_result result = {0};
+ CHECK(build_tracks(scores, FRAMES, 0.2, &result, 0) == 0);
+ CHECK(result.num_tracks == 3);
+ CHECK(extract_events(scores, FRAMES, 1.2, &result, 0) == 0);
+ CHECK(result.num_events == 1 && result.events[0].class_group == 1);
+ framesed_result_free(&result);
+ return 0;
+}
+
+int main(void) {
+ if (test_metadata() || test_probabilities() || test_all_classes() ||
+ test_timing() || test_caption_suppression())
+ return 1;
+ puts("framesed policy: metadata, 447 classes, timing, tracks, caption suppression passed");
+ return 0;
+}
diff --git a/tools/framesed.c b/tools/framesed.c
new file mode 100644
index 0000000..f5b519f
--- /dev/null
+++ b/tools/framesed.c
@@ -0,0 +1,138 @@
+#include <errno.h>
+#include <limits.h>
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "framesed.h"
+#include "wav.h"
+
+static void usage(FILE *output, const char *program) {
+ fprintf(output,
+ "usage: %s [--all-classes] [--threads N] [--track-step SECONDS] "
+ "--weights FILE INPUT.wav\n"
+ " %s --list-classes\n",
+ program, program);
+}
+
+static int parse_threads(const char *text, int *value) {
+ char *end = NULL;
+ errno = 0;
+ long parsed = strtol(text, &end, 10);
+ if (errno || !end || *end || parsed < 1 || parsed > INT_MAX)
+ return -1;
+ *value = (int)parsed;
+ return 0;
+}
+
+static int parse_track_step(const char *text, double *value) {
+ char *end = NULL;
+ errno = 0;
+ double parsed = strtod(text, &end);
+ if (errno || !end || *end || !isfinite(parsed) || parsed < FRAMESED_FRAME_STEP)
+ return -1;
+ *value = parsed;
+ return 0;
+}
+
+int main(int argc, char **argv) {
+ const char *input_path = NULL;
+ const char *weights_path = NULL;
+ int threads = 1;
+ double track_step = 0.0;
+ int all_classes = 0;
+ int list_classes = 0;
+
+ for (int i = 1; i < argc; i++) {
+ const char *argument = argv[i];
+ if (!strcmp(argument, "-h") || !strcmp(argument, "--help")) {
+ usage(stdout, argv[0]);
+ return EXIT_SUCCESS;
+ }
+ if (!strcmp(argument, "--all-classes")) {
+ all_classes = 1;
+ } else if (!strcmp(argument, "--list-classes")) {
+ list_classes = 1;
+ } else if (!strcmp(argument, "--threads")) {
+ if (++i >= argc || parse_threads(argv[i], &threads) != 0) {
+ fprintf(stderr, "%s: invalid thread count\n", argv[0]);
+ return 2;
+ }
+ } else if (!strcmp(argument, "--track-step")) {
+ if (++i >= argc || parse_track_step(argv[i], &track_step) != 0) {
+ fprintf(stderr, "%s: invalid track step\n", argv[0]);
+ return 2;
+ }
+ } else if (!strcmp(argument, "--weights")) {
+ if (++i >= argc) {
+ fprintf(stderr, "%s: --weights needs a file\n", argv[0]);
+ return 2;
+ }
+ weights_path = argv[i];
+ } else if (argument[0] == '-') {
+ fprintf(stderr, "%s: unknown option %s\n", argv[0], argument);
+ return 2;
+ } else if (!input_path) {
+ input_path = argument;
+ } else {
+ fprintf(stderr, "%s: unexpected argument %s\n", argv[0], argument);
+ return 2;
+ }
+ }
+ if (list_classes) {
+ for (size_t i = 0; i < FRAMESED_CLASS_COUNT; ++i)
+ printf("%zu\t%s\t%s\n", i, framesed_class_mid(i), framesed_class_label(i));
+ return EXIT_SUCCESS;
+ }
+ if (!input_path || !weights_path) {
+ usage(stderr, argv[0]);
+ return 2;
+ }
+
+ wav_t wav;
+ const char *error = NULL;
+ if (wav_read_mono_s16(input_path, &wav, &error) != 0) {
+ fprintf(stderr, "%s: %s\n", input_path, error ? error : "could not read input");
+ return EXIT_FAILURE;
+ }
+ if (wav.sample_rate != FRAMESED_SAMPLE_RATE) {
+ fprintf(stderr, "%s: expected %d Hz input\n", input_path, FRAMESED_SAMPLE_RATE);
+ wav_free(&wav);
+ return EXIT_FAILURE;
+ }
+
+ framesed_t *engine = framesed_create(weights_path);
+ if (!engine) {
+ fprintf(stderr, "%s: could not load weights\n", weights_path);
+ wav_free(&wav);
+ return EXIT_FAILURE;
+ }
+ framesed_options options = {
+ .threads = threads, .track_step = track_step, .all_classes = all_classes,
+ };
+ framesed_result result;
+ if (framesed_detect_s16(engine, wav.samples, wav.count, &options, &result) != 0) {
+ fprintf(stderr, "%s: sound-event detection failed\n", input_path);
+ framesed_destroy(engine);
+ wav_free(&wav);
+ return EXIT_FAILURE;
+ }
+
+ for (size_t i = 0; i < result.num_events; i++) {
+ const framesed_event *event = &result.events[i];
+ printf("event %.3f %.3f %.6f %d %s\n", event->start, event->end, event->score,
+ event->class_group, event->label);
+ }
+ for (size_t i = 0; i < result.num_tracks; i++) {
+ const framesed_track *track = &result.tracks[i];
+ for (size_t j = 0; j < track->count; j++)
+ printf("track %.3f %.6f %d %s\n", j * result.track_step, track->values[j],
+ track->class_group, track->label);
+ }
+
+ framesed_result_free(&result);
+ framesed_destroy(engine);
+ wav_free(&wav);
+ return EXIT_SUCCESS;
+}
diff --git a/wav.c b/wav.c
new file mode 100644
index 0000000..b2a5330
--- /dev/null
+++ b/wav.c
@@ -0,0 +1,138 @@
+#include "wav.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+static int rd_u32(FILE *f, uint32_t *v) {
+ unsigned char b[4];
+ if (fread(b, 1, 4, f) != 4)
+ return -1;
+ *v = (uint32_t)b[0] | (uint32_t)b[1] << 8 | (uint32_t)b[2] << 16 | (uint32_t)b[3] << 24;
+ return 0;
+}
+
+static int rd_u16(FILE *f, uint16_t *v) {
+ unsigned char b[2];
+ if (fread(b, 1, 2, f) != 2)
+ return -1;
+ *v = (uint16_t)((uint32_t)b[0] | (uint32_t)b[1] << 8);
+ return 0;
+}
+
+#define FAIL(msg) \
+ do { \
+ if (err) \
+ *err = (msg); \
+ goto fail; \
+ } while (0)
+
+int wav_read_mono_s16(const char *path, wav_t *out, const char **err) {
+ memset(out, 0, sizeof *out);
+ FILE *f = fopen(path, "rb");
+ if (!f) {
+ if (err)
+ *err = "cannot open file";
+ return -1;
+ }
+
+ char id[4];
+ uint32_t riff_size = 0;
+ if (fread(id, 1, 4, f) != 4 || memcmp(id, "RIFF", 4) != 0)
+ FAIL("not a RIFF file");
+ if (rd_u32(f, &riff_size) != 0)
+ FAIL("truncated header");
+ if (fread(id, 1, 4, f) != 4 || memcmp(id, "WAVE", 4) != 0)
+ FAIL("not a WAVE file");
+
+ uint16_t fmt = 0, channels = 0, bits = 0;
+ uint32_t rate = 0;
+ int have_fmt = 0;
+
+ for (;;) {
+ uint32_t size;
+ if (fread(id, 1, 4, f) != 4 || rd_u32(f, &size) != 0)
+ break; /* end of file, or a truncated trailing chunk */
+
+ if (memcmp(id, "fmt ", 4) == 0) {
+ uint16_t block_align;
+ uint32_t byte_rate;
+ if (size < 16)
+ FAIL("short fmt chunk");
+ if (rd_u16(f, &fmt) || rd_u16(f, &channels) || rd_u32(f, &rate) ||
+ rd_u32(f, &byte_rate) || rd_u16(f, &block_align) || rd_u16(f, &bits))
+ FAIL("truncated fmt chunk");
+ if (fseek(f, (long)size - 16, SEEK_CUR) != 0)
+ FAIL("bad fmt chunk size");
+ have_fmt = 1;
+ continue;
+ }
+
+ if (memcmp(id, "data", 4) == 0) {
+ if (!have_fmt)
+ FAIL("data chunk before fmt chunk");
+ /* Format 1 is PCM; 0xFFFE is extensible, whose subformat this
+ * reader does not inspect, so it is accepted only at 16 bits. */
+ if ((fmt != 1 && fmt != 0xFFFE) || bits != 16)
+ FAIL("only 16-bit PCM is supported");
+ if (channels == 0)
+ FAIL("zero channels");
+
+ const size_t frames = size / (2u * channels);
+ if (frames == 0)
+ FAIL("empty data chunk");
+ int16_t *mono = malloc(frames * sizeof(int16_t));
+ if (!mono)
+ FAIL("out of memory");
+
+ const size_t step = channels;
+ int16_t buf[1024];
+ size_t got = 0;
+ const size_t batch_frames = sizeof buf / sizeof buf[0] / step;
+ if (batch_frames == 0) {
+ free(mono);
+ FAIL("too many channels");
+ }
+ while (got < frames) {
+ const size_t take = (frames - got) < batch_frames ? frames - got : batch_frames;
+ const size_t want = take * step;
+ const size_t n = fread(buf, sizeof(int16_t), want, f);
+ if (n < step)
+ break;
+ for (size_t i = 0; i + step <= n; i += step) {
+ int64_t sum = 0;
+ for (size_t c = 0; c < step; ++c)
+ sum += buf[i + c];
+ mono[got++] = (int16_t)lrint((double)sum / channels);
+ }
+ if (n < want)
+ break;
+ }
+ if (got == 0) {
+ free(mono);
+ FAIL("no samples read");
+ }
+ out->samples = mono;
+ out->count = got;
+ out->sample_rate = (int)rate;
+ out->channels = (int)channels;
+ fclose(f);
+ return 0;
+ }
+
+ /* Skip unknown chunks; RIFF chunks are word aligned. */
+ if (fseek(f, (long)size + (size & 1u), SEEK_CUR) != 0)
+ break;
+ }
+ FAIL("no data chunk");
+
+fail:
+ fclose(f);
+ return -1;
+}
+
+void wav_free(wav_t *w) {
+ free(w->samples);
+ memset(w, 0, sizeof *w);
+}
diff --git a/wav.h b/wav.h
new file mode 100644
index 0000000..3bd7c6d
--- /dev/null
+++ b/wav.h
@@ -0,0 +1,23 @@
+/* Minimal RIFF/WAVE reader for the command line tool: uncompressed 16-bit PCM
+ * only. Multichannel input is downmixed by averaging every channel. The
+ * library itself never touches files. */
+
+#ifndef FRAMESED_WAV_H
+#define FRAMESED_WAV_H
+
+#include <stddef.h>
+#include <stdint.h>
+
+typedef struct {
+ int16_t *samples; /* mono, averaged across source channels */
+ size_t count;
+ int sample_rate;
+ int channels;
+} wav_t;
+
+/* Returns 0 on success. On failure returns non-zero and, when err is
+ * non-NULL, writes a short reason there. */
+int wav_read_mono_s16(const char *path, wav_t *out, const char **err);
+void wav_free(wav_t *w);
+
+#endif