diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:03:39 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-13 20:13:28 +0200 |
| commit | 64c3a1b528a88b4ae21bf3cbcff2537200477806 (patch) | |
| tree | 85da00748d7bbb5036aeb5a7a4b60b5abf75aeea /src/pyseg.h | |
| download | pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.tar.gz pyseg-64c3a1b528a88b4ae21bf3cbcff2537200477806.zip | |
Init
Diffstat (limited to 'src/pyseg.h')
| -rw-r--r-- | src/pyseg.h | 215 |
1 files changed, 215 insertions, 0 deletions
diff --git a/src/pyseg.h b/src/pyseg.h new file mode 100644 index 0000000..b08f7b3 --- /dev/null +++ b/src/pyseg.h @@ -0,0 +1,215 @@ +/* pyseg: pyannote/segmentation-3.0 speaker segmentation in portable C. + * + * The model reads exactly ten seconds of 16 kHz mono audio and returns, for + * each of 589 output frames, a distribution over seven classes: no speaker, + * one of three speakers alone, or one of the three pairs. That "powerset" + * encoding is what the model was trained on; pyseg_to_multilabel turns it into + * per-speaker activity, and the sliding-window driver in this header turns a + * recording of any length into speech, overlap and speaker-count tracks. + * + * Speaker identities hold only inside a single window. Two windows that + * overlap may number the same person differently, so joining windows into + * whole-recording speakers needs a speaker embedding model, which is outside + * the scope of this library. Speech activity, overlapped speech and the + * per-frame speaker count do carry across windows and are what the sliding + * window aggregates. + * + * One engine handles one window at a time and owns all its scratch, so a + * process runs as many engines as it has threads. No allocation, no syscall + * and no lock happens inside pyseg_process. */ + +#ifndef PYSEG_H +#define PYSEG_H + +#include <stddef.h> +#include <stdint.h> + +#ifdef __cplusplus +extern "C" { +#endif + +#define PYSEG_VERSION_MAJOR 1 +#define PYSEG_VERSION_MINOR 0 + +#define PYSEG_SAMPLE_RATE 16000 +#define PYSEG_WINDOW_SAMPLES 160000 /* 10 s */ +#define PYSEG_FRAMES 589 +#define PYSEG_CLASSES 7 /* powerset classes */ +#define PYSEG_SPEAKERS 3 + +/* Output frame grid, in samples: frame f covers the 991 input samples starting + * at f * PYSEG_FRAME_STEP, relative to the start of the window. */ +#define PYSEG_FRAME_STEP 270 +#define PYSEG_FRAME_SIZE 991 + +/* ---- engine ---------------------------------------------------------- */ + +typedef struct pyseg pyseg_t; + +/* Windows an engine can hold at once, and engines pyseg_slide will run at + * once. Both are bounded so a caller cannot turn a large number into an + * allocation proportional to it: each engine holds several megabytes. */ +#define PYSEG_MAX_BATCH 64 +#define PYSEG_MAX_THREADS 256 + +/* Creation flags. The defaults (0) select the fastest kernels available. */ +enum { + PYSEG_SCALAR_KERNELS = 1u << 0, /* portable C kernels, no SIMD intrinsics */ + PYSEG_NO_FTZ = 1u << 1, /* leave denormal handling to the caller */ + PYSEG_NO_VERIFY = 1u << 2, /* skip checksum verification of a weight file */ +}; + +/* Uses the weights compiled into the binary. Returns NULL when the build has + * none, or on allocation failure. */ +pyseg_t *pyseg_create(void); + +/* Reads a blob produced by tools/export_weights.py. The mapping is private to + * the engine and released by pyseg_destroy. */ +pyseg_t *pyseg_create_from_file(const char *path); + +pyseg_t *pyseg_create_ex(const char *path, unsigned flags); + +/* An engine that processes batch windows per call. + * + * Batching exists for one reason: the LSTM recurrence reads a 256 KB weight + * matrix per time step to do 65536 multiply-adds, so on its own it runs at the + * speed of memory. Windows are independent, so several of them stepped + * together share each weight load, and a batch of four cuts the recurrence, + * which is nearly half of a window's cost, by better than half. + * + * The price is memory: the front end's scratch is shared, but each window + * needs about 2.5 MB of its own from the LSTM on. A batch of one is the + * smallest footprint and the lowest latency for a single window; four is the + * best throughput per byte on the machines measured. Batches are also the + * wrong lever for using more cores; for that, run one engine per thread. + * + * Returns NULL for a batch outside 1..PYSEG_MAX_BATCH. */ +pyseg_t *pyseg_create_batch(const char *path, unsigned flags, int batch); + +int pyseg_batch(const pyseg_t *v); + +void pyseg_destroy(pyseg_t *v); + +/* Name of the active kernel set, for logs and benchmark output. */ +const char *pyseg_backend_name(const pyseg_t *v); + +/* Bytes of scratch and weights the engine holds, for capacity planning. */ +size_t pyseg_memory_bytes(const pyseg_t *v); + +/* One window. samples holds PYSEG_WINDOW_SAMPLES values; the f32 variant + * expects [-1, 1] and the s16 variant is scaled by 1/32768 to match. probs + * receives PYSEG_FRAMES * PYSEG_CLASSES values, frame-major, each frame a + * distribution summing to one. */ +void pyseg_process_f32(pyseg_t *v, const float *samples, float *probs); +void pyseg_process_s16(pyseg_t *v, const int16_t *samples, float *probs); + +/* count consecutive windows, at most pyseg_batch(); outside that range the + * call does nothing and leaves probs untouched. samples holds + * count * PYSEG_WINDOW_SAMPLES values and probs receives + * count * PYSEG_FRAMES * PYSEG_CLASSES, both window-major. Results do not + * depend on how windows are grouped into batches, only on the windows + * themselves. */ +void pyseg_process_batch_f32(pyseg_t *v, const float *samples, int count, float *probs); + +/* ---- powerset decoding ----------------------------------------------- */ + +/* Per-frame speaker activity from powerset probabilities: active receives + * PYSEG_FRAMES * PYSEG_SPEAKERS values, 1.0 where the most likely class + * contains that speaker and 0.0 elsewhere. This is the hard decoding pyannote + * uses; it is not a per-speaker probability. */ +void pyseg_to_multilabel(const float *probs, float *active); + +/* Class index whose members are exactly the given speaker set, for tests. */ +int pyseg_powerset_class(int speaker_a, int speaker_b); + +/* ---- sliding window over a whole recording --------------------------- */ + +/* Tracks produced per aggregated frame, in this order. */ +enum { + PYSEG_TRACK_SPEECH = 0, /* at least one speaker */ + PYSEG_TRACK_OVERLAP = 1, /* at least two speakers */ + PYSEG_TRACK_COUNT = 2, /* expected number of speakers */ + PYSEG_TRACKS = 3, +}; + +typedef struct { + /* Seconds between consecutive windows. pyannote's default is 1.0, one + * tenth of the window, which is also what its pipelines are tuned for. + * Larger steps cost proportionally less and blur the aggregate. */ + double step; + /* Threads used for the window loop. 0 or 1 runs on the calling thread; + * more than PYSEG_MAX_THREADS is clamped to it. */ + int threads; + /* Windows each thread's engine processes per call; see + * pyseg_create_batch. 0 selects the default. */ + int batch; + /* Passed to pyseg_create_ex for the per-thread engines. */ + unsigned flags; + /* Weight file for those engines; NULL uses the embedded blob. */ + const char *weights; +} pyseg_slide_params; + +void pyseg_slide_params_default(pyseg_slide_params *p); + +/* Number of aggregated frames a recording of num_samples produces. */ +size_t pyseg_slide_frames(size_t num_samples, double step); + +/* Runs the model over the whole recording and overlap-adds the windows the way + * pyannote's Inference does: a Hamming taper across each window's frames, then + * a weighted average per frame. tracks receives + * pyseg_slide_frames() * PYSEG_TRACKS values, frame-major. + * + * Returns the number of aggregated frames, or 0 on failure. */ +size_t pyseg_slide(const float *samples, + size_t num_samples, + const pyseg_slide_params *params, + float *tracks); + +/* The same over 16-bit samples, scaled by 1/32768 one window at a time. + * Recordings arrive as 16-bit PCM, and holding an hour of it as float32 costs + * 230 MB against 115 MB, more than everything else this library allocates put + * together. */ +size_t pyseg_slide_s16(const int16_t *samples, + size_t num_samples, + const pyseg_slide_params *params, + float *tracks); + +/* Seconds covered by aggregated frame f. The grid matches the model's own + * receptive field, so frame centres are PYSEG_FRAME_STEP samples apart. */ +double pyseg_frame_time(size_t frame); + +/* ---- binarization ---------------------------------------------------- */ + +typedef struct { + double start; + double end; +} pyseg_segment; + +typedef struct { + double onset; /* enter a region above this score */ + double offset; /* leave it below this one */ + double min_duration_on; /* drop regions shorter than this, in seconds */ + double min_duration_off; /* fill gaps shorter than this, in seconds */ +} pyseg_binarize_params; + +void pyseg_binarize_params_default(pyseg_binarize_params *p); + +/* Turns one track of an aggregated score sequence into segments, mirroring + * pyannote.audio.utils.signal.Binarize. scores is strided by PYSEG_TRACKS. + * + * Returns the number of segments found. Nothing is written unless the whole + * list fits, because the gap-filling pass is defined over the complete list; + * when out is NULL or too small, size a buffer from the return value and + * call again. */ +size_t pyseg_binarize(const float *scores, + size_t num_frames, + int track, + const pyseg_binarize_params *params, + pyseg_segment *out, + size_t out_cap); + +#ifdef __cplusplus +} +#endif + +#endif /* PYSEG_H */ |
