/* pyseg: pyannote/segmentation-3.0 speaker segmentation in portable C. * * The model reads exactly ten seconds of 16 kHz mono audio and returns, for * each of 589 output frames, a distribution over seven classes: no speaker, * one of three speakers alone, or one of the three pairs. That "powerset" * encoding is what the model was trained on; pyseg_to_multilabel turns it into * per-speaker activity, and the sliding-window driver in this header turns a * recording of any length into speech, overlap and speaker-count tracks. * * Speaker identities hold only inside a single window. Two windows that * overlap may number the same person differently, so joining windows into * whole-recording speakers needs a speaker embedding model, which is outside * the scope of this library. Speech activity, overlapped speech and the * per-frame speaker count do carry across windows and are what the sliding * window aggregates. * * One engine handles one window at a time and owns all its scratch, so a * process runs as many engines as it has threads. No allocation, no syscall * and no lock happens inside pyseg_process. */ #ifndef PYSEG_H #define PYSEG_H #include #include #ifdef __cplusplus extern "C" { #endif #define PYSEG_VERSION_MAJOR 1 #define PYSEG_VERSION_MINOR 0 #define PYSEG_SAMPLE_RATE 16000 #define PYSEG_WINDOW_SAMPLES 160000 /* 10 s */ #define PYSEG_FRAMES 589 #define PYSEG_CLASSES 7 /* powerset classes */ #define PYSEG_SPEAKERS 3 /* Output frame grid, in samples: frame f covers the 991 input samples starting * at f * PYSEG_FRAME_STEP, relative to the start of the window. */ #define PYSEG_FRAME_STEP 270 #define PYSEG_FRAME_SIZE 991 /* ---- engine ---------------------------------------------------------- */ typedef struct pyseg pyseg_t; /* Windows an engine can hold at once, and engines pyseg_slide will run at * once. Both are bounded so a caller cannot turn a large number into an * allocation proportional to it: each engine holds several megabytes. */ #define PYSEG_MAX_BATCH 64 #define PYSEG_MAX_THREADS 256 /* Creation flags. The defaults (0) select the fastest kernels available. */ enum { PYSEG_SCALAR_KERNELS = 1u << 0, /* portable C kernels, no SIMD intrinsics */ PYSEG_NO_FTZ = 1u << 1, /* leave denormal handling to the caller */ PYSEG_NO_VERIFY = 1u << 2, /* skip checksum verification of a weight file */ }; /* Uses the weights compiled into the binary. Returns NULL when the build has * none, or on allocation failure. */ pyseg_t *pyseg_create(void); /* Reads a blob produced by tools/export_weights.py. The mapping is private to * the engine and released by pyseg_destroy. */ pyseg_t *pyseg_create_from_file(const char *path); pyseg_t *pyseg_create_ex(const char *path, unsigned flags); /* An engine that processes batch windows per call. * * Batching exists for one reason: the LSTM recurrence reads a 256 KB weight * matrix per time step to do 65536 multiply-adds, so on its own it runs at the * speed of memory. Windows are independent, so several of them stepped * together share each weight load, and a batch of four cuts the recurrence, * which is nearly half of a window's cost, by better than half. * * The price is memory: the front end's scratch is shared, but each window * needs about 2.5 MB of its own from the LSTM on. A batch of one is the * smallest footprint and the lowest latency for a single window; four is the * best throughput per byte on the machines measured. Batches are also the * wrong lever for using more cores; for that, run one engine per thread. * * Returns NULL for a batch outside 1..PYSEG_MAX_BATCH. */ pyseg_t *pyseg_create_batch(const char *path, unsigned flags, int batch); int pyseg_batch(const pyseg_t *v); void pyseg_destroy(pyseg_t *v); /* Name of the active kernel set, for logs and benchmark output. */ const char *pyseg_backend_name(const pyseg_t *v); /* Bytes of scratch and weights the engine holds, for capacity planning. */ size_t pyseg_memory_bytes(const pyseg_t *v); /* One window. samples holds PYSEG_WINDOW_SAMPLES values; the f32 variant * expects [-1, 1] and the s16 variant is scaled by 1/32768 to match. probs * receives PYSEG_FRAMES * PYSEG_CLASSES values, frame-major, each frame a * distribution summing to one. */ void pyseg_process_f32(pyseg_t *v, const float *samples, float *probs); void pyseg_process_s16(pyseg_t *v, const int16_t *samples, float *probs); /* count consecutive windows, at most pyseg_batch(); outside that range the * call does nothing and leaves probs untouched. samples holds * count * PYSEG_WINDOW_SAMPLES values and probs receives * count * PYSEG_FRAMES * PYSEG_CLASSES, both window-major. Results do not * depend on how windows are grouped into batches, only on the windows * themselves. */ void pyseg_process_batch_f32(pyseg_t *v, const float *samples, int count, float *probs); /* ---- powerset decoding ----------------------------------------------- */ /* Per-frame speaker activity from powerset probabilities: active receives * PYSEG_FRAMES * PYSEG_SPEAKERS values, 1.0 where the most likely class * contains that speaker and 0.0 elsewhere. This is the hard decoding pyannote * uses; it is not a per-speaker probability. */ void pyseg_to_multilabel(const float *probs, float *active); /* Class index whose members are exactly the given speaker set, for tests. */ int pyseg_powerset_class(int speaker_a, int speaker_b); /* ---- sliding window over a whole recording --------------------------- */ /* Tracks produced per aggregated frame, in this order. */ enum { PYSEG_TRACK_SPEECH = 0, /* at least one speaker */ PYSEG_TRACK_OVERLAP = 1, /* at least two speakers */ PYSEG_TRACK_COUNT = 2, /* expected number of speakers */ PYSEG_TRACKS = 3, }; typedef struct { /* Seconds between consecutive windows. pyannote's default is 1.0, one * tenth of the window, which is also what its pipelines are tuned for. * Larger steps cost proportionally less and blur the aggregate. */ double step; /* Threads used for the window loop. 0 or 1 runs on the calling thread; * more than PYSEG_MAX_THREADS is clamped to it. */ int threads; /* Windows each thread's engine processes per call; see * pyseg_create_batch. 0 selects the default. */ int batch; /* Passed to pyseg_create_ex for the per-thread engines. */ unsigned flags; /* Weight file for those engines; NULL uses the embedded blob. */ const char *weights; } pyseg_slide_params; void pyseg_slide_params_default(pyseg_slide_params *p); /* Number of aggregated frames a recording of num_samples produces. */ size_t pyseg_slide_frames(size_t num_samples, double step); /* Runs the model over the whole recording and overlap-adds the windows the way * pyannote's Inference does: a Hamming taper across each window's frames, then * a weighted average per frame. tracks receives * pyseg_slide_frames() * PYSEG_TRACKS values, frame-major. * * Returns the number of aggregated frames, or 0 on failure. */ size_t pyseg_slide(const float *samples, size_t num_samples, const pyseg_slide_params *params, float *tracks); /* The same over 16-bit samples, scaled by 1/32768 one window at a time. * Recordings arrive as 16-bit PCM, and holding an hour of it as float32 costs * 230 MB against 115 MB, more than everything else this library allocates put * together. */ size_t pyseg_slide_s16(const int16_t *samples, size_t num_samples, const pyseg_slide_params *params, float *tracks); /* Seconds covered by aggregated frame f. The grid matches the model's own * receptive field, so frame centres are PYSEG_FRAME_STEP samples apart. */ double pyseg_frame_time(size_t frame); /* ---- binarization ---------------------------------------------------- */ typedef struct { double start; double end; } pyseg_segment; typedef struct { double onset; /* enter a region above this score */ double offset; /* leave it below this one */ double min_duration_on; /* drop regions shorter than this, in seconds */ double min_duration_off; /* fill gaps shorter than this, in seconds */ } pyseg_binarize_params; void pyseg_binarize_params_default(pyseg_binarize_params *p); /* Turns one track of an aggregated score sequence into segments, mirroring * pyannote.audio.utils.signal.Binarize. scores is strided by PYSEG_TRACKS. * * Returns the number of segments found. Nothing is written unless the whole * list fits, because the gap-filling pass is defined over the complete list; * when out is NULL or too small, size a buffer from the return value and * call again. */ size_t pyseg_binarize(const float *scores, size_t num_frames, int track, const pyseg_binarize_params *params, pyseg_segment *out, size_t out_cap); #ifdef __cplusplus } #endif #endif /* PYSEG_H */