1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
|
/* pyseg: pyannote/segmentation-3.0 speaker segmentation in portable C.
*
* The model reads exactly ten seconds of 16 kHz mono audio and returns, for
* each of 589 output frames, a distribution over seven classes: no speaker,
* one of three speakers alone, or one of the three pairs. That "powerset"
* encoding is what the model was trained on; pyseg_to_multilabel turns it into
* per-speaker activity, and the sliding-window driver in this header turns a
* recording of any length into speech, overlap and speaker-count tracks.
*
* Speaker identities hold only inside a single window. Two windows that
* overlap may number the same person differently, so joining windows into
* whole-recording speakers needs a speaker embedding model, which is outside
* the scope of this library. Speech activity, overlapped speech and the
* per-frame speaker count do carry across windows and are what the sliding
* window aggregates.
*
* One engine handles one window at a time and owns all its scratch, so a
* process runs as many engines as it has threads. No allocation, no syscall
* and no lock happens inside pyseg_process. */
#ifndef PYSEG_H
#define PYSEG_H
#include <stddef.h>
#include <stdint.h>
#ifdef __cplusplus
extern "C" {
#endif
#define PYSEG_VERSION_MAJOR 1
#define PYSEG_VERSION_MINOR 0
#define PYSEG_SAMPLE_RATE 16000
#define PYSEG_WINDOW_SAMPLES 160000 /* 10 s */
#define PYSEG_FRAMES 589
#define PYSEG_CLASSES 7 /* powerset classes */
#define PYSEG_SPEAKERS 3
/* Output frame grid, in samples: frame f covers the 991 input samples starting
* at f * PYSEG_FRAME_STEP, relative to the start of the window. */
#define PYSEG_FRAME_STEP 270
#define PYSEG_FRAME_SIZE 991
/* ---- engine ---------------------------------------------------------- */
typedef struct pyseg pyseg_t;
/* Windows an engine can hold at once, and engines pyseg_slide will run at
* once. Both are bounded so a caller cannot turn a large number into an
* allocation proportional to it: each engine holds several megabytes. */
#define PYSEG_MAX_BATCH 64
#define PYSEG_MAX_THREADS 256
/* Creation flags. The defaults (0) select the fastest kernels available. */
enum {
PYSEG_SCALAR_KERNELS = 1u << 0, /* portable C kernels, no SIMD intrinsics */
PYSEG_NO_FTZ = 1u << 1, /* leave denormal handling to the caller */
PYSEG_NO_VERIFY = 1u << 2, /* skip checksum verification of a weight file */
};
/* Uses the weights compiled into the binary. Returns NULL when the build has
* none, or on allocation failure. */
pyseg_t *pyseg_create(void);
/* Reads a blob produced by tools/export_weights.py. The mapping is private to
* the engine and released by pyseg_destroy. */
pyseg_t *pyseg_create_from_file(const char *path);
pyseg_t *pyseg_create_ex(const char *path, unsigned flags);
/* An engine that processes batch windows per call.
*
* Batching exists for one reason: the LSTM recurrence reads a 256 KB weight
* matrix per time step to do 65536 multiply-adds, so on its own it runs at the
* speed of memory. Windows are independent, so several of them stepped
* together share each weight load, and a batch of four cuts the recurrence,
* which is nearly half of a window's cost, by better than half.
*
* The price is memory: the front end's scratch is shared, but each window
* needs about 2.5 MB of its own from the LSTM on. A batch of one is the
* smallest footprint and the lowest latency for a single window; four is the
* best throughput per byte on the machines measured. Batches are also the
* wrong lever for using more cores; for that, run one engine per thread.
*
* Returns NULL for a batch outside 1..PYSEG_MAX_BATCH. */
pyseg_t *pyseg_create_batch(const char *path, unsigned flags, int batch);
int pyseg_batch(const pyseg_t *v);
void pyseg_destroy(pyseg_t *v);
/* Name of the active kernel set, for logs and benchmark output. */
const char *pyseg_backend_name(const pyseg_t *v);
/* Bytes of scratch and weights the engine holds, for capacity planning. */
size_t pyseg_memory_bytes(const pyseg_t *v);
/* One window. samples holds PYSEG_WINDOW_SAMPLES values; the f32 variant
* expects [-1, 1] and the s16 variant is scaled by 1/32768 to match. probs
* receives PYSEG_FRAMES * PYSEG_CLASSES values, frame-major, each frame a
* distribution summing to one. */
void pyseg_process_f32(pyseg_t *v, const float *samples, float *probs);
void pyseg_process_s16(pyseg_t *v, const int16_t *samples, float *probs);
/* count consecutive windows, at most pyseg_batch(); outside that range the
* call does nothing and leaves probs untouched. samples holds
* count * PYSEG_WINDOW_SAMPLES values and probs receives
* count * PYSEG_FRAMES * PYSEG_CLASSES, both window-major. Results do not
* depend on how windows are grouped into batches, only on the windows
* themselves. */
void pyseg_process_batch_f32(pyseg_t *v, const float *samples, int count, float *probs);
/* ---- powerset decoding ----------------------------------------------- */
/* Per-frame speaker activity from powerset probabilities: active receives
* PYSEG_FRAMES * PYSEG_SPEAKERS values, 1.0 where the most likely class
* contains that speaker and 0.0 elsewhere. This is the hard decoding pyannote
* uses; it is not a per-speaker probability. */
void pyseg_to_multilabel(const float *probs, float *active);
/* Class index whose members are exactly the given speaker set, for tests. */
int pyseg_powerset_class(int speaker_a, int speaker_b);
/* ---- sliding window over a whole recording --------------------------- */
/* Tracks produced per aggregated frame, in this order. */
enum {
PYSEG_TRACK_SPEECH = 0, /* at least one speaker */
PYSEG_TRACK_OVERLAP = 1, /* at least two speakers */
PYSEG_TRACK_COUNT = 2, /* expected number of speakers */
PYSEG_TRACKS = 3,
};
typedef struct {
/* Seconds between consecutive windows. pyannote's default is 1.0, one
* tenth of the window, which is also what its pipelines are tuned for.
* Larger steps cost proportionally less and blur the aggregate. */
double step;
/* Threads used for the window loop. 0 or 1 runs on the calling thread;
* more than PYSEG_MAX_THREADS is clamped to it. */
int threads;
/* Windows each thread's engine processes per call; see
* pyseg_create_batch. 0 selects the default. */
int batch;
/* Passed to pyseg_create_ex for the per-thread engines. */
unsigned flags;
/* Weight file for those engines; NULL uses the embedded blob. */
const char *weights;
} pyseg_slide_params;
void pyseg_slide_params_default(pyseg_slide_params *p);
/* Number of aggregated frames a recording of num_samples produces. */
size_t pyseg_slide_frames(size_t num_samples, double step);
/* Runs the model over the whole recording and overlap-adds the windows the way
* pyannote's Inference does: a Hamming taper across each window's frames, then
* a weighted average per frame. tracks receives
* pyseg_slide_frames() * PYSEG_TRACKS values, frame-major.
*
* Returns the number of aggregated frames, or 0 on failure. */
size_t pyseg_slide(const float *samples,
size_t num_samples,
const pyseg_slide_params *params,
float *tracks);
/* The same over 16-bit samples, scaled by 1/32768 one window at a time.
* Recordings arrive as 16-bit PCM, and holding an hour of it as float32 costs
* 230 MB against 115 MB, more than everything else this library allocates put
* together. */
size_t pyseg_slide_s16(const int16_t *samples,
size_t num_samples,
const pyseg_slide_params *params,
float *tracks);
/* Seconds covered by aggregated frame f. The grid matches the model's own
* receptive field, so frame centres are PYSEG_FRAME_STEP samples apart. */
double pyseg_frame_time(size_t frame);
/* ---- binarization ---------------------------------------------------- */
typedef struct {
double start;
double end;
} pyseg_segment;
typedef struct {
double onset; /* enter a region above this score */
double offset; /* leave it below this one */
double min_duration_on; /* drop regions shorter than this, in seconds */
double min_duration_off; /* fill gaps shorter than this, in seconds */
} pyseg_binarize_params;
void pyseg_binarize_params_default(pyseg_binarize_params *p);
/* Turns one track of an aggregated score sequence into segments, mirroring
* pyannote.audio.utils.signal.Binarize. scores is strided by PYSEG_TRACKS.
*
* Returns the number of segments found. Nothing is written unless the whole
* list fits, because the gap-filling pass is defined over the complete list;
* when out is NULL or too small, size a buffer from the return value and
* call again. */
size_t pyseg_binarize(const float *scores,
size_t num_frames,
int track,
const pyseg_binarize_params *params,
pyseg_segment *out,
size_t out_cap);
#ifdef __cplusplus
}
#endif
#endif /* PYSEG_H */
|