1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
|
/* silero-vad v6.2 forward pass.
*
* Per chunk: reflect-pad, four windowed transforms, four convolutions with
* ReLU, one LSTM cell, one logistic head. Every buffer below is allocated once
* in svad_create and reused, so a chunk costs no allocation, no syscall and no
* lock. The whole working set is about 16 KiB of activations plus a megabyte
* of read-only weights.
*
* Activation buffers carry a zero guard row before the first time step and
* after the last one. A kernel-3, pad-1 convolution therefore reads three
* consecutive rows starting at row (stride * output_step), which is one
* contiguous span of memory, and the whole encoder reduces to the same matvec
* the LSTM and the head use. */
#include "svad.h"
#include <math.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "kernels.h"
#include "rfft.h"
#include "weights.h"
/* Time steps produced by each encoder stage; fixed because the transform
* always yields SVAD_FRAMES frames and the strides are 1, 2, 2, 1. */
#define T1 SVAD_FRAMES /* 4 */
#define T2 (SVAD_FRAMES / 2) /* 2 */
#define T3 1
#define T4 1
/* Rows including the two guard rows. */
#define ROWS(t) ((t) + 2)
struct svad {
const svad_weights *w;
const svad_kernels *k;
svad_rfft fft;
int use_fft;
int ftz;
char backend[32];
void *map;
size_t map_len;
svad_weights mapped;
SVAD_ALIGN64 float h[SVAD_HID];
SVAD_ALIGN64 float c[SVAD_HID];
SVAD_ALIGN64 float xp[SVAD_MAX_XP];
SVAD_ALIGN64 float mag[ROWS(T1) * SVAD_MAX_BINS_PAD];
SVAD_ALIGN64 float e1[ROWS(T1) * SVAD_C1];
SVAD_ALIGN64 float e2[ROWS(T2) * SVAD_C2];
SVAD_ALIGN64 float e3[ROWS(T3) * SVAD_C3];
SVAD_ALIGN64 float lstm_in[2 * SVAD_HID];
SVAD_ALIGN64 float gates[SVAD_GATES];
SVAD_ALIGN64 float hr[SVAD_HID];
SVAD_ALIGN64 float spec[SVAD_FRAMES * 2 * SVAD_MAX_BINS];
SVAD_ALIGN64 float fft_scratch[SVAD_RFFT_SCRATCH];
};
/* ---- denormal handling ------------------------------------------------
*
* The LSTM cell state decays toward zero across long silences, and denormal
* operands are slow on several targets. Flushing them is scoped to the call so
* the library never leaves the caller's floating point mode altered. */
typedef struct {
unsigned long long saved;
int active;
} fp_scope;
#if defined(__aarch64__)
# define FPCR_FZ (1ull << 24)
static inline void fp_scope_enter(fp_scope *s, int enable) {
s->active = enable;
if (!enable)
return;
unsigned long long r;
__asm__ __volatile__("mrs %0, fpcr" : "=r"(r));
s->saved = r;
if (!(r & FPCR_FZ))
__asm__ __volatile__("msr fpcr, %0" : : "r"(r | FPCR_FZ));
}
static inline void fp_scope_leave(const fp_scope *s) {
if (s->active)
__asm__ __volatile__("msr fpcr, %0" : : "r"(s->saved));
}
#elif defined(__SSE2__) || defined(__x86_64__) || defined(_M_X64)
# include <xmmintrin.h>
/* Bit 15 of MXCSR flushes a denormal result to zero. The neighbouring
* denormals-are-zero bit is deliberately left alone: it only affects denormal
* inputs, which cannot arise here because samples come from a 16-bit integer
* divided by 32768 and every intermediate that could decay that far is already
* covered by flushing results. It is also absent on the earliest x86-64 parts,
* where writing it would fault. */
# define MXCSR_FTZ 0x8000u
static inline void fp_scope_enter(fp_scope *s, int enable) {
s->active = enable;
if (!enable)
return;
s->saved = _mm_getcsr();
_mm_setcsr((unsigned)s->saved | MXCSR_FTZ);
}
static inline void fp_scope_leave(const fp_scope *s) {
if (s->active)
_mm_setcsr((unsigned)s->saved);
}
#else
static inline void fp_scope_enter(fp_scope *s, int enable) {
s->saved = 0;
s->active = enable;
(void)s->active;
}
static inline void fp_scope_leave(const fp_scope *s) { (void)s; }
#endif
/* ---- pointwise ------------------------------------------------------- */
static inline float sigmoidf_(float x) { return 1.0f / (1.0f + expf(-x)); }
static void relu_(float *p, int n) {
for (int i = 0; i < n; ++i)
if (!(p[i] > 0.0f))
p[i] = 0.0f;
}
/* ---- forward pass ---------------------------------------------------- */
static void stft(svad_t *v) {
const svad_weights *w = v->w;
const int bp = w->bins_pad;
float *rows = v->mag + bp; /* skip the leading guard row */
if (v->use_fft) {
svad_rfft_mag(&v->fft, v->xp, w->hop, w->frames, w->stft_window, rows, bp, v->fft_scratch);
return;
}
/* Every frame shares the basis, so one pass over it covers all of them. */
const int stride = 2 * w->bins;
v->k->matvec(v->spec, stride, w->stft_basis, v->xp, w->hop, NULL, stride, w->n_fft, w->frames);
for (int t = 0; t < w->frames; ++t) {
const float *src = v->spec + (size_t)t * stride;
float *dst = rows + (size_t)t * bp;
for (int f = 0; f < w->bins; ++f) {
const float re = src[f], im = src[w->bins + f];
dst[f] = sqrtf(re * re + im * im);
}
}
}
/* Each encoder stage runs all its output time steps in a single call, so the
* weights are streamed once per chunk rather than once per time step. Output
* step s reads the window starting at input row s * stride and writes row
* s + 1, leaving the guard rows untouched. */
static float forward(svad_t *v) {
const svad_weights *w = v->w;
const svad_kernels *k = v->k;
const int bp = w->bins_pad;
stft(v);
k->matvec(v->e1 + SVAD_C1, SVAD_C1, w->conv1_w, v->mag, bp, w->conv1_b, SVAD_C1, 3 * bp, T1);
relu_(v->e1 + SVAD_C1, T1 * SVAD_C1);
k->matvec(v->e2 + SVAD_C2, SVAD_C2, w->conv2_w, v->e1, 2 * SVAD_C1, w->conv2_b, SVAD_C2,
3 * SVAD_C1, T2);
relu_(v->e2 + SVAD_C2, T2 * SVAD_C2);
k->matvec(v->e3 + SVAD_C3, 0, w->conv3_w, v->e2, 0, w->conv3_b, SVAD_C3, 3 * SVAD_C2, T3);
relu_(v->e3 + SVAD_C3, T3 * SVAD_C3);
/* conv4 writes the LSTM input directly; the hidden half follows it. */
k->matvec(v->lstm_in, 0, w->conv4_w, v->e3, 0, w->conv4_b, SVAD_C4, 3 * SVAD_C3, T4);
relu_(v->lstm_in, SVAD_C4);
memcpy(v->lstm_in + SVAD_HID, v->h, sizeof v->h);
k->matvec(v->gates, 0, w->lstm_w, v->lstm_in, 0, w->lstm_b, SVAD_GATES, 2 * SVAD_HID, 1);
for (int i = 0; i < SVAD_HID; ++i) {
const float ig = sigmoidf_(v->gates[i]);
const float fg = sigmoidf_(v->gates[SVAD_HID + i]);
const float gg = tanhf(v->gates[2 * SVAD_HID + i]);
const float og = sigmoidf_(v->gates[3 * SVAD_HID + i]);
const float cn = fg * v->c[i] + ig * gg;
const float hn = og * tanhf(cn);
v->c[i] = cn;
v->h[i] = hn;
v->hr[i] = hn > 0.0f ? hn : 0.0f;
}
return sigmoidf_(k->dot(w->head_w, v->hr, SVAD_HID) + w->head_b);
}
/* ---- lifecycle ------------------------------------------------------- */
svad_t *svad_create_ex(const char *path, int sample_rate, unsigned flags) {
const svad_weights *builtin = svad_weights_builtin(sample_rate);
if (!builtin)
return NULL;
svad_t *v = NULL;
if (posix_memalign((void **)&v, 64, sizeof *v) != 0 || !v)
return NULL;
memset(v, 0, sizeof *v); /* zeroes the guard rows and the channel padding */
if (path) {
const int verify = !(flags & SVAD_NO_VERIFY);
if (svad_weights_map(path, sample_rate, verify, &v->mapped, &v->map, &v->map_len) != 0) {
free(v);
return NULL;
}
v->w = &v->mapped;
} else {
v->w = builtin;
}
const int scalar = (flags & SVAD_SCALAR_KERNELS) != 0;
v->k = scalar ? svad_kernels_scalar() : svad_kernels_simd();
v->use_fft = (flags & SVAD_DENSE_STFT) == 0;
v->ftz = (flags & SVAD_NO_FTZ) == 0;
svad_rfft_init(&v->fft, v->w->n_fft, !scalar);
snprintf(v->backend, sizeof v->backend, "%s+%s", v->k->name, v->use_fft ? "rfft" : "dense");
svad_reset(v);
return v;
}
svad_t *svad_create(int sample_rate) { return svad_create_ex(NULL, sample_rate, 0); }
svad_t *svad_create_from_file(const char *path, int sample_rate) {
return svad_create_ex(path, sample_rate, 0);
}
void svad_destroy(svad_t *v) {
if (!v)
return;
svad_weights_unmap(v->map, v->map_len);
free(v);
}
void svad_reset(svad_t *v) {
memset(v->h, 0, sizeof v->h);
memset(v->c, 0, sizeof v->c);
memset(v->xp, 0, (size_t)v->w->context * sizeof(float));
}
int svad_chunk_samples(const svad_t *v) { return v->w->chunk; }
int svad_sample_rate(const svad_t *v) { return v->w->sample_rate; }
const char *svad_backend_name(const svad_t *v) { return v->backend; }
void svad_get_state(const svad_t *v, float *out) {
memcpy(out, v->h, sizeof v->h);
memcpy(out + SVAD_HID, v->c, sizeof v->c);
}
void svad_set_state(svad_t *v, const float *in) {
memcpy(v->h, in, sizeof v->h);
memcpy(v->c, in + SVAD_HID, sizeof v->c);
}
/* ---- per-chunk entry points ------------------------------------------ */
/* Right-side reflect pad only: xp[total + i] = xp[total - 2 - i]. A symmetric
* or left-side pad yields plausible but wrong probabilities. */
static void reflect_pad(svad_t *v) {
const int total = v->w->context + v->w->chunk;
for (int i = 0; i < v->w->pad; ++i)
v->xp[total + i] = v->xp[total - 2 - i];
}
/* The next chunk's context is the tail of this chunk's model input. */
static void carry_context(svad_t *v) {
memmove(v->xp, v->xp + v->w->chunk, (size_t)v->w->context * sizeof(float));
}
float svad_process_f32(svad_t *v, const float *chunk) {
fp_scope fp;
fp_scope_enter(&fp, v->ftz);
memcpy(v->xp + v->w->context, chunk, (size_t)v->w->chunk * sizeof(float));
reflect_pad(v);
const float p = forward(v);
carry_context(v);
fp_scope_leave(&fp);
return p;
}
float svad_process_s16(svad_t *v, const int16_t *chunk) {
fp_scope fp;
fp_scope_enter(&fp, v->ftz);
float *dst = v->xp + v->w->context;
for (int i = 0; i < v->w->chunk; ++i)
dst[i] = (float)chunk[i] * (1.0f / 32768.0f);
reflect_pad(v);
const float p = forward(v);
carry_context(v);
fp_scope_leave(&fp);
return p;
}
|