/* silero-vad v6.2 forward pass. * * Per chunk: reflect-pad, four windowed transforms, four convolutions with * ReLU, one LSTM cell, one logistic head. Every buffer below is allocated once * in svad_create and reused, so a chunk costs no allocation, no syscall and no * lock. The whole working set is about 16 KiB of activations plus a megabyte * of read-only weights. * * Activation buffers carry a zero guard row before the first time step and * after the last one. A kernel-3, pad-1 convolution therefore reads three * consecutive rows starting at row (stride * output_step), which is one * contiguous span of memory, and the whole encoder reduces to the same matvec * the LSTM and the head use. */ #include "svad.h" #include #include #include #include #include "kernels.h" #include "rfft.h" #include "weights.h" /* Time steps produced by each encoder stage; fixed because the transform * always yields SVAD_FRAMES frames and the strides are 1, 2, 2, 1. */ #define T1 SVAD_FRAMES /* 4 */ #define T2 (SVAD_FRAMES / 2) /* 2 */ #define T3 1 #define T4 1 /* Rows including the two guard rows. */ #define ROWS(t) ((t) + 2) struct svad { const svad_weights *w; const svad_kernels *k; svad_rfft fft; int use_fft; int ftz; char backend[32]; void *map; size_t map_len; svad_weights mapped; SVAD_ALIGN64 float h[SVAD_HID]; SVAD_ALIGN64 float c[SVAD_HID]; SVAD_ALIGN64 float xp[SVAD_MAX_XP]; SVAD_ALIGN64 float mag[ROWS(T1) * SVAD_MAX_BINS_PAD]; SVAD_ALIGN64 float e1[ROWS(T1) * SVAD_C1]; SVAD_ALIGN64 float e2[ROWS(T2) * SVAD_C2]; SVAD_ALIGN64 float e3[ROWS(T3) * SVAD_C3]; SVAD_ALIGN64 float lstm_in[2 * SVAD_HID]; SVAD_ALIGN64 float gates[SVAD_GATES]; SVAD_ALIGN64 float hr[SVAD_HID]; SVAD_ALIGN64 float spec[SVAD_FRAMES * 2 * SVAD_MAX_BINS]; SVAD_ALIGN64 float fft_scratch[SVAD_RFFT_SCRATCH]; }; /* ---- denormal handling ------------------------------------------------ * * The LSTM cell state decays toward zero across long silences, and denormal * operands are slow on several targets. Flushing them is scoped to the call so * the library never leaves the caller's floating point mode altered. */ typedef struct { unsigned long long saved; int active; } fp_scope; #if defined(__aarch64__) # define FPCR_FZ (1ull << 24) static inline void fp_scope_enter(fp_scope *s, int enable) { s->active = enable; if (!enable) return; unsigned long long r; __asm__ __volatile__("mrs %0, fpcr" : "=r"(r)); s->saved = r; if (!(r & FPCR_FZ)) __asm__ __volatile__("msr fpcr, %0" : : "r"(r | FPCR_FZ)); } static inline void fp_scope_leave(const fp_scope *s) { if (s->active) __asm__ __volatile__("msr fpcr, %0" : : "r"(s->saved)); } #elif defined(__SSE2__) || defined(__x86_64__) || defined(_M_X64) # include /* Bit 15 of MXCSR flushes a denormal result to zero. The neighbouring * denormals-are-zero bit is deliberately left alone: it only affects denormal * inputs, which cannot arise here because samples come from a 16-bit integer * divided by 32768 and every intermediate that could decay that far is already * covered by flushing results. It is also absent on the earliest x86-64 parts, * where writing it would fault. */ # define MXCSR_FTZ 0x8000u static inline void fp_scope_enter(fp_scope *s, int enable) { s->active = enable; if (!enable) return; s->saved = _mm_getcsr(); _mm_setcsr((unsigned)s->saved | MXCSR_FTZ); } static inline void fp_scope_leave(const fp_scope *s) { if (s->active) _mm_setcsr((unsigned)s->saved); } #else static inline void fp_scope_enter(fp_scope *s, int enable) { s->saved = 0; s->active = enable; (void)s->active; } static inline void fp_scope_leave(const fp_scope *s) { (void)s; } #endif /* ---- pointwise ------------------------------------------------------- */ static inline float sigmoidf_(float x) { return 1.0f / (1.0f + expf(-x)); } static void relu_(float *p, int n) { for (int i = 0; i < n; ++i) if (!(p[i] > 0.0f)) p[i] = 0.0f; } /* ---- forward pass ---------------------------------------------------- */ static void stft(svad_t *v) { const svad_weights *w = v->w; const int bp = w->bins_pad; float *rows = v->mag + bp; /* skip the leading guard row */ if (v->use_fft) { svad_rfft_mag(&v->fft, v->xp, w->hop, w->frames, w->stft_window, rows, bp, v->fft_scratch); return; } /* Every frame shares the basis, so one pass over it covers all of them. */ const int stride = 2 * w->bins; v->k->matvec(v->spec, stride, w->stft_basis, v->xp, w->hop, NULL, stride, w->n_fft, w->frames); for (int t = 0; t < w->frames; ++t) { const float *src = v->spec + (size_t)t * stride; float *dst = rows + (size_t)t * bp; for (int f = 0; f < w->bins; ++f) { const float re = src[f], im = src[w->bins + f]; dst[f] = sqrtf(re * re + im * im); } } } /* Each encoder stage runs all its output time steps in a single call, so the * weights are streamed once per chunk rather than once per time step. Output * step s reads the window starting at input row s * stride and writes row * s + 1, leaving the guard rows untouched. */ static float forward(svad_t *v) { const svad_weights *w = v->w; const svad_kernels *k = v->k; const int bp = w->bins_pad; stft(v); k->matvec(v->e1 + SVAD_C1, SVAD_C1, w->conv1_w, v->mag, bp, w->conv1_b, SVAD_C1, 3 * bp, T1); relu_(v->e1 + SVAD_C1, T1 * SVAD_C1); k->matvec(v->e2 + SVAD_C2, SVAD_C2, w->conv2_w, v->e1, 2 * SVAD_C1, w->conv2_b, SVAD_C2, 3 * SVAD_C1, T2); relu_(v->e2 + SVAD_C2, T2 * SVAD_C2); k->matvec(v->e3 + SVAD_C3, 0, w->conv3_w, v->e2, 0, w->conv3_b, SVAD_C3, 3 * SVAD_C2, T3); relu_(v->e3 + SVAD_C3, T3 * SVAD_C3); /* conv4 writes the LSTM input directly; the hidden half follows it. */ k->matvec(v->lstm_in, 0, w->conv4_w, v->e3, 0, w->conv4_b, SVAD_C4, 3 * SVAD_C3, T4); relu_(v->lstm_in, SVAD_C4); memcpy(v->lstm_in + SVAD_HID, v->h, sizeof v->h); k->matvec(v->gates, 0, w->lstm_w, v->lstm_in, 0, w->lstm_b, SVAD_GATES, 2 * SVAD_HID, 1); for (int i = 0; i < SVAD_HID; ++i) { const float ig = sigmoidf_(v->gates[i]); const float fg = sigmoidf_(v->gates[SVAD_HID + i]); const float gg = tanhf(v->gates[2 * SVAD_HID + i]); const float og = sigmoidf_(v->gates[3 * SVAD_HID + i]); const float cn = fg * v->c[i] + ig * gg; const float hn = og * tanhf(cn); v->c[i] = cn; v->h[i] = hn; v->hr[i] = hn > 0.0f ? hn : 0.0f; } return sigmoidf_(k->dot(w->head_w, v->hr, SVAD_HID) + w->head_b); } /* ---- lifecycle ------------------------------------------------------- */ svad_t *svad_create_ex(const char *path, int sample_rate, unsigned flags) { const svad_weights *builtin = svad_weights_builtin(sample_rate); if (!builtin) return NULL; svad_t *v = NULL; if (posix_memalign((void **)&v, 64, sizeof *v) != 0 || !v) return NULL; memset(v, 0, sizeof *v); /* zeroes the guard rows and the channel padding */ if (path) { const int verify = !(flags & SVAD_NO_VERIFY); if (svad_weights_map(path, sample_rate, verify, &v->mapped, &v->map, &v->map_len) != 0) { free(v); return NULL; } v->w = &v->mapped; } else { v->w = builtin; } const int scalar = (flags & SVAD_SCALAR_KERNELS) != 0; v->k = scalar ? svad_kernels_scalar() : svad_kernels_simd(); v->use_fft = (flags & SVAD_DENSE_STFT) == 0; v->ftz = (flags & SVAD_NO_FTZ) == 0; svad_rfft_init(&v->fft, v->w->n_fft, !scalar); snprintf(v->backend, sizeof v->backend, "%s+%s", v->k->name, v->use_fft ? "rfft" : "dense"); svad_reset(v); return v; } svad_t *svad_create(int sample_rate) { return svad_create_ex(NULL, sample_rate, 0); } svad_t *svad_create_from_file(const char *path, int sample_rate) { return svad_create_ex(path, sample_rate, 0); } void svad_destroy(svad_t *v) { if (!v) return; svad_weights_unmap(v->map, v->map_len); free(v); } void svad_reset(svad_t *v) { memset(v->h, 0, sizeof v->h); memset(v->c, 0, sizeof v->c); memset(v->xp, 0, (size_t)v->w->context * sizeof(float)); } int svad_chunk_samples(const svad_t *v) { return v->w->chunk; } int svad_sample_rate(const svad_t *v) { return v->w->sample_rate; } const char *svad_backend_name(const svad_t *v) { return v->backend; } void svad_get_state(const svad_t *v, float *out) { memcpy(out, v->h, sizeof v->h); memcpy(out + SVAD_HID, v->c, sizeof v->c); } void svad_set_state(svad_t *v, const float *in) { memcpy(v->h, in, sizeof v->h); memcpy(v->c, in + SVAD_HID, sizeof v->c); } /* ---- per-chunk entry points ------------------------------------------ */ /* Right-side reflect pad only: xp[total + i] = xp[total - 2 - i]. A symmetric * or left-side pad yields plausible but wrong probabilities. */ static void reflect_pad(svad_t *v) { const int total = v->w->context + v->w->chunk; for (int i = 0; i < v->w->pad; ++i) v->xp[total + i] = v->xp[total - 2 - i]; } /* The next chunk's context is the tail of this chunk's model input. */ static void carry_context(svad_t *v) { memmove(v->xp, v->xp + v->w->chunk, (size_t)v->w->context * sizeof(float)); } float svad_process_f32(svad_t *v, const float *chunk) { fp_scope fp; fp_scope_enter(&fp, v->ftz); memcpy(v->xp + v->w->context, chunk, (size_t)v->w->chunk * sizeof(float)); reflect_pad(v); const float p = forward(v); carry_context(v); fp_scope_leave(&fp); return p; } float svad_process_s16(svad_t *v, const int16_t *chunk) { fp_scope fp; fp_scope_enter(&fp, v->ftz); float *dst = v->xp + v->w->context; for (int i = 0; i < v->w->chunk; ++i) dst[i] = (float)chunk[i] * (1.0f / 32768.0f); reflect_pad(v); const float p = forward(v); carry_context(v); fp_scope_leave(&fp); return p; }