diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-10-08 06:49:33 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-10-08 07:21:37 +0200 |
| commit | a5af9c3d351085f4005a949f8c382a3aad4e0b29 (patch) | |
| tree | 0ad133d931553e68a68588e983976a66b3309083 /svad.c | |
| download | svad-0.1.0.tar.gz svad-0.1.0.zip | |
initv0.1.0
Diffstat (limited to 'svad.c')
| -rw-r--r-- | svad.c | 300 |
1 files changed, 300 insertions, 0 deletions
@@ -0,0 +1,300 @@ +/* silero-vad v6.2 forward pass. + * + * Per chunk: reflect-pad, four windowed transforms, four convolutions with + * ReLU, one LSTM cell, one logistic head. Every buffer below is allocated once + * in svad_create and reused, so a chunk costs no allocation, no syscall and no + * lock. The whole working set is about 16 KiB of activations plus a megabyte + * of read-only weights. + * + * Activation buffers carry a zero guard row before the first time step and + * after the last one. A kernel-3, pad-1 convolution therefore reads three + * consecutive rows starting at row (stride * output_step), which is one + * contiguous span of memory, and the whole encoder reduces to the same matvec + * the LSTM and the head use. */ + +#include "svad.h" + +#include <math.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> + +#include "kernels.h" +#include "rfft.h" +#include "weights.h" + +/* Time steps produced by each encoder stage; fixed because the transform + * always yields SVAD_FRAMES frames and the strides are 1, 2, 2, 1. */ +#define T1 SVAD_FRAMES /* 4 */ +#define T2 (SVAD_FRAMES / 2) /* 2 */ +#define T3 1 +#define T4 1 + +/* Rows including the two guard rows. */ +#define ROWS(t) ((t) + 2) + +struct svad { + const svad_weights *w; + const svad_kernels *k; + svad_rfft fft; + int use_fft; + int ftz; + char backend[32]; + + void *map; + size_t map_len; + svad_weights mapped; + + SVAD_ALIGN64 float h[SVAD_HID]; + SVAD_ALIGN64 float c[SVAD_HID]; + + SVAD_ALIGN64 float xp[SVAD_MAX_XP]; + SVAD_ALIGN64 float mag[ROWS(T1) * SVAD_MAX_BINS_PAD]; + SVAD_ALIGN64 float e1[ROWS(T1) * SVAD_C1]; + SVAD_ALIGN64 float e2[ROWS(T2) * SVAD_C2]; + SVAD_ALIGN64 float e3[ROWS(T3) * SVAD_C3]; + SVAD_ALIGN64 float lstm_in[2 * SVAD_HID]; + SVAD_ALIGN64 float gates[SVAD_GATES]; + SVAD_ALIGN64 float hr[SVAD_HID]; + SVAD_ALIGN64 float spec[SVAD_FRAMES * 2 * SVAD_MAX_BINS]; + SVAD_ALIGN64 float fft_scratch[SVAD_RFFT_SCRATCH]; +}; + +/* ---- denormal handling ------------------------------------------------ + * + * The LSTM cell state decays toward zero across long silences, and denormal + * operands are slow on several targets. Flushing them is scoped to the call so + * the library never leaves the caller's floating point mode altered. */ + +typedef struct { + unsigned long long saved; + int active; +} fp_scope; + +#if defined(__aarch64__) +# define FPCR_FZ (1ull << 24) +static inline void fp_scope_enter(fp_scope *s, int enable) { + s->active = enable; + if (!enable) + return; + unsigned long long r; + __asm__ __volatile__("mrs %0, fpcr" : "=r"(r)); + s->saved = r; + if (!(r & FPCR_FZ)) + __asm__ __volatile__("msr fpcr, %0" : : "r"(r | FPCR_FZ)); +} +static inline void fp_scope_leave(const fp_scope *s) { + if (s->active) + __asm__ __volatile__("msr fpcr, %0" : : "r"(s->saved)); +} +#elif defined(__SSE2__) || defined(__x86_64__) || defined(_M_X64) +# include <xmmintrin.h> +/* Bit 15 of MXCSR flushes a denormal result to zero. The neighbouring + * denormals-are-zero bit is deliberately left alone: it only affects denormal + * inputs, which cannot arise here because samples come from a 16-bit integer + * divided by 32768 and every intermediate that could decay that far is already + * covered by flushing results. It is also absent on the earliest x86-64 parts, + * where writing it would fault. */ +# define MXCSR_FTZ 0x8000u +static inline void fp_scope_enter(fp_scope *s, int enable) { + s->active = enable; + if (!enable) + return; + s->saved = _mm_getcsr(); + _mm_setcsr((unsigned)s->saved | MXCSR_FTZ); +} +static inline void fp_scope_leave(const fp_scope *s) { + if (s->active) + _mm_setcsr((unsigned)s->saved); +} +#else +static inline void fp_scope_enter(fp_scope *s, int enable) { + s->saved = 0; + s->active = enable; + (void)s->active; +} +static inline void fp_scope_leave(const fp_scope *s) { (void)s; } +#endif + +/* ---- pointwise ------------------------------------------------------- */ + +static inline float sigmoidf_(float x) { return 1.0f / (1.0f + expf(-x)); } + +static void relu_(float *p, int n) { + for (int i = 0; i < n; ++i) + if (!(p[i] > 0.0f)) + p[i] = 0.0f; +} + +/* ---- forward pass ---------------------------------------------------- */ + +static void stft(svad_t *v) { + const svad_weights *w = v->w; + const int bp = w->bins_pad; + float *rows = v->mag + bp; /* skip the leading guard row */ + + if (v->use_fft) { + svad_rfft_mag(&v->fft, v->xp, w->hop, w->frames, w->stft_window, rows, bp, v->fft_scratch); + return; + } + /* Every frame shares the basis, so one pass over it covers all of them. */ + const int stride = 2 * w->bins; + v->k->matvec(v->spec, stride, w->stft_basis, v->xp, w->hop, NULL, stride, w->n_fft, w->frames); + for (int t = 0; t < w->frames; ++t) { + const float *src = v->spec + (size_t)t * stride; + float *dst = rows + (size_t)t * bp; + for (int f = 0; f < w->bins; ++f) { + const float re = src[f], im = src[w->bins + f]; + dst[f] = sqrtf(re * re + im * im); + } + } +} + +/* Each encoder stage runs all its output time steps in a single call, so the + * weights are streamed once per chunk rather than once per time step. Output + * step s reads the window starting at input row s * stride and writes row + * s + 1, leaving the guard rows untouched. */ +static float forward(svad_t *v) { + const svad_weights *w = v->w; + const svad_kernels *k = v->k; + const int bp = w->bins_pad; + + stft(v); + + k->matvec(v->e1 + SVAD_C1, SVAD_C1, w->conv1_w, v->mag, bp, w->conv1_b, SVAD_C1, 3 * bp, T1); + relu_(v->e1 + SVAD_C1, T1 * SVAD_C1); + + k->matvec(v->e2 + SVAD_C2, SVAD_C2, w->conv2_w, v->e1, 2 * SVAD_C1, w->conv2_b, SVAD_C2, + 3 * SVAD_C1, T2); + relu_(v->e2 + SVAD_C2, T2 * SVAD_C2); + + k->matvec(v->e3 + SVAD_C3, 0, w->conv3_w, v->e2, 0, w->conv3_b, SVAD_C3, 3 * SVAD_C2, T3); + relu_(v->e3 + SVAD_C3, T3 * SVAD_C3); + + /* conv4 writes the LSTM input directly; the hidden half follows it. */ + k->matvec(v->lstm_in, 0, w->conv4_w, v->e3, 0, w->conv4_b, SVAD_C4, 3 * SVAD_C3, T4); + relu_(v->lstm_in, SVAD_C4); + memcpy(v->lstm_in + SVAD_HID, v->h, sizeof v->h); + + k->matvec(v->gates, 0, w->lstm_w, v->lstm_in, 0, w->lstm_b, SVAD_GATES, 2 * SVAD_HID, 1); + for (int i = 0; i < SVAD_HID; ++i) { + const float ig = sigmoidf_(v->gates[i]); + const float fg = sigmoidf_(v->gates[SVAD_HID + i]); + const float gg = tanhf(v->gates[2 * SVAD_HID + i]); + const float og = sigmoidf_(v->gates[3 * SVAD_HID + i]); + const float cn = fg * v->c[i] + ig * gg; + const float hn = og * tanhf(cn); + v->c[i] = cn; + v->h[i] = hn; + v->hr[i] = hn > 0.0f ? hn : 0.0f; + } + return sigmoidf_(k->dot(w->head_w, v->hr, SVAD_HID) + w->head_b); +} + +/* ---- lifecycle ------------------------------------------------------- */ + +svad_t *svad_create_ex(const char *path, int sample_rate, unsigned flags) { + const svad_weights *builtin = svad_weights_builtin(sample_rate); + if (!builtin) + return NULL; + + svad_t *v = NULL; + if (posix_memalign((void **)&v, 64, sizeof *v) != 0 || !v) + return NULL; + memset(v, 0, sizeof *v); /* zeroes the guard rows and the channel padding */ + + if (path) { + const int verify = !(flags & SVAD_NO_VERIFY); + if (svad_weights_map(path, sample_rate, verify, &v->mapped, &v->map, &v->map_len) != 0) { + free(v); + return NULL; + } + v->w = &v->mapped; + } else { + v->w = builtin; + } + + const int scalar = (flags & SVAD_SCALAR_KERNELS) != 0; + v->k = scalar ? svad_kernels_scalar() : svad_kernels_simd(); + v->use_fft = (flags & SVAD_DENSE_STFT) == 0; + v->ftz = (flags & SVAD_NO_FTZ) == 0; + svad_rfft_init(&v->fft, v->w->n_fft, !scalar); + snprintf(v->backend, sizeof v->backend, "%s+%s", v->k->name, v->use_fft ? "rfft" : "dense"); + + svad_reset(v); + return v; +} + +svad_t *svad_create(int sample_rate) { return svad_create_ex(NULL, sample_rate, 0); } + +svad_t *svad_create_from_file(const char *path, int sample_rate) { + return svad_create_ex(path, sample_rate, 0); +} + +void svad_destroy(svad_t *v) { + if (!v) + return; + svad_weights_unmap(v->map, v->map_len); + free(v); +} + +void svad_reset(svad_t *v) { + memset(v->h, 0, sizeof v->h); + memset(v->c, 0, sizeof v->c); + memset(v->xp, 0, (size_t)v->w->context * sizeof(float)); +} + +int svad_chunk_samples(const svad_t *v) { return v->w->chunk; } + +int svad_sample_rate(const svad_t *v) { return v->w->sample_rate; } + +const char *svad_backend_name(const svad_t *v) { return v->backend; } + +void svad_get_state(const svad_t *v, float *out) { + memcpy(out, v->h, sizeof v->h); + memcpy(out + SVAD_HID, v->c, sizeof v->c); +} + +void svad_set_state(svad_t *v, const float *in) { + memcpy(v->h, in, sizeof v->h); + memcpy(v->c, in + SVAD_HID, sizeof v->c); +} + +/* ---- per-chunk entry points ------------------------------------------ */ + +/* Right-side reflect pad only: xp[total + i] = xp[total - 2 - i]. A symmetric + * or left-side pad yields plausible but wrong probabilities. */ +static void reflect_pad(svad_t *v) { + const int total = v->w->context + v->w->chunk; + for (int i = 0; i < v->w->pad; ++i) + v->xp[total + i] = v->xp[total - 2 - i]; +} + +/* The next chunk's context is the tail of this chunk's model input. */ +static void carry_context(svad_t *v) { + memmove(v->xp, v->xp + v->w->chunk, (size_t)v->w->context * sizeof(float)); +} + +float svad_process_f32(svad_t *v, const float *chunk) { + fp_scope fp; + fp_scope_enter(&fp, v->ftz); + memcpy(v->xp + v->w->context, chunk, (size_t)v->w->chunk * sizeof(float)); + reflect_pad(v); + const float p = forward(v); + carry_context(v); + fp_scope_leave(&fp); + return p; +} + +float svad_process_s16(svad_t *v, const int16_t *chunk) { + fp_scope fp; + fp_scope_enter(&fp, v->ftz); + float *dst = v->xp + v->w->context; + for (int i = 0; i < v->w->chunk; ++i) + dst[i] = (float)chunk[i] * (1.0f / 32768.0f); + reflect_pad(v); + const float p = forward(v); + carry_context(v); + fp_scope_leave(&fp); + return p; +} |
