aboutsummaryrefslogtreecommitdiff
path: root/svad.c
diff options
context:
space:
mode:
Diffstat (limited to 'svad.c')
-rw-r--r--svad.c300
1 files changed, 300 insertions, 0 deletions
diff --git a/svad.c b/svad.c
new file mode 100644
index 0000000..6b6d993
--- /dev/null
+++ b/svad.c
@@ -0,0 +1,300 @@
+/* silero-vad v6.2 forward pass.
+ *
+ * Per chunk: reflect-pad, four windowed transforms, four convolutions with
+ * ReLU, one LSTM cell, one logistic head. Every buffer below is allocated once
+ * in svad_create and reused, so a chunk costs no allocation, no syscall and no
+ * lock. The whole working set is about 16 KiB of activations plus a megabyte
+ * of read-only weights.
+ *
+ * Activation buffers carry a zero guard row before the first time step and
+ * after the last one. A kernel-3, pad-1 convolution therefore reads three
+ * consecutive rows starting at row (stride * output_step), which is one
+ * contiguous span of memory, and the whole encoder reduces to the same matvec
+ * the LSTM and the head use. */
+
+#include "svad.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "kernels.h"
+#include "rfft.h"
+#include "weights.h"
+
+/* Time steps produced by each encoder stage; fixed because the transform
+ * always yields SVAD_FRAMES frames and the strides are 1, 2, 2, 1. */
+#define T1 SVAD_FRAMES /* 4 */
+#define T2 (SVAD_FRAMES / 2) /* 2 */
+#define T3 1
+#define T4 1
+
+/* Rows including the two guard rows. */
+#define ROWS(t) ((t) + 2)
+
+struct svad {
+ const svad_weights *w;
+ const svad_kernels *k;
+ svad_rfft fft;
+ int use_fft;
+ int ftz;
+ char backend[32];
+
+ void *map;
+ size_t map_len;
+ svad_weights mapped;
+
+ SVAD_ALIGN64 float h[SVAD_HID];
+ SVAD_ALIGN64 float c[SVAD_HID];
+
+ SVAD_ALIGN64 float xp[SVAD_MAX_XP];
+ SVAD_ALIGN64 float mag[ROWS(T1) * SVAD_MAX_BINS_PAD];
+ SVAD_ALIGN64 float e1[ROWS(T1) * SVAD_C1];
+ SVAD_ALIGN64 float e2[ROWS(T2) * SVAD_C2];
+ SVAD_ALIGN64 float e3[ROWS(T3) * SVAD_C3];
+ SVAD_ALIGN64 float lstm_in[2 * SVAD_HID];
+ SVAD_ALIGN64 float gates[SVAD_GATES];
+ SVAD_ALIGN64 float hr[SVAD_HID];
+ SVAD_ALIGN64 float spec[SVAD_FRAMES * 2 * SVAD_MAX_BINS];
+ SVAD_ALIGN64 float fft_scratch[SVAD_RFFT_SCRATCH];
+};
+
+/* ---- denormal handling ------------------------------------------------
+ *
+ * The LSTM cell state decays toward zero across long silences, and denormal
+ * operands are slow on several targets. Flushing them is scoped to the call so
+ * the library never leaves the caller's floating point mode altered. */
+
+typedef struct {
+ unsigned long long saved;
+ int active;
+} fp_scope;
+
+#if defined(__aarch64__)
+# define FPCR_FZ (1ull << 24)
+static inline void fp_scope_enter(fp_scope *s, int enable) {
+ s->active = enable;
+ if (!enable)
+ return;
+ unsigned long long r;
+ __asm__ __volatile__("mrs %0, fpcr" : "=r"(r));
+ s->saved = r;
+ if (!(r & FPCR_FZ))
+ __asm__ __volatile__("msr fpcr, %0" : : "r"(r | FPCR_FZ));
+}
+static inline void fp_scope_leave(const fp_scope *s) {
+ if (s->active)
+ __asm__ __volatile__("msr fpcr, %0" : : "r"(s->saved));
+}
+#elif defined(__SSE2__) || defined(__x86_64__) || defined(_M_X64)
+# include <xmmintrin.h>
+/* Bit 15 of MXCSR flushes a denormal result to zero. The neighbouring
+ * denormals-are-zero bit is deliberately left alone: it only affects denormal
+ * inputs, which cannot arise here because samples come from a 16-bit integer
+ * divided by 32768 and every intermediate that could decay that far is already
+ * covered by flushing results. It is also absent on the earliest x86-64 parts,
+ * where writing it would fault. */
+# define MXCSR_FTZ 0x8000u
+static inline void fp_scope_enter(fp_scope *s, int enable) {
+ s->active = enable;
+ if (!enable)
+ return;
+ s->saved = _mm_getcsr();
+ _mm_setcsr((unsigned)s->saved | MXCSR_FTZ);
+}
+static inline void fp_scope_leave(const fp_scope *s) {
+ if (s->active)
+ _mm_setcsr((unsigned)s->saved);
+}
+#else
+static inline void fp_scope_enter(fp_scope *s, int enable) {
+ s->saved = 0;
+ s->active = enable;
+ (void)s->active;
+}
+static inline void fp_scope_leave(const fp_scope *s) { (void)s; }
+#endif
+
+/* ---- pointwise ------------------------------------------------------- */
+
+static inline float sigmoidf_(float x) { return 1.0f / (1.0f + expf(-x)); }
+
+static void relu_(float *p, int n) {
+ for (int i = 0; i < n; ++i)
+ if (!(p[i] > 0.0f))
+ p[i] = 0.0f;
+}
+
+/* ---- forward pass ---------------------------------------------------- */
+
+static void stft(svad_t *v) {
+ const svad_weights *w = v->w;
+ const int bp = w->bins_pad;
+ float *rows = v->mag + bp; /* skip the leading guard row */
+
+ if (v->use_fft) {
+ svad_rfft_mag(&v->fft, v->xp, w->hop, w->frames, w->stft_window, rows, bp, v->fft_scratch);
+ return;
+ }
+ /* Every frame shares the basis, so one pass over it covers all of them. */
+ const int stride = 2 * w->bins;
+ v->k->matvec(v->spec, stride, w->stft_basis, v->xp, w->hop, NULL, stride, w->n_fft, w->frames);
+ for (int t = 0; t < w->frames; ++t) {
+ const float *src = v->spec + (size_t)t * stride;
+ float *dst = rows + (size_t)t * bp;
+ for (int f = 0; f < w->bins; ++f) {
+ const float re = src[f], im = src[w->bins + f];
+ dst[f] = sqrtf(re * re + im * im);
+ }
+ }
+}
+
+/* Each encoder stage runs all its output time steps in a single call, so the
+ * weights are streamed once per chunk rather than once per time step. Output
+ * step s reads the window starting at input row s * stride and writes row
+ * s + 1, leaving the guard rows untouched. */
+static float forward(svad_t *v) {
+ const svad_weights *w = v->w;
+ const svad_kernels *k = v->k;
+ const int bp = w->bins_pad;
+
+ stft(v);
+
+ k->matvec(v->e1 + SVAD_C1, SVAD_C1, w->conv1_w, v->mag, bp, w->conv1_b, SVAD_C1, 3 * bp, T1);
+ relu_(v->e1 + SVAD_C1, T1 * SVAD_C1);
+
+ k->matvec(v->e2 + SVAD_C2, SVAD_C2, w->conv2_w, v->e1, 2 * SVAD_C1, w->conv2_b, SVAD_C2,
+ 3 * SVAD_C1, T2);
+ relu_(v->e2 + SVAD_C2, T2 * SVAD_C2);
+
+ k->matvec(v->e3 + SVAD_C3, 0, w->conv3_w, v->e2, 0, w->conv3_b, SVAD_C3, 3 * SVAD_C2, T3);
+ relu_(v->e3 + SVAD_C3, T3 * SVAD_C3);
+
+ /* conv4 writes the LSTM input directly; the hidden half follows it. */
+ k->matvec(v->lstm_in, 0, w->conv4_w, v->e3, 0, w->conv4_b, SVAD_C4, 3 * SVAD_C3, T4);
+ relu_(v->lstm_in, SVAD_C4);
+ memcpy(v->lstm_in + SVAD_HID, v->h, sizeof v->h);
+
+ k->matvec(v->gates, 0, w->lstm_w, v->lstm_in, 0, w->lstm_b, SVAD_GATES, 2 * SVAD_HID, 1);
+ for (int i = 0; i < SVAD_HID; ++i) {
+ const float ig = sigmoidf_(v->gates[i]);
+ const float fg = sigmoidf_(v->gates[SVAD_HID + i]);
+ const float gg = tanhf(v->gates[2 * SVAD_HID + i]);
+ const float og = sigmoidf_(v->gates[3 * SVAD_HID + i]);
+ const float cn = fg * v->c[i] + ig * gg;
+ const float hn = og * tanhf(cn);
+ v->c[i] = cn;
+ v->h[i] = hn;
+ v->hr[i] = hn > 0.0f ? hn : 0.0f;
+ }
+ return sigmoidf_(k->dot(w->head_w, v->hr, SVAD_HID) + w->head_b);
+}
+
+/* ---- lifecycle ------------------------------------------------------- */
+
+svad_t *svad_create_ex(const char *path, int sample_rate, unsigned flags) {
+ const svad_weights *builtin = svad_weights_builtin(sample_rate);
+ if (!builtin)
+ return NULL;
+
+ svad_t *v = NULL;
+ if (posix_memalign((void **)&v, 64, sizeof *v) != 0 || !v)
+ return NULL;
+ memset(v, 0, sizeof *v); /* zeroes the guard rows and the channel padding */
+
+ if (path) {
+ const int verify = !(flags & SVAD_NO_VERIFY);
+ if (svad_weights_map(path, sample_rate, verify, &v->mapped, &v->map, &v->map_len) != 0) {
+ free(v);
+ return NULL;
+ }
+ v->w = &v->mapped;
+ } else {
+ v->w = builtin;
+ }
+
+ const int scalar = (flags & SVAD_SCALAR_KERNELS) != 0;
+ v->k = scalar ? svad_kernels_scalar() : svad_kernels_simd();
+ v->use_fft = (flags & SVAD_DENSE_STFT) == 0;
+ v->ftz = (flags & SVAD_NO_FTZ) == 0;
+ svad_rfft_init(&v->fft, v->w->n_fft, !scalar);
+ snprintf(v->backend, sizeof v->backend, "%s+%s", v->k->name, v->use_fft ? "rfft" : "dense");
+
+ svad_reset(v);
+ return v;
+}
+
+svad_t *svad_create(int sample_rate) { return svad_create_ex(NULL, sample_rate, 0); }
+
+svad_t *svad_create_from_file(const char *path, int sample_rate) {
+ return svad_create_ex(path, sample_rate, 0);
+}
+
+void svad_destroy(svad_t *v) {
+ if (!v)
+ return;
+ svad_weights_unmap(v->map, v->map_len);
+ free(v);
+}
+
+void svad_reset(svad_t *v) {
+ memset(v->h, 0, sizeof v->h);
+ memset(v->c, 0, sizeof v->c);
+ memset(v->xp, 0, (size_t)v->w->context * sizeof(float));
+}
+
+int svad_chunk_samples(const svad_t *v) { return v->w->chunk; }
+
+int svad_sample_rate(const svad_t *v) { return v->w->sample_rate; }
+
+const char *svad_backend_name(const svad_t *v) { return v->backend; }
+
+void svad_get_state(const svad_t *v, float *out) {
+ memcpy(out, v->h, sizeof v->h);
+ memcpy(out + SVAD_HID, v->c, sizeof v->c);
+}
+
+void svad_set_state(svad_t *v, const float *in) {
+ memcpy(v->h, in, sizeof v->h);
+ memcpy(v->c, in + SVAD_HID, sizeof v->c);
+}
+
+/* ---- per-chunk entry points ------------------------------------------ */
+
+/* Right-side reflect pad only: xp[total + i] = xp[total - 2 - i]. A symmetric
+ * or left-side pad yields plausible but wrong probabilities. */
+static void reflect_pad(svad_t *v) {
+ const int total = v->w->context + v->w->chunk;
+ for (int i = 0; i < v->w->pad; ++i)
+ v->xp[total + i] = v->xp[total - 2 - i];
+}
+
+/* The next chunk's context is the tail of this chunk's model input. */
+static void carry_context(svad_t *v) {
+ memmove(v->xp, v->xp + v->w->chunk, (size_t)v->w->context * sizeof(float));
+}
+
+float svad_process_f32(svad_t *v, const float *chunk) {
+ fp_scope fp;
+ fp_scope_enter(&fp, v->ftz);
+ memcpy(v->xp + v->w->context, chunk, (size_t)v->w->chunk * sizeof(float));
+ reflect_pad(v);
+ const float p = forward(v);
+ carry_context(v);
+ fp_scope_leave(&fp);
+ return p;
+}
+
+float svad_process_s16(svad_t *v, const int16_t *chunk) {
+ fp_scope fp;
+ fp_scope_enter(&fp, v->ftz);
+ float *dst = v->xp + v->w->context;
+ for (int i = 0; i < v->w->chunk; ++i)
+ dst[i] = (float)chunk[i] * (1.0f / 32768.0f);
+ reflect_pad(v);
+ const float p = forward(v);
+ carry_context(v);
+ fp_scope_leave(&fp);
+ return p;
+}