aboutsummaryrefslogtreecommitdiff
path: root/tests/contextual_batch.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:36:55 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commit6652fcbc653d84e75cb5c08c5f5edf52263ed50f (patch)
tree1ccffa48e1dc86c59c61dcd4672ffedcf1e0c873 /tests/contextual_batch.c
parentdbdf714bb57578431924e5d069089bdfd9f4c604 (diff)
downloadffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.tar.gz
ffwd-6652fcbc653d84e75cb5c08c5f5edf52263ed50f.zip
tests: upgrade unit tests and add integration/parity tests
Diffstat (limited to 'tests/contextual_batch.c')
-rw-r--r--tests/contextual_batch.c329
1 files changed, 329 insertions, 0 deletions
diff --git a/tests/contextual_batch.c b/tests/contextual_batch.c
new file mode 100644
index 0000000..4f17699
--- /dev/null
+++ b/tests/contextual_batch.c
@@ -0,0 +1,329 @@
+/* Parity driver for contextual span embeddings: compares singleton encoding
+ * (one document at a time) against the batched path, for the same documents.
+ * No external library is involved; it checks the C code against itself. With a
+ * positive run count it also times the batched path. check_contextual_batch_parity.py
+ * builds this with `make parity-context-driver` (or parity-context-driver-mlx)
+ * and passes the model dir plus tolerances as positional arguments. */
+#include "internal.h"
+#ifdef CHECK_MLX
+# include "mlx.h"
+#endif
+#include "bpe.h"
+
+#include <math.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <time.h>
+
+typedef struct {
+ int *ids;
+ ffwd_span_t *spans;
+ int n_spans;
+ int n_tokens;
+} test_doc_t;
+
+#ifdef CHECK_MLX
+typedef ffwd_mlx_ctx_t test_model_t;
+typedef void test_workspace_t;
+static int g_mlx_quantize_bits = 0;
+static int g_mlx_quantize_group_size = 64;
+static test_model_t *test_model_load(const char *dir) {
+ if (g_mlx_quantize_bits) {
+ ffwd_mlx_options_t opts = {
+ .quantize_bits = g_mlx_quantize_bits,
+ .quantize_group_size = g_mlx_quantize_group_size,
+ };
+ return ffwd_mlx_load_with_options(dir, &opts);
+ }
+ return ffwd_mlx_load(dir);
+}
+static void test_model_free(test_model_t *m) { ffwd_mlx_free(m); }
+static test_workspace_t *test_workspace_new(test_model_t *m) { return m; }
+static void test_workspace_free(test_workspace_t *ws) { (void)ws; }
+static const ffwd_config_t *test_config(test_model_t *m) { return ffwd_mlx_config(m); }
+static int
+test_ffwd_spans(test_model_t *m, test_workspace_t *ws, const test_doc_t *doc, float *out) {
+ (void)ws;
+ return ffwd_mlx_encode_spans(m, doc->ids, doc->n_tokens, doc->spans, doc->n_spans, out);
+}
+static int test_ffwd_spans_batch(test_model_t *m,
+ test_workspace_t *ws,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out) {
+ (void)ws;
+ return ffwd_mlx_encode_spans_batch(m, inputs, batch, out);
+}
+#else
+typedef ffwd_model_t test_model_t;
+typedef ffwd_workspace_t test_workspace_t;
+static test_model_t *test_model_load(const char *dir) { return ffwd_model_load(dir); }
+static void test_model_free(test_model_t *m) { ffwd_model_free(m); }
+static test_workspace_t *test_workspace_new(test_model_t *m) { return ffwd_workspace_new(m); }
+static void test_workspace_free(test_workspace_t *ws) { ffwd_workspace_free(ws); }
+static const ffwd_config_t *test_config(test_model_t *m) { return ffwd_model_config(m); }
+static int
+test_ffwd_spans(test_model_t *m, test_workspace_t *ws, const test_doc_t *doc, float *out) {
+ return ffwd_model_encode_spans(m, ws, doc->ids, doc->n_tokens, doc->spans, doc->n_spans, out);
+}
+static int test_ffwd_spans_batch(test_model_t *m,
+ test_workspace_t *ws,
+ const ffwd_context_input_t *inputs,
+ int batch,
+ float *out) {
+ return ffwd_model_encode_spans_batch(m, ws, inputs, batch, out);
+}
+#endif
+
+static int
+init_doc(test_doc_t *doc, tok_bpe_t *tok, const char **chunks, int n_chunks, int separator_id) {
+ int **ids = calloc((size_t)n_chunks, sizeof(*ids));
+ int *lengths = calloc((size_t)n_chunks, sizeof(*lengths));
+ doc->spans = calloc((size_t)n_chunks, sizeof(*doc->spans));
+ if (!ids || !lengths || !doc->spans)
+ return -1;
+ doc->n_spans = n_chunks;
+ doc->n_tokens = n_chunks > 1 ? n_chunks - 1 : 0;
+ for (int i = 0; i < n_chunks; i++) {
+ ids[i] = tok_bpe_encode(tok, chunks[i], &lengths[i]);
+ if (!ids[i] || lengths[i] <= 0)
+ return -1;
+ doc->n_tokens += lengths[i];
+ }
+ doc->ids = malloc((size_t)doc->n_tokens * sizeof(*doc->ids));
+ if (!doc->ids)
+ return -1;
+ int pos = 0;
+ for (int i = 0; i < n_chunks; i++) {
+ doc->spans[i].start = pos;
+ doc->spans[i].n_tokens = lengths[i];
+ memcpy(doc->ids + pos, ids[i], (size_t)lengths[i] * sizeof(*doc->ids));
+ pos += lengths[i];
+ if (i + 1 < n_chunks) {
+ doc->ids[pos++] = separator_id;
+ }
+ free(ids[i]);
+ }
+ free(lengths);
+ free(ids);
+ return 0;
+}
+
+static void free_doc(test_doc_t *doc) {
+ free(doc->ids);
+ free(doc->spans);
+}
+
+static float max_abs_diff(const float *a, const float *b, int n) {
+ float worst = 0.0f;
+ for (int i = 0; i < n; i++) {
+ float diff = fabsf(a[i] - b[i]);
+ if (diff > worst)
+ worst = diff;
+ }
+ return worst;
+}
+
+static double now_ms(void) {
+ struct timespec ts;
+ clock_gettime(CLOCK_MONOTONIC, &ts);
+ return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1000000.0;
+}
+
+static int cmp_double(const void *a, const void *b) {
+ double aa = *(const double *)a;
+ double bb = *(const double *)b;
+ return (aa > bb) - (aa < bb);
+}
+
+static char *make_synthetic_chunk(int doc, int chunk, int repeats) {
+ const char *base = "Contextual embedding inference processes complete documents before "
+ "pooling chunk spans. This deterministic text stresses long "
+ "bidirectional attention and compact span result handling. ";
+ if (repeats < 1)
+ repeats = 1;
+ size_t base_len = strlen(base);
+ size_t cap = 128 + (size_t)repeats * base_len;
+ char *s = malloc(cap);
+ if (!s)
+ return NULL;
+ int n = snprintf(s, cap, "document %d chunk %d: ", doc, chunk);
+ if (n < 0 || (size_t)n >= cap) {
+ free(s);
+ return NULL;
+ }
+ size_t off = (size_t)n;
+ for (int i = 0; i < repeats; i++) {
+ if (off + base_len + 1 > cap) {
+ free(s);
+ return NULL;
+ }
+ memcpy(s + off, base, base_len);
+ off += base_len;
+ }
+ s[off] = '\0';
+ return s;
+}
+
+int main(int argc, char **argv) {
+ if (argc < 2 || argc > 11)
+ return 2;
+ int runs = argc >= 3 ? atoi(argv[2]) : 0;
+ float max_allowed_diff = argc >= 4 ? strtof(argv[3], NULL) : 0.00005f;
+ float min_allowed_cosine = argc >= 5 ? strtof(argv[4], NULL) : 0.99999f;
+#ifdef CHECK_MLX
+ g_mlx_quantize_bits = argc >= 6 ? atoi(argv[5]) : 0;
+ g_mlx_quantize_group_size = argc >= 7 ? atoi(argv[6]) : 64;
+ if (g_mlx_quantize_bits != 0 && g_mlx_quantize_bits != 8)
+ return 2;
+ if (g_mlx_quantize_group_size <= 0)
+ return 2;
+#endif
+ int synthetic_docs = argc >= 8 ? atoi(argv[7]) : 0;
+ int synthetic_chunks = argc >= 9 ? atoi(argv[8]) : 0;
+ int synthetic_repeats = argc >= 10 ? atoi(argv[9]) : 1;
+ int synthetic_ragged = argc >= 11 ? atoi(argv[10]) : 0;
+ if (runs < 0)
+ return 2;
+ if (synthetic_docs < 0 || synthetic_chunks < 0 || synthetic_repeats < 0)
+ return 2;
+ char vocab_path[4096];
+ snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", argv[1]);
+
+ test_model_t *model = test_model_load(argv[1]);
+ test_workspace_t *ws = test_workspace_new(model);
+ tok_bpe_t *tok = tok_bpe_load(vocab_path);
+ if (!model || !ws || !tok) {
+ fprintf(stderr, "failed to initialize model or tokenizer\n");
+ return 1;
+ }
+ int separator_id = FFWD_CONTEXT_SEPARATOR_TOKEN_ID;
+
+ int batch = synthetic_docs > 0 ? synthetic_docs : 3;
+ int total_spans = 0;
+ test_doc_t *docs = calloc((size_t)batch, sizeof(*docs));
+ ffwd_context_input_t *inputs = calloc((size_t)batch, sizeof(*inputs));
+ if (!docs || !inputs)
+ return 1;
+
+ if (synthetic_docs > 0) {
+ if (synthetic_chunks <= 0)
+ return 2;
+ for (int i = 0; i < batch; i++) {
+ int doc_chunks = synthetic_ragged ? synthetic_chunks * (i + 1) : synthetic_chunks;
+ char **texts = calloc((size_t)doc_chunks, sizeof(*texts));
+ if (!texts)
+ return 1;
+ int ok = 1;
+ for (int c = 0; c < doc_chunks; c++) {
+ texts[c] = make_synthetic_chunk(i, c, synthetic_repeats);
+ if (!texts[c])
+ ok = 0;
+ }
+ if (!ok || init_doc(&docs[i], tok, (const char **)texts, doc_chunks, separator_id) != 0) {
+ fprintf(stderr, "failed to prepare synthetic document %d\n", i);
+ for (int c = 0; c < doc_chunks; c++)
+ free(texts[c]);
+ free(texts);
+ return 1;
+ }
+ for (int c = 0; c < doc_chunks; c++)
+ free(texts[c]);
+ free(texts);
+ total_spans += docs[i].n_spans;
+ }
+ } else {
+ const char *doc0[] = {"Python is useful for scripting.",
+ "Paris is the capital city of France."};
+ const char *doc1[] = {"Redis stores in-memory data structures.",
+ "SQLite stores relational data in a local file.",
+ "Neural networks produce useful representations for retrieval."};
+ const char *doc2[] = {"A deliberately short final document."};
+ const char **chunks[] = {doc0, doc1, doc2};
+ int chunk_counts[] = {2, 3, 1};
+ for (int i = 0; i < batch; i++) {
+ if (init_doc(&docs[i], tok, chunks[i], chunk_counts[i], separator_id) != 0) {
+ fprintf(stderr, "failed to prepare document %d\n", i);
+ return 1;
+ }
+ total_spans += docs[i].n_spans;
+ }
+ }
+
+ for (int i = 0; i < batch; i++) {
+ inputs[i].input.ids = docs[i].ids;
+ inputs[i].input.n_tokens = docs[i].n_tokens;
+ inputs[i].spans = docs[i].spans;
+ inputs[i].n_spans = docs[i].n_spans;
+ }
+
+ int hidden = test_config(model)->hidden_size;
+ float *expected = calloc((size_t)total_spans * hidden, sizeof(*expected));
+ float *actual = calloc((size_t)total_spans * hidden, sizeof(*actual));
+ if (!expected || !actual)
+ return 1;
+
+ int span_offset = 0;
+ for (int i = 0; i < batch; i++) {
+ if (test_ffwd_spans(model, ws, &docs[i], expected + (size_t)span_offset * hidden) != 0) {
+ fprintf(stderr, "singleton contextual embedding failed at %d\n", i);
+ return 1;
+ }
+ span_offset += docs[i].n_spans;
+ }
+ if (test_ffwd_spans_batch(model, ws, inputs, batch, actual) != 0) {
+ fprintf(stderr, "batched contextual embedding failed\n");
+ return 1;
+ }
+
+ float worst_diff = 0.0f;
+ float worst_cosine = 1.0f;
+ for (int i = 0; i < total_spans; i++) {
+ float *want = expected + (size_t)i * hidden;
+ float *got = actual + (size_t)i * hidden;
+ float diff = max_abs_diff(want, got, hidden);
+ float cosine = ffwd_cosine_similarity(want, got, hidden);
+ if (diff > worst_diff)
+ worst_diff = diff;
+ if (cosine < worst_cosine)
+ worst_cosine = cosine;
+ }
+
+ if (runs > 0) {
+ double *times = calloc((size_t)runs, sizeof(*times));
+ if (!times)
+ return 1;
+ for (int i = 0; i < runs; i++) {
+ double start = now_ms();
+ if (test_ffwd_spans_batch(model, ws, inputs, batch, actual) != 0) {
+ fprintf(stderr, "timed contextual batch failed at %d\n", i);
+ return 1;
+ }
+ times[i] = now_ms() - start;
+ }
+ qsort(times, (size_t)runs, sizeof(*times), cmp_double);
+ printf("benchmark: contextual batch docs=%d spans=%d runs=%d p50_ms=%.3f\n", batch,
+ total_spans, runs, times[runs / 2]);
+ free(times);
+ }
+
+ free(actual);
+ free(expected);
+ for (int i = 0; i < batch; i++)
+ free_doc(&docs[i]);
+ free(inputs);
+ free(docs);
+ tok_bpe_free(tok);
+ test_workspace_free(ws);
+ test_model_free(model);
+
+ if (worst_diff > max_allowed_diff || worst_cosine < min_allowed_cosine) {
+ fprintf(stderr, "contextual batch parity failed: max_abs_diff=%g cosine=%g\n", worst_diff,
+ worst_cosine);
+ return 1;
+ }
+ printf("ok: contextual batch parity docs=%d spans=%d max_abs_diff=%g cosine=%g\n", batch,
+ total_spans, worst_diff, worst_cosine);
+ return 0;
+}