From 6652fcbc653d84e75cb5c08c5f5edf52263ed50f Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 20 Jun 2026 20:36:55 +0200 Subject: tests: upgrade unit tests and add integration/parity tests --- tests/bert_dump.c | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 tests/bert_dump.c (limited to 'tests/bert_dump.c') diff --git a/tests/bert_dump.c b/tests/bert_dump.c new file mode 100644 index 0000000..48bbab1 --- /dev/null +++ b/tests/bert_dump.c @@ -0,0 +1,65 @@ +/* Parity driver for the BERT-family forward. Loads a sentence-transformers + * checkpoint, wraps each argument with [CLS]/[SEP], runs the BERT forward + * (mean pool + L2 normalize inside ffwd_model_encode_into), and prints + * " ..." per line. check_bert_parity.py builds this with + * `make parity-bert-driver` and compares against SentenceTransformer.encode. + * This one needs the real model weights, so it stays a manual --model-dir + * check rather than a hermetic stored-vector test. */ +#include "internal.h" +#include "wordpiece.h" + +#include +#include +#include + +int main(int argc, char **argv) { + if (argc < 3) { + fprintf(stderr, "usage: %s MODEL_DIR TEXT...\n", argv[0]); + return 2; + } + const char *dir = argv[1]; + tok_wp_t *tok = tok_wp_load(dir); + ffwd_model_t *model = ffwd_model_load(dir); + if (!tok || !model) { + fprintf(stderr, "load failed\n"); + return 1; + } + ffwd_workspace_t *ws = ffwd_workspace_new(model); + const ffwd_config_t *cfg = ffwd_model_config(model); + int dim = cfg->hidden_size; + int cls = tok_wp_token_id(tok, "[CLS]"); + int sep = tok_wp_token_id(tok, "[SEP]"); + if (!ws || cls < 0 || sep < 0) { + fprintf(stderr, "setup failed\n"); + return 1; + } + for (int a = 2; a < argc; a++) { + int n = 0; + int *ids = tok_wp_encode(tok, argv[a], &n); + if (!ids && n != 0) { + fprintf(stderr, "tokenize failed\n"); + return 1; + } + int *full = (int *)malloc((size_t)(n + 2) * sizeof(int)); + full[0] = cls; + for (int i = 0; i < n; i++) + full[1 + i] = ids[i]; + full[n + 1] = sep; + float *emb = (float *)malloc((size_t)dim * sizeof(float)); + if (ffwd_model_encode_into(model, ws, full, n + 2, emb) != 0) { + fprintf(stderr, "embed failed\n"); + return 1; + } + printf("%d", dim); + for (int i = 0; i < dim; i++) + printf(" %.7g", emb[i]); + printf("\n"); + free(ids); + free(full); + free(emb); + } + ffwd_workspace_free(ws); + ffwd_model_free(model); + tok_wp_free(tok); + return 0; +} -- cgit v1.3.1