From 3947e97fbf2a0b79c500ce925bcc9fe15f89899e Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Wed, 10 Jun 2026 12:51:10 +0200 Subject: tests: drive the HTTP request path in-process in test_server --- tests/test_server.c | 418 ++++++++++++++++++++++++++++++++++++++++++++++++- tests/test_tokenizer.c | 77 +-------- tests/tiny_model.h | 68 +++++--- tests/tok_fixture.h | 88 +++++++++++ 4 files changed, 548 insertions(+), 103 deletions(-) create mode 100644 tests/tok_fixture.h (limited to 'tests') diff --git a/tests/test_server.c b/tests/test_server.c index dcedd21..a2a8492 100644 --- a/tests/test_server.c +++ b/tests/test_server.c @@ -1,10 +1,19 @@ -/* tests/test_server.c - unit tests for embed_server.c static helpers. +/* tests/test_server.c - tests for embed_server.c. * ds4-style: includes the whole server translation unit with the real main - * compiled out, so statics are directly testable. Runs via `make test`. */ + * compiled out, so statics are directly testable. Unit-tests the encoding + * helpers, then boots the real server in-process (tiny 1024-dim hermetic + * model + byte-vocab tokenizer fixture) and drives the full HTTP request + * path over loopback sockets. Runs via `make test`. */ #define EMBED_SERVER_TEST #include "../embed_server.c" +#include "tiny_model.h" +#include "tok_fixture.h" + +#include +#include + static int test_failures = 0; #define TEST_ASSERT(cond) do { \ @@ -73,16 +82,419 @@ static void test_encode_embedding_binary(void) free(want); } +/* ==================================================================== + * HTTP request-path tests against an in-process server + * ==================================================================== */ + +#define TEST_API_KEY "tt-test-key" + +static int find_free_port(void) +{ + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) return -1; + struct sockaddr_in a; + memset(&a, 0, sizeof(a)); + a.sin_family = AF_INET; + a.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + socklen_t alen = sizeof(a); + if (bind(fd, (struct sockaddr *)&a, sizeof(a)) != 0 || + getsockname(fd, (struct sockaddr *)&a, &alen) != 0) { + close(fd); + return -1; + } + int port = ntohs(a.sin_port); + close(fd); + return port; +} + +static int tcp_connect(int port) +{ + int fd = socket(AF_INET, SOCK_STREAM, 0); + if (fd < 0) return -1; + struct sockaddr_in a; + memset(&a, 0, sizeof(a)); + a.sin_family = AF_INET; + a.sin_port = htons((uint16_t)port); + a.sin_addr.s_addr = htonl(INADDR_LOOPBACK); + if (connect(fd, (struct sockaddr *)&a, sizeof(a)) != 0) { + close(fd); + return -1; + } + return fd; +} + +static int send_all(int fd, const char *buf, size_t n) +{ + size_t off = 0; + while (off < n) { + ssize_t w = write(fd, buf + off, n - off); + if (w < 0) { + if (errno == EINTR) continue; + return -1; + } + off += (size_t)w; + } + return 0; +} + +/* Send one request; raw == NULL builds a normal request, otherwise `raw` + * bytes go on the wire verbatim (for malformed-HTTP checks). */ +static int http_send(int fd, const char *method, const char *path, + const char *auth, const char *body, const char *raw) +{ + if (raw) return send_all(fd, raw, strlen(raw)); + char head[512]; + size_t blen = body ? strlen(body) : 0; + int n = snprintf(head, sizeof(head), + "%s %s HTTP/1.1\r\n" + "Host: 127.0.0.1\r\n" + "%s%s%s" + "Content-Type: application/json\r\n" + "Content-Length: %zu\r\n" + "\r\n", + method, path, + auth ? "Authorization: Bearer " : "", + auth ? auth : "", auth ? "\r\n" : "", + blen); + if (n < 0 || (size_t)n >= sizeof(head)) return -1; + if (send_all(fd, head, (size_t)n) != 0) return -1; + return blen ? send_all(fd, body, blen) : 0; +} + +/* Read one response: returns the status code (or -1) and the body, which + * the caller frees. Reads exactly Content-Length body bytes so it works + * with keep-alive connections. */ +static int http_recv(int fd, char **out_body) +{ + *out_body = NULL; + size_t cap = 8192, len = 0; + char *buf = malloc(cap); + if (!buf) return -1; + char *hdr_end = NULL; + while (!hdr_end) { + if (len + 1 >= cap) { + cap *= 2; + char *nb = realloc(buf, cap); + if (!nb) { free(buf); return -1; } + buf = nb; + } + ssize_t r = read(fd, buf + len, cap - len - 1); + if (r <= 0) { free(buf); return -1; } + len += (size_t)r; + buf[len] = '\0'; + hdr_end = strstr(buf, "\r\n\r\n"); + } + + int status = -1; + if (sscanf(buf, "HTTP/1.1 %d", &status) != 1) { free(buf); return -1; } + size_t content_length = 0; + const char *cl = strstr(buf, "Content-Length:"); + if (cl && cl < hdr_end) content_length = strtoul(cl + 15, NULL, 10); + + size_t body_off = (size_t)(hdr_end - buf) + 4; + size_t need = body_off + content_length; + if (need + 1 > cap) { + char *nb = realloc(buf, need + 1); + if (!nb) { free(buf); return -1; } + buf = nb; + } + while (len < need) { + ssize_t r = read(fd, buf + len, need - len); + if (r <= 0) { free(buf); return -1; } + len += (size_t)r; + } + buf[need] = '\0'; + *out_body = strdup(buf + body_off); + free(buf); + return *out_body ? status : -1; +} + +/* One-shot request helper; returns status, body via out_body (caller frees, + * may be NULL when only the status matters). */ +static int http_req(int port, const char *method, const char *path, + const char *auth, const char *body, const char *raw, + char **out_body) +{ + int fd = tcp_connect(port); + if (fd < 0) return -1; + char *resp = NULL; + int status = -1; + if (http_send(fd, method, path, auth, body, raw) == 0) + status = http_recv(fd, &resp); + close(fd); + if (out_body) *out_body = resp; + else free(resp); + return status; +} + +typedef struct { + pplx_server_model_spec_t spec; + pplx_server_config_t cfg; + int rc; +} srv_ctx; + +static void *srv_thread_main(void *arg) +{ + srv_ctx *s = (srv_ctx *)arg; + s->rc = pplx_run_server(&s->cfg); + return NULL; +} + +static cJSON *parse_embeddings_body(const char *body, int want_items) +{ + cJSON *root = cJSON_Parse(body); + TEST_ASSERT(root != NULL); + if (!root) return NULL; + cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data"); + TEST_ASSERT(cJSON_IsArray(data)); + TEST_ASSERT(cJSON_GetArraySize(data) == want_items); + if (!cJSON_IsArray(data) || cJSON_GetArraySize(data) != want_items) { + cJSON_Delete(root); + return NULL; + } + return root; +} + +static void test_http_embeddings(int port) +{ + char *body = NULL; + + /* CORS preflight and routing. */ + TEST_ASSERT(http_req(port, "OPTIONS", "/v1/embeddings", NULL, NULL, NULL, + NULL) == 204); + TEST_ASSERT(http_req(port, "GET", "/v1/embeddings", TEST_API_KEY, NULL, + NULL, NULL) == 404); + TEST_ASSERT(http_req(port, "POST", "/nope", TEST_API_KEY, "{}", NULL, + NULL) == 404); + + /* Auth: required, and the exact key. */ + const char *ok_req = + "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hello world\"," + "\"encoding_format\":\"float\"}"; + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", NULL, ok_req, NULL, + NULL) == 401); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", "wrong-key", ok_req, + NULL, NULL) == 401); + + /* Malformed HTTP (400) and malformed JSON (422 detail). */ + TEST_ASSERT(http_req(port, NULL, NULL, NULL, NULL, + "GARBAGE\r\n\r\n", NULL) == 400); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{nope", NULL, NULL) == 422); + + /* Validation: missing/unknown model, wrong endpoint family, valid but + * unloaded id, bad encoding enum, dimensions out of range; the + * API answers validation errors with 422 detail arrays. */ + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"input\":\"hi\"}", NULL, NULL) == 422); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"foo\",\"input\":\"hi\"}", NULL, + NULL) == 422); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"pplx-embed-context-v1-0.6b\"," + "\"input\":\"hi\"}", NULL, NULL) == 422); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"pplx-embed-v1-4b\",\"input\":\"hi\"}", + NULL, NULL) == 503); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hi\"," + "\"encoding_format\":\"yaml\"}", NULL, NULL) == 422); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hi\"," + "\"dimensions\":64}", NULL, NULL) == 422); + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"pplx-embed-v1-0.6b\"," + "\"input\":[]}", NULL, NULL) == 422); + + /* Contextual endpoint: standard id is the wrong enum, contextual id is + * valid but not loaded. */ + TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", + TEST_API_KEY, + "{\"model\":\"pplx-embed-v1-0.6b\"," + "\"input\":[[\"a\"]]}", NULL, NULL) == 422); + TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings", + TEST_API_KEY, + "{\"model\":\"pplx-embed-context-v1-0.6b\"," + "\"input\":[[\"a\"]]}", NULL, NULL) == 503); + + /* Happy path, float encoding, single string input. */ + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, ok_req, + NULL, &body) == 200); + cJSON *root = body ? parse_embeddings_body(body, 1) : NULL; + if (root) { + cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data"); + cJSON *item = cJSON_GetArrayItem(data, 0); + cJSON *emb = cJSON_GetObjectItemCaseSensitive(item, "embedding"); + cJSON *idx = cJSON_GetObjectItemCaseSensitive(item, "index"); + TEST_ASSERT(cJSON_IsArray(emb)); + TEST_ASSERT(cJSON_GetArraySize(emb) == 1024); + TEST_ASSERT(cJSON_IsNumber(idx) && idx->valueint == 0); + int nonzero = 0; + cJSON *v; + cJSON_ArrayForEach(v, emb) { + TEST_ASSERT(cJSON_IsNumber(v)); + if (v->valuedouble != 0.0) nonzero = 1; + } + TEST_ASSERT(nonzero); + cJSON *usage = cJSON_GetObjectItemCaseSensitive(root, "usage"); + cJSON *total = usage + ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL; + TEST_ASSERT(total && cJSON_IsNumber(total) && total->valueint >= 1); + cJSON_Delete(root); + } + free(body); + body = NULL; + + /* Batch order: one embedding per input, indexes 0..n-1, and different + * texts produce different vectors. */ + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"pplx-embed-v1-0.6b\"," + "\"input\":[\"hello\",\"world\",\"held\"]," + "\"encoding_format\":\"float\"}", + NULL, &body) == 200); + root = body ? parse_embeddings_body(body, 3) : NULL; + if (root) { + cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data"); + cJSON *e0 = cJSON_GetObjectItemCaseSensitive( + cJSON_GetArrayItem(data, 0), "embedding"); + cJSON *e1 = cJSON_GetObjectItemCaseSensitive( + cJSON_GetArrayItem(data, 1), "embedding"); + for (int i = 0; i < 3; i++) { + cJSON *idx = cJSON_GetObjectItemCaseSensitive( + cJSON_GetArrayItem(data, i), "index"); + TEST_ASSERT(cJSON_IsNumber(idx) && idx->valueint == i); + } + int differ = 0; + for (int i = 0; i < 1024 && !differ; i++) { + if (cJSON_GetArrayItem(e0, i)->valuedouble != + cJSON_GetArrayItem(e1, i)->valuedouble) + differ = 1; + } + TEST_ASSERT(differ); + cJSON_Delete(root); + } + free(body); + body = NULL; + + /* Default encoding is base64_int8: a string of 1024 quantized bytes. */ + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"pplx-embed-v1-0.6b\"," + "\"input\":\"hello\"}", NULL, &body) == 200); + root = body ? parse_embeddings_body(body, 1) : NULL; + if (root) { + cJSON *emb = cJSON_GetObjectItemCaseSensitive( + cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"), + 0), "embedding"); + TEST_ASSERT(cJSON_IsString(emb)); + TEST_ASSERT(strlen(emb->valuestring) == ((1024 + 2) / 3) * 4); + cJSON_Delete(root); + } + free(body); + body = NULL; + + /* Matryoshka truncation via dimensions. */ + TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, + "{\"model\":\"pplx-embed-v1-0.6b\"," + "\"input\":\"hello\",\"dimensions\":128," + "\"encoding_format\":\"float\"}", + NULL, &body) == 200); + root = body ? parse_embeddings_body(body, 1) : NULL; + if (root) { + cJSON *emb = cJSON_GetObjectItemCaseSensitive( + cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"), + 0), "embedding"); + TEST_ASSERT(cJSON_IsArray(emb) && cJSON_GetArraySize(emb) == 128); + cJSON_Delete(root); + } + free(body); + body = NULL; + + /* Concurrent clients: more requests in flight than the micro-batch cap + * (-b 2) so the worker has to assemble several batches. */ + enum { N_CONC = 6 }; + int fds[N_CONC]; + for (int i = 0; i < N_CONC; i++) { + fds[i] = tcp_connect(port); + TEST_ASSERT(fds[i] >= 0); + if (fds[i] >= 0) + TEST_ASSERT(http_send(fds[i], "POST", "/v1/embeddings", + TEST_API_KEY, ok_req, NULL) == 0); + } + for (int i = 0; i < N_CONC; i++) { + if (fds[i] < 0) continue; + char *resp = NULL; + TEST_ASSERT(http_recv(fds[i], &resp) == 200); + free(resp); + close(fds[i]); + } +} + +static void test_http_server(void) +{ + char dir[1024]; + snprintf(dir, sizeof(dir), "%s/pplx-srv-test-XXXXXX", + getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); + if (!mkdtemp(dir)) { + fprintf(stderr, "FAIL: mkdtemp\n"); + test_failures++; + return; + } + /* The 0.6b server slot requires hidden_size 1024; everything else stays + * tiny. The model vocab covers every tokenizer fixture id. */ + tm_dims_t dims = {1024, 2, 1, 64, 8, TF_VOCAB_SIZE}; + if (tf_write_vocab(dir) != 0 || + tm_write_model_dims(dir, "F32", &dims) != 0) { + fprintf(stderr, "FAIL: fixture write\n"); + test_failures++; + return; + } + + int port = find_free_port(); + TEST_ASSERT(port > 0); + + srv_ctx ctx; + memset(&ctx, 0, sizeof(ctx)); + ctx.spec.id = "pplx-embed-v1-0.6b"; + ctx.spec.path = dir; + ctx.cfg.models = &ctx.spec; + ctx.cfg.n_models = 1; + ctx.cfg.port = port; + ctx.cfg.batch_size = 2; + ctx.cfg.batch_wait_us = 1000; + ctx.cfg.api_key = TEST_API_KEY; + + pthread_t th; + if (pthread_create(&th, NULL, srv_thread_main, &ctx) != 0) { + fprintf(stderr, "FAIL: server thread\n"); + test_failures++; + return; + } + + int up = 0; + for (int i = 0; i < 200; i++) { /* up to ~10 s */ + int fd = tcp_connect(port); + if (fd >= 0) { close(fd); up = 1; break; } + usleep(50 * 1000); + } + TEST_ASSERT(up); + if (up) test_http_embeddings(port); + + stop_signal_handler(SIGTERM); + pthread_join(th, NULL); + TEST_ASSERT(ctx.rc == 0); +} + int main(void) { test_base64_rfc4648(); test_quantize_int8_tanh(); test_encode_embedding_int8(); test_encode_embedding_binary(); + test_http_server(); if (test_failures) { fprintf(stderr, "server tests: %d failure(s)\n", test_failures); return 1; } - puts("ok: server unit tests passed"); + puts("ok: server unit and HTTP request-path tests passed"); return 0; } diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c index 8c20308..1de61b6 100644 --- a/tests/test_tokenizer.c +++ b/tests/test_tokenizer.c @@ -5,6 +5,7 @@ * string-based fallback) produces identical ids. Runs via `make test`. */ #include "qwen_tokenizer.h" +#include "tok_fixture.h" #include #include @@ -21,80 +22,6 @@ int qwen_verbose = 0; } \ } while (0) -/* GPT-2 byte -> unicode codepoint mapping (mirrors the tokenizer's rules). */ -static int byte_to_cp(int b) -{ - static int table[256]; - static int init = 0; - if (!init) { - int n = 0; - for (int i = 0; i < 256; i++) { - int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) || - (i >= 174 && i <= 255); - table[i] = normal ? i : 256 + n++; - } - init = 1; - } - return table[b]; -} - -static int cp_to_utf8(int cp, char *out) -{ - if (cp < 0x80) { out[0] = (char)cp; out[1] = '\0'; return 1; } - out[0] = (char)(0xC0 | (cp >> 6)); - out[1] = (char)(0x80 | (cp & 0x3F)); - out[2] = '\0'; - return 2; -} - -/* JSON-escape a byte-unicode token (the byte alphabet needs only \" and \\). */ -static const char *json_escape(const char *s, char *buf, size_t cap) -{ - size_t o = 0; - for (; *s && o + 2 < cap; s++) { - if (*s == '"' || *s == '\\') buf[o++] = '\\'; - buf[o++] = *s; - } - buf[o] = '\0'; - return buf; -} - -/* Merged tokens (ids 256+) and the merge list that produces them. */ -static const char *MERGED[] = { - "he", "ll", "hell", "hello", "\xC4\xA0w", "or", "ld", "\xC4\xA0wor", -}; -static const char *MERGES[] = { - "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d", - "\xC4\xA0w or", -}; -enum { N_MERGED = sizeof(MERGED) / sizeof(MERGED[0]) }; - -static int write_fixture(const char *dir) -{ - char path[2048], tok[4], esc[16]; - snprintf(path, sizeof(path), "%s/vocab.json", dir); - FILE *f = fopen(path, "w"); - if (!f) return -1; - fputs("{", f); - for (int b = 0; b < 256; b++) { - cp_to_utf8(byte_to_cp(b), tok); - fprintf(f, "%s\"%s\": %d", b ? ", " : "", - json_escape(tok, esc, sizeof(esc)), b); - } - for (int i = 0; i < N_MERGED; i++) - fprintf(f, ", \"%s\": %d", MERGED[i], 256 + i); - fputs("}", f); - fclose(f); - - snprintf(path, sizeof(path), "%s/merges.txt", dir); - f = fopen(path, "w"); - if (!f) return -1; - for (size_t i = 0; i < sizeof(MERGES) / sizeof(MERGES[0]); i++) - fprintf(f, "%s\n", MERGES[i]); - fclose(f); - return 0; -} - static void check_ids(const qwen_tokenizer_t *tok, qwen_tokenizer_workspace_t *ws, const char *text, const int *want, int n_want, const char *label) @@ -177,7 +104,7 @@ int main(void) char dir[1024]; snprintf(dir, sizeof(dir), "%s/pplx-tok-test-XXXXXX", getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp"); - if (!mkdtemp(dir) || write_fixture(dir) != 0) { + if (!mkdtemp(dir) || tf_write_vocab(dir) != 0) { fprintf(stderr, "fixture creation failed\n"); return 2; } diff --git a/tests/tiny_model.h b/tests/tiny_model.h index fc8cf48..0a4e67b 100644 --- a/tests/tiny_model.h +++ b/tests/tiny_model.h @@ -46,35 +46,46 @@ typedef struct { int rows, cols; /* cols == 0 means a 1-D tensor of `rows` values */ } tm_spec_t; -static const tm_spec_t TM_SPECS[] = { - {"embed_tokens.weight", 16, 4}, - {"layers.0.self_attn.q_proj.weight", 4, 4}, - {"layers.0.self_attn.k_proj.weight", 2, 4}, - {"layers.0.self_attn.v_proj.weight", 2, 4}, - {"layers.0.self_attn.o_proj.weight", 4, 4}, - {"layers.0.self_attn.q_norm.weight", 2, 0}, - {"layers.0.self_attn.k_norm.weight", 2, 0}, - {"layers.0.input_layernorm.weight", 4, 0}, - {"layers.0.post_attention_layernorm.weight", 4, 0}, - {"layers.0.mlp.gate_proj.weight", 8, 4}, - {"layers.0.mlp.up_proj.weight", 8, 4}, - {"layers.0.mlp.down_proj.weight", 4, 8}, - {"norm.weight", 4, 0}, -}; -enum { TM_N_SPECS = sizeof(TM_SPECS) / sizeof(TM_SPECS[0]) }; +/* Always one layer; everything else is free so a test can match a real + * model family's dimensions (e.g. hidden 1024 for the 0.6B server slot). */ +typedef struct { + int hidden, heads, kv_heads, head_dim, intermediate, vocab; +} tm_dims_t; + +enum { TM_N_SPECS = 13 }; -/* Write config.json + model.safetensors into dir. dtype: "F32" or "BF16". - * Returns 0 on success. */ -static int tm_write_model(const char *dir, const char *dtype) +/* Write config.json + model.safetensors with the given dimensions into dir. + * dtype: "F32" or "BF16". Returns 0 on success. */ +static int tm_write_model_dims(const char *dir, const char *dtype, + const tm_dims_t *d) { + int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim; + const tm_spec_t specs[TM_N_SPECS] = { + {"embed_tokens.weight", d->vocab, d->hidden}, + {"layers.0.self_attn.q_proj.weight", q, d->hidden}, + {"layers.0.self_attn.k_proj.weight", kv, d->hidden}, + {"layers.0.self_attn.v_proj.weight", kv, d->hidden}, + {"layers.0.self_attn.o_proj.weight", d->hidden, q}, + {"layers.0.self_attn.q_norm.weight", d->head_dim, 0}, + {"layers.0.self_attn.k_norm.weight", d->head_dim, 0}, + {"layers.0.input_layernorm.weight", d->hidden, 0}, + {"layers.0.post_attention_layernorm.weight", d->hidden, 0}, + {"layers.0.mlp.gate_proj.weight", d->intermediate, d->hidden}, + {"layers.0.mlp.up_proj.weight", d->intermediate, d->hidden}, + {"layers.0.mlp.down_proj.weight", d->hidden, d->intermediate}, + {"norm.weight", d->hidden, 0}, + }; + char path[2048]; snprintf(path, sizeof(path), "%s/config.json", dir); FILE *f = fopen(path, "w"); if (!f) return -1; - fputs("{\"hidden_size\":4,\"num_hidden_layers\":1," - "\"num_attention_heads\":2,\"num_key_value_heads\":1," - "\"head_dim\":2,\"intermediate_size\":8,\"vocab_size\":16," - "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0}", f); + fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1," + "\"num_attention_heads\":%d,\"num_key_value_heads\":%d," + "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d," + "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0}", + d->hidden, d->heads, d->kv_heads, d->head_dim, + d->intermediate, d->vocab); fclose(f); int bf16 = strcmp(dtype, "BF16") == 0; @@ -84,7 +95,7 @@ static int tm_write_model(const char *dir, const char *dtype) size_t hoff = 0, doff = 0; hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff, "{"); for (int t = 0; t < TM_N_SPECS; t++) { - const tm_spec_t *s = &TM_SPECS[t]; + const tm_spec_t *s = &specs[t]; size_t n = (size_t)s->rows * (s->cols ? (size_t)s->cols : 1); size_t end = doff + n * esize; if (s->cols) @@ -110,7 +121,7 @@ static int tm_write_model(const char *dir, const char *dtype) fwrite(len8, 1, 8, f); fwrite(header, 1, hoff, f); for (int t = 0; t < TM_N_SPECS; t++) { - const tm_spec_t *s = &TM_SPECS[t]; + const tm_spec_t *s = &specs[t]; size_t n = (size_t)s->rows * (s->cols ? (size_t)s->cols : 1); for (size_t i = 0; i < n; i++) { float v = tm_value(s->name, i); @@ -126,4 +137,11 @@ static int tm_write_model(const char *dir, const char *dtype) return 0; } +/* The historical default fixture: 4-dim hidden, 16-token vocab. */ +static inline int tm_write_model(const char *dir, const char *dtype) +{ + static const tm_dims_t d = {4, 2, 1, 2, 8, 16}; + return tm_write_model_dims(dir, dtype, &d); +} + #endif /* TINY_MODEL_H */ diff --git a/tests/tok_fixture.h b/tests/tok_fixture.h new file mode 100644 index 0000000..0ac81c8 --- /dev/null +++ b/tests/tok_fixture.h @@ -0,0 +1,88 @@ +/* tests/tok_fixture.h - synthesize a complete GPT-2-style byte-level BPE + * tokenizer fixture into a directory: vocab.json with all 256 byte tokens + * (ids 0-255, so any input tokenizes) plus a few merged tokens (ids 256+), + * and the matching merges.txt. Shared by the tokenizer and server tests. */ + +#ifndef TOK_FIXTURE_H +#define TOK_FIXTURE_H + +#include +#include + +/* GPT-2 byte -> unicode codepoint mapping (mirrors the tokenizer's rules). */ +static int tf_byte_to_cp(int b) +{ + static int table[256]; + static int init = 0; + if (!init) { + int n = 0; + for (int i = 0; i < 256; i++) { + int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) || + (i >= 174 && i <= 255); + table[i] = normal ? i : 256 + n++; + } + init = 1; + } + return table[b]; +} + +static int tf_cp_to_utf8(int cp, char *out) +{ + if (cp < 0x80) { out[0] = (char)cp; out[1] = '\0'; return 1; } + out[0] = (char)(0xC0 | (cp >> 6)); + out[1] = (char)(0x80 | (cp & 0x3F)); + out[2] = '\0'; + return 2; +} + +/* JSON-escape a byte-unicode token (the byte alphabet needs only \" and \\). */ +static const char *tf_json_escape(const char *s, char *buf, size_t cap) +{ + size_t o = 0; + for (; *s && o + 2 < cap; s++) { + if (*s == '"' || *s == '\\') buf[o++] = '\\'; + buf[o++] = *s; + } + buf[o] = '\0'; + return buf; +} + +/* Merged tokens (ids 256+) and the merge list that produces them. */ +static const char *TF_MERGED[] = { + "he", "ll", "hell", "hello", "\xC4\xA0w", "or", "ld", "\xC4\xA0wor", +}; +static const char *TF_MERGES[] = { + "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d", + "\xC4\xA0w or", +}; +enum { TF_N_MERGED = sizeof(TF_MERGED) / sizeof(TF_MERGED[0]) }; +enum { TF_VOCAB_SIZE = 256 + TF_N_MERGED }; + +/* Write vocab.json + merges.txt into dir. Returns 0 on success. */ +static int tf_write_vocab(const char *dir) +{ + char path[2048], tok[4], esc[16]; + snprintf(path, sizeof(path), "%s/vocab.json", dir); + FILE *f = fopen(path, "w"); + if (!f) return -1; + fputs("{", f); + for (int b = 0; b < 256; b++) { + tf_cp_to_utf8(tf_byte_to_cp(b), tok); + fprintf(f, "%s\"%s\": %d", b ? ", " : "", + tf_json_escape(tok, esc, sizeof(esc)), b); + } + for (int i = 0; i < TF_N_MERGED; i++) + fprintf(f, ", \"%s\": %d", TF_MERGED[i], 256 + i); + fputs("}", f); + fclose(f); + + snprintf(path, sizeof(path), "%s/merges.txt", dir); + f = fopen(path, "w"); + if (!f) return -1; + for (size_t i = 0; i < sizeof(TF_MERGES) / sizeof(TF_MERGES[0]); i++) + fprintf(f, "%s\n", TF_MERGES[i]); + fclose(f); + return 0; +} + +#endif /* TOK_FIXTURE_H */ -- cgit v1.3.1