aboutsummaryrefslogtreecommitdiff
path: root/tests
diff options
context:
space:
mode:
Diffstat (limited to 'tests')
-rw-r--r--tests/test_server.c418
-rw-r--r--tests/test_tokenizer.c77
-rw-r--r--tests/tiny_model.h68
-rw-r--r--tests/tok_fixture.h88
4 files changed, 548 insertions, 103 deletions
diff --git a/tests/test_server.c b/tests/test_server.c
index dcedd21..a2a8492 100644
--- a/tests/test_server.c
+++ b/tests/test_server.c
@@ -1,10 +1,19 @@
-/* tests/test_server.c - unit tests for embed_server.c static helpers.
+/* tests/test_server.c - tests for embed_server.c.
* ds4-style: includes the whole server translation unit with the real main
- * compiled out, so statics are directly testable. Runs via `make test`. */
+ * compiled out, so statics are directly testable. Unit-tests the encoding
+ * helpers, then boots the real server in-process (tiny 1024-dim hermetic
+ * model + byte-vocab tokenizer fixture) and drives the full HTTP request
+ * path over loopback sockets. Runs via `make test`. */
#define EMBED_SERVER_TEST
#include "../embed_server.c"
+#include "tiny_model.h"
+#include "tok_fixture.h"
+
+#include <arpa/inet.h>
+#include <netinet/in.h>
+
static int test_failures = 0;
#define TEST_ASSERT(cond) do { \
@@ -73,16 +82,419 @@ static void test_encode_embedding_binary(void)
free(want);
}
+/* ====================================================================
+ * HTTP request-path tests against an in-process server
+ * ==================================================================== */
+
+#define TEST_API_KEY "tt-test-key"
+
+static int find_free_port(void)
+{
+ int fd = socket(AF_INET, SOCK_STREAM, 0);
+ if (fd < 0) return -1;
+ struct sockaddr_in a;
+ memset(&a, 0, sizeof(a));
+ a.sin_family = AF_INET;
+ a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+ socklen_t alen = sizeof(a);
+ if (bind(fd, (struct sockaddr *)&a, sizeof(a)) != 0 ||
+ getsockname(fd, (struct sockaddr *)&a, &alen) != 0) {
+ close(fd);
+ return -1;
+ }
+ int port = ntohs(a.sin_port);
+ close(fd);
+ return port;
+}
+
+static int tcp_connect(int port)
+{
+ int fd = socket(AF_INET, SOCK_STREAM, 0);
+ if (fd < 0) return -1;
+ struct sockaddr_in a;
+ memset(&a, 0, sizeof(a));
+ a.sin_family = AF_INET;
+ a.sin_port = htons((uint16_t)port);
+ a.sin_addr.s_addr = htonl(INADDR_LOOPBACK);
+ if (connect(fd, (struct sockaddr *)&a, sizeof(a)) != 0) {
+ close(fd);
+ return -1;
+ }
+ return fd;
+}
+
+static int send_all(int fd, const char *buf, size_t n)
+{
+ size_t off = 0;
+ while (off < n) {
+ ssize_t w = write(fd, buf + off, n - off);
+ if (w < 0) {
+ if (errno == EINTR) continue;
+ return -1;
+ }
+ off += (size_t)w;
+ }
+ return 0;
+}
+
+/* Send one request; raw == NULL builds a normal request, otherwise `raw`
+ * bytes go on the wire verbatim (for malformed-HTTP checks). */
+static int http_send(int fd, const char *method, const char *path,
+ const char *auth, const char *body, const char *raw)
+{
+ if (raw) return send_all(fd, raw, strlen(raw));
+ char head[512];
+ size_t blen = body ? strlen(body) : 0;
+ int n = snprintf(head, sizeof(head),
+ "%s %s HTTP/1.1\r\n"
+ "Host: 127.0.0.1\r\n"
+ "%s%s%s"
+ "Content-Type: application/json\r\n"
+ "Content-Length: %zu\r\n"
+ "\r\n",
+ method, path,
+ auth ? "Authorization: Bearer " : "",
+ auth ? auth : "", auth ? "\r\n" : "",
+ blen);
+ if (n < 0 || (size_t)n >= sizeof(head)) return -1;
+ if (send_all(fd, head, (size_t)n) != 0) return -1;
+ return blen ? send_all(fd, body, blen) : 0;
+}
+
+/* Read one response: returns the status code (or -1) and the body, which
+ * the caller frees. Reads exactly Content-Length body bytes so it works
+ * with keep-alive connections. */
+static int http_recv(int fd, char **out_body)
+{
+ *out_body = NULL;
+ size_t cap = 8192, len = 0;
+ char *buf = malloc(cap);
+ if (!buf) return -1;
+ char *hdr_end = NULL;
+ while (!hdr_end) {
+ if (len + 1 >= cap) {
+ cap *= 2;
+ char *nb = realloc(buf, cap);
+ if (!nb) { free(buf); return -1; }
+ buf = nb;
+ }
+ ssize_t r = read(fd, buf + len, cap - len - 1);
+ if (r <= 0) { free(buf); return -1; }
+ len += (size_t)r;
+ buf[len] = '\0';
+ hdr_end = strstr(buf, "\r\n\r\n");
+ }
+
+ int status = -1;
+ if (sscanf(buf, "HTTP/1.1 %d", &status) != 1) { free(buf); return -1; }
+ size_t content_length = 0;
+ const char *cl = strstr(buf, "Content-Length:");
+ if (cl && cl < hdr_end) content_length = strtoul(cl + 15, NULL, 10);
+
+ size_t body_off = (size_t)(hdr_end - buf) + 4;
+ size_t need = body_off + content_length;
+ if (need + 1 > cap) {
+ char *nb = realloc(buf, need + 1);
+ if (!nb) { free(buf); return -1; }
+ buf = nb;
+ }
+ while (len < need) {
+ ssize_t r = read(fd, buf + len, need - len);
+ if (r <= 0) { free(buf); return -1; }
+ len += (size_t)r;
+ }
+ buf[need] = '\0';
+ *out_body = strdup(buf + body_off);
+ free(buf);
+ return *out_body ? status : -1;
+}
+
+/* One-shot request helper; returns status, body via out_body (caller frees,
+ * may be NULL when only the status matters). */
+static int http_req(int port, const char *method, const char *path,
+ const char *auth, const char *body, const char *raw,
+ char **out_body)
+{
+ int fd = tcp_connect(port);
+ if (fd < 0) return -1;
+ char *resp = NULL;
+ int status = -1;
+ if (http_send(fd, method, path, auth, body, raw) == 0)
+ status = http_recv(fd, &resp);
+ close(fd);
+ if (out_body) *out_body = resp;
+ else free(resp);
+ return status;
+}
+
+typedef struct {
+ pplx_server_model_spec_t spec;
+ pplx_server_config_t cfg;
+ int rc;
+} srv_ctx;
+
+static void *srv_thread_main(void *arg)
+{
+ srv_ctx *s = (srv_ctx *)arg;
+ s->rc = pplx_run_server(&s->cfg);
+ return NULL;
+}
+
+static cJSON *parse_embeddings_body(const char *body, int want_items)
+{
+ cJSON *root = cJSON_Parse(body);
+ TEST_ASSERT(root != NULL);
+ if (!root) return NULL;
+ cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data");
+ TEST_ASSERT(cJSON_IsArray(data));
+ TEST_ASSERT(cJSON_GetArraySize(data) == want_items);
+ if (!cJSON_IsArray(data) || cJSON_GetArraySize(data) != want_items) {
+ cJSON_Delete(root);
+ return NULL;
+ }
+ return root;
+}
+
+static void test_http_embeddings(int port)
+{
+ char *body = NULL;
+
+ /* CORS preflight and routing. */
+ TEST_ASSERT(http_req(port, "OPTIONS", "/v1/embeddings", NULL, NULL, NULL,
+ NULL) == 204);
+ TEST_ASSERT(http_req(port, "GET", "/v1/embeddings", TEST_API_KEY, NULL,
+ NULL, NULL) == 404);
+ TEST_ASSERT(http_req(port, "POST", "/nope", TEST_API_KEY, "{}", NULL,
+ NULL) == 404);
+
+ /* Auth: required, and the exact key. */
+ const char *ok_req =
+ "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hello world\","
+ "\"encoding_format\":\"float\"}";
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", NULL, ok_req, NULL,
+ NULL) == 401);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", "wrong-key", ok_req,
+ NULL, NULL) == 401);
+
+ /* Malformed HTTP (400) and malformed JSON (422 detail). */
+ TEST_ASSERT(http_req(port, NULL, NULL, NULL, NULL,
+ "GARBAGE\r\n\r\n", NULL) == 400);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{nope", NULL, NULL) == 422);
+
+ /* Validation: missing/unknown model, wrong endpoint family, valid but
+ * unloaded id, bad encoding enum, dimensions out of range; the
+ * API answers validation errors with 422 detail arrays. */
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"input\":\"hi\"}", NULL, NULL) == 422);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"foo\",\"input\":\"hi\"}", NULL,
+ NULL) == 422);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"pplx-embed-context-v1-0.6b\","
+ "\"input\":\"hi\"}", NULL, NULL) == 422);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"pplx-embed-v1-4b\",\"input\":\"hi\"}",
+ NULL, NULL) == 503);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hi\","
+ "\"encoding_format\":\"yaml\"}", NULL, NULL) == 422);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"pplx-embed-v1-0.6b\",\"input\":\"hi\","
+ "\"dimensions\":64}", NULL, NULL) == 422);
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"pplx-embed-v1-0.6b\","
+ "\"input\":[]}", NULL, NULL) == 422);
+
+ /* Contextual endpoint: standard id is the wrong enum, contextual id is
+ * valid but not loaded. */
+ TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings",
+ TEST_API_KEY,
+ "{\"model\":\"pplx-embed-v1-0.6b\","
+ "\"input\":[[\"a\"]]}", NULL, NULL) == 422);
+ TEST_ASSERT(http_req(port, "POST", "/v1/contextualizedembeddings",
+ TEST_API_KEY,
+ "{\"model\":\"pplx-embed-context-v1-0.6b\","
+ "\"input\":[[\"a\"]]}", NULL, NULL) == 503);
+
+ /* Happy path, float encoding, single string input. */
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY, ok_req,
+ NULL, &body) == 200);
+ cJSON *root = body ? parse_embeddings_body(body, 1) : NULL;
+ if (root) {
+ cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data");
+ cJSON *item = cJSON_GetArrayItem(data, 0);
+ cJSON *emb = cJSON_GetObjectItemCaseSensitive(item, "embedding");
+ cJSON *idx = cJSON_GetObjectItemCaseSensitive(item, "index");
+ TEST_ASSERT(cJSON_IsArray(emb));
+ TEST_ASSERT(cJSON_GetArraySize(emb) == 1024);
+ TEST_ASSERT(cJSON_IsNumber(idx) && idx->valueint == 0);
+ int nonzero = 0;
+ cJSON *v;
+ cJSON_ArrayForEach(v, emb) {
+ TEST_ASSERT(cJSON_IsNumber(v));
+ if (v->valuedouble != 0.0) nonzero = 1;
+ }
+ TEST_ASSERT(nonzero);
+ cJSON *usage = cJSON_GetObjectItemCaseSensitive(root, "usage");
+ cJSON *total = usage
+ ? cJSON_GetObjectItemCaseSensitive(usage, "total_tokens") : NULL;
+ TEST_ASSERT(total && cJSON_IsNumber(total) && total->valueint >= 1);
+ cJSON_Delete(root);
+ }
+ free(body);
+ body = NULL;
+
+ /* Batch order: one embedding per input, indexes 0..n-1, and different
+ * texts produce different vectors. */
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"pplx-embed-v1-0.6b\","
+ "\"input\":[\"hello\",\"world\",\"held\"],"
+ "\"encoding_format\":\"float\"}",
+ NULL, &body) == 200);
+ root = body ? parse_embeddings_body(body, 3) : NULL;
+ if (root) {
+ cJSON *data = cJSON_GetObjectItemCaseSensitive(root, "data");
+ cJSON *e0 = cJSON_GetObjectItemCaseSensitive(
+ cJSON_GetArrayItem(data, 0), "embedding");
+ cJSON *e1 = cJSON_GetObjectItemCaseSensitive(
+ cJSON_GetArrayItem(data, 1), "embedding");
+ for (int i = 0; i < 3; i++) {
+ cJSON *idx = cJSON_GetObjectItemCaseSensitive(
+ cJSON_GetArrayItem(data, i), "index");
+ TEST_ASSERT(cJSON_IsNumber(idx) && idx->valueint == i);
+ }
+ int differ = 0;
+ for (int i = 0; i < 1024 && !differ; i++) {
+ if (cJSON_GetArrayItem(e0, i)->valuedouble !=
+ cJSON_GetArrayItem(e1, i)->valuedouble)
+ differ = 1;
+ }
+ TEST_ASSERT(differ);
+ cJSON_Delete(root);
+ }
+ free(body);
+ body = NULL;
+
+ /* Default encoding is base64_int8: a string of 1024 quantized bytes. */
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"pplx-embed-v1-0.6b\","
+ "\"input\":\"hello\"}", NULL, &body) == 200);
+ root = body ? parse_embeddings_body(body, 1) : NULL;
+ if (root) {
+ cJSON *emb = cJSON_GetObjectItemCaseSensitive(
+ cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"),
+ 0), "embedding");
+ TEST_ASSERT(cJSON_IsString(emb));
+ TEST_ASSERT(strlen(emb->valuestring) == ((1024 + 2) / 3) * 4);
+ cJSON_Delete(root);
+ }
+ free(body);
+ body = NULL;
+
+ /* Matryoshka truncation via dimensions. */
+ TEST_ASSERT(http_req(port, "POST", "/v1/embeddings", TEST_API_KEY,
+ "{\"model\":\"pplx-embed-v1-0.6b\","
+ "\"input\":\"hello\",\"dimensions\":128,"
+ "\"encoding_format\":\"float\"}",
+ NULL, &body) == 200);
+ root = body ? parse_embeddings_body(body, 1) : NULL;
+ if (root) {
+ cJSON *emb = cJSON_GetObjectItemCaseSensitive(
+ cJSON_GetArrayItem(cJSON_GetObjectItemCaseSensitive(root, "data"),
+ 0), "embedding");
+ TEST_ASSERT(cJSON_IsArray(emb) && cJSON_GetArraySize(emb) == 128);
+ cJSON_Delete(root);
+ }
+ free(body);
+ body = NULL;
+
+ /* Concurrent clients: more requests in flight than the micro-batch cap
+ * (-b 2) so the worker has to assemble several batches. */
+ enum { N_CONC = 6 };
+ int fds[N_CONC];
+ for (int i = 0; i < N_CONC; i++) {
+ fds[i] = tcp_connect(port);
+ TEST_ASSERT(fds[i] >= 0);
+ if (fds[i] >= 0)
+ TEST_ASSERT(http_send(fds[i], "POST", "/v1/embeddings",
+ TEST_API_KEY, ok_req, NULL) == 0);
+ }
+ for (int i = 0; i < N_CONC; i++) {
+ if (fds[i] < 0) continue;
+ char *resp = NULL;
+ TEST_ASSERT(http_recv(fds[i], &resp) == 200);
+ free(resp);
+ close(fds[i]);
+ }
+}
+
+static void test_http_server(void)
+{
+ char dir[1024];
+ snprintf(dir, sizeof(dir), "%s/pplx-srv-test-XXXXXX",
+ getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
+ if (!mkdtemp(dir)) {
+ fprintf(stderr, "FAIL: mkdtemp\n");
+ test_failures++;
+ return;
+ }
+ /* The 0.6b server slot requires hidden_size 1024; everything else stays
+ * tiny. The model vocab covers every tokenizer fixture id. */
+ tm_dims_t dims = {1024, 2, 1, 64, 8, TF_VOCAB_SIZE};
+ if (tf_write_vocab(dir) != 0 ||
+ tm_write_model_dims(dir, "F32", &dims) != 0) {
+ fprintf(stderr, "FAIL: fixture write\n");
+ test_failures++;
+ return;
+ }
+
+ int port = find_free_port();
+ TEST_ASSERT(port > 0);
+
+ srv_ctx ctx;
+ memset(&ctx, 0, sizeof(ctx));
+ ctx.spec.id = "pplx-embed-v1-0.6b";
+ ctx.spec.path = dir;
+ ctx.cfg.models = &ctx.spec;
+ ctx.cfg.n_models = 1;
+ ctx.cfg.port = port;
+ ctx.cfg.batch_size = 2;
+ ctx.cfg.batch_wait_us = 1000;
+ ctx.cfg.api_key = TEST_API_KEY;
+
+ pthread_t th;
+ if (pthread_create(&th, NULL, srv_thread_main, &ctx) != 0) {
+ fprintf(stderr, "FAIL: server thread\n");
+ test_failures++;
+ return;
+ }
+
+ int up = 0;
+ for (int i = 0; i < 200; i++) { /* up to ~10 s */
+ int fd = tcp_connect(port);
+ if (fd >= 0) { close(fd); up = 1; break; }
+ usleep(50 * 1000);
+ }
+ TEST_ASSERT(up);
+ if (up) test_http_embeddings(port);
+
+ stop_signal_handler(SIGTERM);
+ pthread_join(th, NULL);
+ TEST_ASSERT(ctx.rc == 0);
+}
+
int main(void)
{
test_base64_rfc4648();
test_quantize_int8_tanh();
test_encode_embedding_int8();
test_encode_embedding_binary();
+ test_http_server();
if (test_failures) {
fprintf(stderr, "server tests: %d failure(s)\n", test_failures);
return 1;
}
- puts("ok: server unit tests passed");
+ puts("ok: server unit and HTTP request-path tests passed");
return 0;
}
diff --git a/tests/test_tokenizer.c b/tests/test_tokenizer.c
index 8c20308..1de61b6 100644
--- a/tests/test_tokenizer.c
+++ b/tests/test_tokenizer.c
@@ -5,6 +5,7 @@
* string-based fallback) produces identical ids. Runs via `make test`. */
#include "qwen_tokenizer.h"
+#include "tok_fixture.h"
#include <stdio.h>
#include <stdlib.h>
@@ -21,80 +22,6 @@ int qwen_verbose = 0;
} \
} while (0)
-/* GPT-2 byte -> unicode codepoint mapping (mirrors the tokenizer's rules). */
-static int byte_to_cp(int b)
-{
- static int table[256];
- static int init = 0;
- if (!init) {
- int n = 0;
- for (int i = 0; i < 256; i++) {
- int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) ||
- (i >= 174 && i <= 255);
- table[i] = normal ? i : 256 + n++;
- }
- init = 1;
- }
- return table[b];
-}
-
-static int cp_to_utf8(int cp, char *out)
-{
- if (cp < 0x80) { out[0] = (char)cp; out[1] = '\0'; return 1; }
- out[0] = (char)(0xC0 | (cp >> 6));
- out[1] = (char)(0x80 | (cp & 0x3F));
- out[2] = '\0';
- return 2;
-}
-
-/* JSON-escape a byte-unicode token (the byte alphabet needs only \" and \\). */
-static const char *json_escape(const char *s, char *buf, size_t cap)
-{
- size_t o = 0;
- for (; *s && o + 2 < cap; s++) {
- if (*s == '"' || *s == '\\') buf[o++] = '\\';
- buf[o++] = *s;
- }
- buf[o] = '\0';
- return buf;
-}
-
-/* Merged tokens (ids 256+) and the merge list that produces them. */
-static const char *MERGED[] = {
- "he", "ll", "hell", "hello", "\xC4\xA0w", "or", "ld", "\xC4\xA0wor",
-};
-static const char *MERGES[] = {
- "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d",
- "\xC4\xA0w or",
-};
-enum { N_MERGED = sizeof(MERGED) / sizeof(MERGED[0]) };
-
-static int write_fixture(const char *dir)
-{
- char path[2048], tok[4], esc[16];
- snprintf(path, sizeof(path), "%s/vocab.json", dir);
- FILE *f = fopen(path, "w");
- if (!f) return -1;
- fputs("{", f);
- for (int b = 0; b < 256; b++) {
- cp_to_utf8(byte_to_cp(b), tok);
- fprintf(f, "%s\"%s\": %d", b ? ", " : "",
- json_escape(tok, esc, sizeof(esc)), b);
- }
- for (int i = 0; i < N_MERGED; i++)
- fprintf(f, ", \"%s\": %d", MERGED[i], 256 + i);
- fputs("}", f);
- fclose(f);
-
- snprintf(path, sizeof(path), "%s/merges.txt", dir);
- f = fopen(path, "w");
- if (!f) return -1;
- for (size_t i = 0; i < sizeof(MERGES) / sizeof(MERGES[0]); i++)
- fprintf(f, "%s\n", MERGES[i]);
- fclose(f);
- return 0;
-}
-
static void check_ids(const qwen_tokenizer_t *tok,
qwen_tokenizer_workspace_t *ws, const char *text,
const int *want, int n_want, const char *label)
@@ -177,7 +104,7 @@ int main(void)
char dir[1024];
snprintf(dir, sizeof(dir), "%s/pplx-tok-test-XXXXXX",
getenv("TMPDIR") ? getenv("TMPDIR") : "/tmp");
- if (!mkdtemp(dir) || write_fixture(dir) != 0) {
+ if (!mkdtemp(dir) || tf_write_vocab(dir) != 0) {
fprintf(stderr, "fixture creation failed\n");
return 2;
}
diff --git a/tests/tiny_model.h b/tests/tiny_model.h
index fc8cf48..0a4e67b 100644
--- a/tests/tiny_model.h
+++ b/tests/tiny_model.h
@@ -46,35 +46,46 @@ typedef struct {
int rows, cols; /* cols == 0 means a 1-D tensor of `rows` values */
} tm_spec_t;
-static const tm_spec_t TM_SPECS[] = {
- {"embed_tokens.weight", 16, 4},
- {"layers.0.self_attn.q_proj.weight", 4, 4},
- {"layers.0.self_attn.k_proj.weight", 2, 4},
- {"layers.0.self_attn.v_proj.weight", 2, 4},
- {"layers.0.self_attn.o_proj.weight", 4, 4},
- {"layers.0.self_attn.q_norm.weight", 2, 0},
- {"layers.0.self_attn.k_norm.weight", 2, 0},
- {"layers.0.input_layernorm.weight", 4, 0},
- {"layers.0.post_attention_layernorm.weight", 4, 0},
- {"layers.0.mlp.gate_proj.weight", 8, 4},
- {"layers.0.mlp.up_proj.weight", 8, 4},
- {"layers.0.mlp.down_proj.weight", 4, 8},
- {"norm.weight", 4, 0},
-};
-enum { TM_N_SPECS = sizeof(TM_SPECS) / sizeof(TM_SPECS[0]) };
+/* Always one layer; everything else is free so a test can match a real
+ * model family's dimensions (e.g. hidden 1024 for the 0.6B server slot). */
+typedef struct {
+ int hidden, heads, kv_heads, head_dim, intermediate, vocab;
+} tm_dims_t;
+
+enum { TM_N_SPECS = 13 };
-/* Write config.json + model.safetensors into dir. dtype: "F32" or "BF16".
- * Returns 0 on success. */
-static int tm_write_model(const char *dir, const char *dtype)
+/* Write config.json + model.safetensors with the given dimensions into dir.
+ * dtype: "F32" or "BF16". Returns 0 on success. */
+static int tm_write_model_dims(const char *dir, const char *dtype,
+ const tm_dims_t *d)
{
+ int q = d->heads * d->head_dim, kv = d->kv_heads * d->head_dim;
+ const tm_spec_t specs[TM_N_SPECS] = {
+ {"embed_tokens.weight", d->vocab, d->hidden},
+ {"layers.0.self_attn.q_proj.weight", q, d->hidden},
+ {"layers.0.self_attn.k_proj.weight", kv, d->hidden},
+ {"layers.0.self_attn.v_proj.weight", kv, d->hidden},
+ {"layers.0.self_attn.o_proj.weight", d->hidden, q},
+ {"layers.0.self_attn.q_norm.weight", d->head_dim, 0},
+ {"layers.0.self_attn.k_norm.weight", d->head_dim, 0},
+ {"layers.0.input_layernorm.weight", d->hidden, 0},
+ {"layers.0.post_attention_layernorm.weight", d->hidden, 0},
+ {"layers.0.mlp.gate_proj.weight", d->intermediate, d->hidden},
+ {"layers.0.mlp.up_proj.weight", d->intermediate, d->hidden},
+ {"layers.0.mlp.down_proj.weight", d->hidden, d->intermediate},
+ {"norm.weight", d->hidden, 0},
+ };
+
char path[2048];
snprintf(path, sizeof(path), "%s/config.json", dir);
FILE *f = fopen(path, "w");
if (!f) return -1;
- fputs("{\"hidden_size\":4,\"num_hidden_layers\":1,"
- "\"num_attention_heads\":2,\"num_key_value_heads\":1,"
- "\"head_dim\":2,\"intermediate_size\":8,\"vocab_size\":16,"
- "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0}", f);
+ fprintf(f, "{\"hidden_size\":%d,\"num_hidden_layers\":1,"
+ "\"num_attention_heads\":%d,\"num_key_value_heads\":%d,"
+ "\"head_dim\":%d,\"intermediate_size\":%d,\"vocab_size\":%d,"
+ "\"rms_norm_eps\":1e-6,\"rope_theta\":10000.0}",
+ d->hidden, d->heads, d->kv_heads, d->head_dim,
+ d->intermediate, d->vocab);
fclose(f);
int bf16 = strcmp(dtype, "BF16") == 0;
@@ -84,7 +95,7 @@ static int tm_write_model(const char *dir, const char *dtype)
size_t hoff = 0, doff = 0;
hoff += (size_t)snprintf(header + hoff, sizeof(header) - hoff, "{");
for (int t = 0; t < TM_N_SPECS; t++) {
- const tm_spec_t *s = &TM_SPECS[t];
+ const tm_spec_t *s = &specs[t];
size_t n = (size_t)s->rows * (s->cols ? (size_t)s->cols : 1);
size_t end = doff + n * esize;
if (s->cols)
@@ -110,7 +121,7 @@ static int tm_write_model(const char *dir, const char *dtype)
fwrite(len8, 1, 8, f);
fwrite(header, 1, hoff, f);
for (int t = 0; t < TM_N_SPECS; t++) {
- const tm_spec_t *s = &TM_SPECS[t];
+ const tm_spec_t *s = &specs[t];
size_t n = (size_t)s->rows * (s->cols ? (size_t)s->cols : 1);
for (size_t i = 0; i < n; i++) {
float v = tm_value(s->name, i);
@@ -126,4 +137,11 @@ static int tm_write_model(const char *dir, const char *dtype)
return 0;
}
+/* The historical default fixture: 4-dim hidden, 16-token vocab. */
+static inline int tm_write_model(const char *dir, const char *dtype)
+{
+ static const tm_dims_t d = {4, 2, 1, 2, 8, 16};
+ return tm_write_model_dims(dir, dtype, &d);
+}
+
#endif /* TINY_MODEL_H */
diff --git a/tests/tok_fixture.h b/tests/tok_fixture.h
new file mode 100644
index 0000000..0ac81c8
--- /dev/null
+++ b/tests/tok_fixture.h
@@ -0,0 +1,88 @@
+/* tests/tok_fixture.h - synthesize a complete GPT-2-style byte-level BPE
+ * tokenizer fixture into a directory: vocab.json with all 256 byte tokens
+ * (ids 0-255, so any input tokenizes) plus a few merged tokens (ids 256+),
+ * and the matching merges.txt. Shared by the tokenizer and server tests. */
+
+#ifndef TOK_FIXTURE_H
+#define TOK_FIXTURE_H
+
+#include <stdio.h>
+#include <string.h>
+
+/* GPT-2 byte -> unicode codepoint mapping (mirrors the tokenizer's rules). */
+static int tf_byte_to_cp(int b)
+{
+ static int table[256];
+ static int init = 0;
+ if (!init) {
+ int n = 0;
+ for (int i = 0; i < 256; i++) {
+ int normal = (i >= 33 && i <= 126) || (i >= 161 && i <= 172) ||
+ (i >= 174 && i <= 255);
+ table[i] = normal ? i : 256 + n++;
+ }
+ init = 1;
+ }
+ return table[b];
+}
+
+static int tf_cp_to_utf8(int cp, char *out)
+{
+ if (cp < 0x80) { out[0] = (char)cp; out[1] = '\0'; return 1; }
+ out[0] = (char)(0xC0 | (cp >> 6));
+ out[1] = (char)(0x80 | (cp & 0x3F));
+ out[2] = '\0';
+ return 2;
+}
+
+/* JSON-escape a byte-unicode token (the byte alphabet needs only \" and \\). */
+static const char *tf_json_escape(const char *s, char *buf, size_t cap)
+{
+ size_t o = 0;
+ for (; *s && o + 2 < cap; s++) {
+ if (*s == '"' || *s == '\\') buf[o++] = '\\';
+ buf[o++] = *s;
+ }
+ buf[o] = '\0';
+ return buf;
+}
+
+/* Merged tokens (ids 256+) and the merge list that produces them. */
+static const char *TF_MERGED[] = {
+ "he", "ll", "hell", "hello", "\xC4\xA0w", "or", "ld", "\xC4\xA0wor",
+};
+static const char *TF_MERGES[] = {
+ "h e", "l l", "he ll", "hell o", "\xC4\xA0 w", "o r", "l d",
+ "\xC4\xA0w or",
+};
+enum { TF_N_MERGED = sizeof(TF_MERGED) / sizeof(TF_MERGED[0]) };
+enum { TF_VOCAB_SIZE = 256 + TF_N_MERGED };
+
+/* Write vocab.json + merges.txt into dir. Returns 0 on success. */
+static int tf_write_vocab(const char *dir)
+{
+ char path[2048], tok[4], esc[16];
+ snprintf(path, sizeof(path), "%s/vocab.json", dir);
+ FILE *f = fopen(path, "w");
+ if (!f) return -1;
+ fputs("{", f);
+ for (int b = 0; b < 256; b++) {
+ tf_cp_to_utf8(tf_byte_to_cp(b), tok);
+ fprintf(f, "%s\"%s\": %d", b ? ", " : "",
+ tf_json_escape(tok, esc, sizeof(esc)), b);
+ }
+ for (int i = 0; i < TF_N_MERGED; i++)
+ fprintf(f, ", \"%s\": %d", TF_MERGED[i], 256 + i);
+ fputs("}", f);
+ fclose(f);
+
+ snprintf(path, sizeof(path), "%s/merges.txt", dir);
+ f = fopen(path, "w");
+ if (!f) return -1;
+ for (size_t i = 0; i < sizeof(TF_MERGES) / sizeof(TF_MERGES[0]); i++)
+ fprintf(f, "%s\n", TF_MERGES[i]);
+ fclose(f);
+ return 0;
+}
+
+#endif /* TOK_FIXTURE_H */