aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 09:08:28 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit3999ed4fba0a0cf7c4b9ae941d387682839640e6 (patch)
treefd6be5db659201962c2df0f5b8f4c591eb4a5791
parentfbdc3ae0cc650f7f7e9a2e81e63c3731651b8271 (diff)
downloadffwd-3999ed4fba0a0cf7c4b9ae941d387682839640e6.tar.gz
ffwd-3999ed4fba0a0cf7c4b9ae941d387682839640e6.zip
tests: add server encoder unit tests via EMBED_SERVER_TEST include
-rw-r--r--.gitignore1
-rw-r--r--Makefile9
-rw-r--r--tests/test_server.c88
3 files changed, 97 insertions, 1 deletions
diff --git a/.gitignore b/.gitignore
index 451787b..3e4a1de 100644
--- a/.gitignore
+++ b/.gitignore
@@ -5,6 +5,7 @@ libpplxembed.*
tests/test_kernels_*
tests/test_safetensors
tests/test_bf16_model
+tests/test_server
bench/bench_tokenizer
book/
.venv/
diff --git a/Makefile b/Makefile
index 7dd1c9b..05a0088 100644
--- a/Makefile
+++ b/Makefile
@@ -171,6 +171,12 @@ test:
tests/test_bf16_model.c embed.c qwen_safetensors.c $(KERNEL_SRCS) \
-lm -lpthread
./tests/test_bf16_model
+ $(CC) -Wall -Wextra -O2 $(TEST_BLAS_CFLAGS) $(CJSON_CFLAGS) -I. -Ideps/ae \
+ -o tests/test_server tests/test_server.c embed.c embed_distributed.c \
+ qwen_tokenizer.c qwen_safetensors.c $(KERNEL_SRCS) \
+ deps/ae/ae.c deps/ae/anet.c deps/ae/monotonic.c \
+ -lm -lpthread $(TEST_BLAS_LDFLAGS) $(CJSON_LDFLAGS)
+ ./tests/test_server
bench-tokenizer:
$(CC) -Wall -Wextra -O3 -march=native -I. -o bench/bench_tokenizer \
@@ -245,4 +251,5 @@ clean:
rm -f $(OBJS) embed_mlx.o embed_cuda.o embed_cli.o embed_server.o \
$(TARGET) $(SERVER_TARGET) $(LIB) libpplxembed.dylib libpplxembed.so \
tests/test_kernels_generic tests/test_kernels_blas \
- tests/test_safetensors tests/test_bf16_model bench/bench_tokenizer
+ tests/test_safetensors tests/test_bf16_model tests/test_server \
+ bench/bench_tokenizer
diff --git a/tests/test_server.c b/tests/test_server.c
new file mode 100644
index 0000000..dcedd21
--- /dev/null
+++ b/tests/test_server.c
@@ -0,0 +1,88 @@
+/* tests/test_server.c - unit tests for embed_server.c static helpers.
+ * ds4-style: includes the whole server translation unit with the real main
+ * compiled out, so statics are directly testable. Runs via `make test`. */
+
+#define EMBED_SERVER_TEST
+#include "../embed_server.c"
+
+static int test_failures = 0;
+
+#define TEST_ASSERT(cond) do { \
+ if (!(cond)) { \
+ fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
+ test_failures++; \
+ } \
+} while (0)
+
+static void test_base64_rfc4648(void)
+{
+ /* RFC 4648 test vectors. */
+ static const struct { const char *in, *out; } cases[] = {
+ {"", ""}, {"f", "Zg=="}, {"fo", "Zm8="}, {"foo", "Zm9v"},
+ {"foob", "Zm9vYg=="}, {"fooba", "Zm9vYmE="}, {"foobar", "Zm9vYmFy"},
+ };
+ for (size_t i = 0; i < sizeof(cases) / sizeof(cases[0]); i++) {
+ char *got = base64_encode((const unsigned char *)cases[i].in,
+ strlen(cases[i].in));
+ TEST_ASSERT(strcmp(got, cases[i].out) == 0);
+ free(got);
+ }
+}
+
+static void test_quantize_int8_tanh(void)
+{
+ TEST_ASSERT(quantize_int8_tanh(0.0f) == 0);
+ /* tanh saturates to +-1, scaled by 127. */
+ TEST_ASSERT(quantize_int8_tanh(100.0f) == 127);
+ TEST_ASSERT(quantize_int8_tanh(-100.0f) == -127);
+ /* Symmetry. */
+ TEST_ASSERT(quantize_int8_tanh(0.5f) == -quantize_int8_tanh(-0.5f));
+ /* round(tanh(0.5) * 127) = round(58.69) = 59. */
+ TEST_ASSERT(quantize_int8_tanh(0.5f) == 59);
+}
+
+static void test_encode_embedding_int8(void)
+{
+ float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f};
+ signed char expect[4] = {0, 59, -59, 127};
+ char *got = encode_embedding(emb, 4, "base64_int8");
+ char *want = base64_encode((const unsigned char *)expect, 4);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+}
+
+static void test_encode_embedding_binary(void)
+{
+ /* Sign bits pack LSB-first: bit i of byte i/8 is 1 when emb[i] >= 0. */
+ float emb[8] = {1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f};
+ unsigned char expect = 0x85; /* bits 0, 2, 7 set */
+ char *got = encode_embedding(emb, 8, "base64_binary");
+ char *want = base64_encode(&expect, 1);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+
+ /* Zero counts as non-negative (sign bit set). */
+ float zeros[8] = {0};
+ unsigned char all = 0xff;
+ got = encode_embedding(zeros, 8, "base64_binary");
+ want = base64_encode(&all, 1);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+}
+
+int main(void)
+{
+ test_base64_rfc4648();
+ test_quantize_int8_tanh();
+ test_encode_embedding_int8();
+ test_encode_embedding_binary();
+ if (test_failures) {
+ fprintf(stderr, "server tests: %d failure(s)\n", test_failures);
+ return 1;
+ }
+ puts("ok: server unit tests passed");
+ return 0;
+}