aboutsummaryrefslogtreecommitdiff
path: root/tools/server/test_encode.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:10:56 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 05:57:19 +0200
commit6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch)
treec23afb50c28b9deaddae5c6526a74ee11cf2d865 /tools/server/test_encode.c
parentd4bdb8dae753462f621507d5fc3582e0e53da35c (diff)
downloadffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz
ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip
all: split Makefile and reorg library/tools
Diffstat (limited to 'tools/server/test_encode.c')
-rw-r--r--tools/server/test_encode.c90
1 files changed, 90 insertions, 0 deletions
diff --git a/tools/server/test_encode.c b/tools/server/test_encode.c
new file mode 100644
index 0000000..a7db9c4
--- /dev/null
+++ b/tools/server/test_encode.c
@@ -0,0 +1,90 @@
+/* test_encode.c - embedding quantization and the output encodings
+ * (base64_int8, base64_binary, base64 float32) plus float rendering. */
+
+#include "base64.h"
+#include "server_internal.h"
+#include "test_util.h"
+
+#include <stdlib.h>
+#include <string.h>
+
+static void test_quantize_int8_tanh(void) {
+ TEST_ASSERT(quantize_int8_tanh(0.0f) == 0);
+ /* tanh saturates to +-1, scaled by 127. */
+ TEST_ASSERT(quantize_int8_tanh(100.0f) == 127);
+ TEST_ASSERT(quantize_int8_tanh(-100.0f) == -127);
+ /* Symmetry. */
+ TEST_ASSERT(quantize_int8_tanh(0.5f) == -quantize_int8_tanh(-0.5f));
+ /* round(tanh(0.5) * 127) = round(58.69) = 59. */
+ TEST_ASSERT(quantize_int8_tanh(0.5f) == 59);
+}
+
+static void test_encode_embedding_int8(void) {
+ float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f};
+ signed char expect[4] = {0, 59, -59, 127};
+ char *got = encode_embedding(emb, 4, "base64_int8");
+ char *want = base64_encode((const unsigned char *)expect, 4);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+}
+
+static void test_encode_embedding_binary(void) {
+ /* Sign bits pack LSB-first: bit i of byte i/8 is 1 when emb[i] >= 0. */
+ float emb[8] = {1.0f, -1.0f, 1.0f, -1.0f, -1.0f, -1.0f, -1.0f, 1.0f};
+ unsigned char expect = 0x85; /* bits 0, 2, 7 set */
+ char *got = encode_embedding(emb, 8, "base64_binary");
+ char *want = base64_encode(&expect, 1);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+
+ /* Zero counts as non-negative (sign bit set). */
+ float zeros[8] = {0};
+ unsigned char all = 0xff;
+ got = encode_embedding(zeros, 8, "base64_binary");
+ want = base64_encode(&all, 1);
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+}
+
+static void test_encode_embedding_base64_float32(void) {
+ /* OpenAI/DashScope "base64" is base64 of the raw little-endian float32
+ * vector - lossless, unlike the int8 formats. */
+ float emb[4] = {0.0f, 0.5f, -0.5f, 100.0f};
+ /* Reference bytes via memcpy: a float->byte reinterpret cast confuses the
+ * static analyzer's uninitialized-read model. */
+ unsigned char bytes[sizeof emb];
+ memcpy(bytes, emb, sizeof emb);
+ char *want = base64_encode(bytes, sizeof emb);
+ char *got = encode_embedding(emb, 4, "base64");
+ TEST_ASSERT(strcmp(got, want) == 0);
+ free(got);
+ free(want);
+}
+
+static void test_append_embedding_value_float(void) {
+ float emb[1] = {0.5f};
+
+ /* OpenAI-compatible models render the true float32 value. */
+ sbuf b = {0};
+ append_embedding_value(&b, 0, emb, 1, "float", FFWD_API_OPENAI);
+ TEST_ASSERT(strstr(b.ptr, "\"embedding\":[0.5]") != NULL);
+ sbuf_free(&b);
+
+ /* Perplexity-compatible models render the int8-decoded view. */
+ sbuf b2 = {0};
+ append_embedding_value(&b2, 0, emb, 1, "float", FFWD_API_PERPLEXITY);
+ TEST_ASSERT(strstr(b2.ptr, "0.4609375") != NULL);
+ sbuf_free(&b2);
+}
+
+int main(void) {
+ test_quantize_int8_tanh();
+ test_encode_embedding_int8();
+ test_encode_embedding_binary();
+ test_encode_embedding_base64_float32();
+ test_append_embedding_value_float();
+ return TEST_REPORT("encode");
+}