aboutsummaryrefslogtreecommitdiff
path: root/embed_server.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 13:04:50 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commit4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3 (patch)
tree2e441978cc9badf091976aa53bc64297432e862a /embed_server.c
parent3947e97fbf2a0b79c500ce925bcc9fe15f89899e (diff)
downloadffwd-4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3.tar.gz
ffwd-4db7a34e62afbfbf05d38d66ffcc5fc0f4de06c3.zip
server: resolve contextual separator from vocab, test contextual path
Diffstat (limited to '')
-rw-r--r--embed_server.c9
1 files changed, 8 insertions, 1 deletions
diff --git a/embed_server.c b/embed_server.c
index 2ab22ba..061930c 100644
--- a/embed_server.c
+++ b/embed_server.c
@@ -2151,7 +2151,14 @@ static int load_one_model(http_server *s, model_slot slot, const char *path) {
path);
return -1;
}
- m->context_separator_id = PPLX_CONTEXT_SEPARATOR_TOKEN_ID;
+ /* The contextual chunk separator is the tokenizer's <|endoftext|>. The
+ * released models keep added special tokens out of vocab.json, so the
+ * family constant is the normal case; a vocab that does define the
+ * token (e.g. small test models) takes precedence so the id stays
+ * inside that model's embedding table. */
+ int sep_id = qwen_tokenizer_token_id(m->tok, "<|endoftext|>");
+ m->context_separator_id = sep_id >= 0
+ ? sep_id : PPLX_CONTEXT_SEPARATOR_TOKEN_ID;
#ifdef USE_MLX
if (s->use_mlx) {
/* MLX streams are thread-local. The inference worker loads the MLX