diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 20:30:55 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-16 23:38:17 +0200 |
| commit | 1ea03a65ed2a836005ea7a0061db100e470cc69d (patch) | |
| tree | 9274145ea702cdc5d9574e5dc3b2dcaa0bd726ae /src/server_internal.h | |
| parent | 7500042ac2c33e0501e3d2ebae5a423e61cc39d1 (diff) | |
| download | ffwd-1ea03a65ed2a836005ea7a0061db100e470cc69d.tar.gz ffwd-1ea03a65ed2a836005ea7a0061db100e470cc69d.zip | |
all: add SentencePiece XLM-R support
Diffstat (limited to 'src/server_internal.h')
| -rw-r--r-- | src/server_internal.h | 9 |
1 files changed, 9 insertions, 0 deletions
diff --git a/src/server_internal.h b/src/server_internal.h index e4d7992..56bf25b 100644 --- a/src/server_internal.h +++ b/src/server_internal.h @@ -10,6 +10,7 @@ #include "embed.h" #include "sbuf.h" #include "tokenizer_bpe.h" +#include "tokenizer_sentencepiece.h" #include "tokenizer_wordpiece.h" #ifdef USE_MLX @@ -54,6 +55,10 @@ #define EMBED_GTE_QWEN2_QUERY_INSTRUCT \ "Instruct: Given a web search query, retrieve relevant passages that answer the " \ "query\nQuery: " +#define EMBED_E5_QUERY_INSTRUCT \ + "Instruct: Given a web search query, retrieve relevant passages that answer the " \ + "query\nQuery: " +#define EMBED_SNOWFLAKE_QUERY_INSTRUCT "query: " /* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent * short-request throughput over 8 with no long-document penalty, while 128 * was no faster and inflated long-document tail latency. */ @@ -80,6 +85,8 @@ typedef enum { MODEL_BGE_SMALL, MODEL_BGE_BASE, MODEL_BGE_LARGE, + MODEL_E5_MULTI_LARGE_INSTRUCT, + MODEL_SNOWFLAKE_ARCTIC_L_V2, MODEL_CTX_06, MODEL_CTX_4, MODEL_LATE_06, @@ -124,6 +131,8 @@ typedef struct { * Qwen byte-level BPE fields above stay NULL. */ wordpiece_tokenizer_t *wp_tok; wordpiece_workspace_t *wp_tok_ws; + sentencepiece_tokenizer_t *sp_tok; + sentencepiece_workspace_t *sp_tok_ws; int cls_id; int sep_id; int context_separator_id; |
