aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:10:56 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 05:57:19 +0200
commit6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch)
treec23afb50c28b9deaddae5c6526a74ee11cf2d865 /src
parentd4bdb8dae753462f621507d5fc3582e0e53da35c (diff)
downloadffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz
ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip
all: split Makefile and reorg library/tools
Diffstat (limited to 'src')
-rw-r--r--src/alloc.c90
-rw-r--r--src/alloc.h34
-rw-r--r--src/base64.c47
-rw-r--r--src/base64.h10
-rw-r--r--src/build.h64
-rw-r--r--src/cli.c911
-rw-r--r--src/config.c321
-rw-r--r--src/config.h8
-rw-r--r--src/cuda.cu2056
-rw-r--r--src/cuda.h80
-rw-r--r--src/dtype.h55
-rw-r--r--src/embed.h357
-rw-r--r--src/forward.c710
-rw-r--r--src/internal.h114
-rw-r--r--src/kernels.h216
-rw-r--r--src/kernels_act.c95
-rw-r--r--src/kernels_attn.c469
-rw-r--r--src/kernels_avx.c439
-rw-r--r--src/kernels_gemm.c434
-rw-r--r--src/kernels_generic.c44
-rw-r--r--src/kernels_impl.h53
-rw-r--r--src/kernels_neon.c179
-rw-r--r--src/kernels_norm.c245
-rw-r--r--src/kernels_rope.c137
-rw-r--r--src/late.c383
-rw-r--r--src/mlx.c2783
-rw-r--r--src/mlx.h167
-rw-r--r--src/model.c688
-rw-r--r--src/model_internal.h76
-rw-r--r--src/safetensors.c559
-rw-r--r--src/safetensors.h76
-rw-r--r--src/sbuf.c74
-rw-r--r--src/sbuf.h24
-rw-r--r--src/server.c1042
-rw-r--r--src/server.h58
-rw-r--r--src/server_encode.c152
-rw-r--r--src/server_handlers.c907
-rw-r--r--src/server_http.c399
-rw-r--r--src/server_internal.h361
-rw-r--r--src/server_json.c221
-rw-r--r--src/server_models.c627
-rw-r--r--src/server_schedule.c341
-rw-r--r--src/server_util.c82
-rw-r--r--src/server_util.h36
-rw-r--r--src/threadpool.c231
-rw-r--r--src/threadpool.h36
-rw-r--r--src/tokenizer_bpe.c2018
-rw-r--r--src/tokenizer_bpe.h86
-rw-r--r--src/tokenizer_sentencepiece.c963
-rw-r--r--src/tokenizer_sentencepiece.h99
-rw-r--r--src/tokenizer_unicode.h4985
-rw-r--r--src/tokenizer_wordpiece.c678
-rw-r--r--src/tokenizer_wordpiece.h79
-rw-r--r--src/vec.c53
54 files changed, 0 insertions, 25452 deletions
diff --git a/src/alloc.c b/src/alloc.c
deleted file mode 100644
index 41a59a4..0000000
--- a/src/alloc.c
+++ /dev/null
@@ -1,90 +0,0 @@
-#include "alloc.h"
-
-#include <stdlib.h>
-#include <stdint.h>
-#include <limits.h>
-
-/* Sequence-capacity growth policy for the forward-pass workspaces: for new
- * growth, never allocate fewer than the minimum, and round up to the
- * granularity so small length changes do not trigger a realloc on every call.
- * Existing larger capacity is preserved. */
-#define EMBED_MIN_WORKSPACE_SEQ_CAP 16
-#define EMBED_WORKSPACE_SEQ_GRANULARITY 16
-
-int mul_size(size_t a, size_t b, size_t *out) {
- if (a != 0 && b > SIZE_MAX / a)
- return -1;
- *out = a * b;
- return 0;
-}
-
-int add_size(size_t a, size_t b, size_t *out) {
- if (b > SIZE_MAX - a)
- return -1;
- *out = a + b;
- return 0;
-}
-
-int grow_cap(int current, int needed, int *out) {
- if (current < 0 || needed <= 0)
- return -1;
-
- int cap = needed;
- if (cap < EMBED_MIN_WORKSPACE_SEQ_CAP)
- cap = EMBED_MIN_WORKSPACE_SEQ_CAP;
-
- int rem = cap % EMBED_WORKSPACE_SEQ_GRANULARITY;
- if (rem != 0) {
- int pad = EMBED_WORKSPACE_SEQ_GRANULARITY - rem;
- if (cap > INT_MAX - pad)
- return -1;
- cap += pad;
- }
-
- if (cap < current)
- cap = current;
-
- *out = cap;
- return 0;
-}
-
-int realloc_floats(float **ptr, size_t count) {
- size_t bytes;
- if (mul_size(count, sizeof(float), &bytes) != 0)
- return -1;
-
- void *p = realloc(*ptr, bytes);
- if (!p && bytes != 0)
- return -1;
- *ptr = (float *)p;
- return 0;
-}
-
-float *malloc_floats(size_t count) {
- size_t bytes;
- if (mul_size(count, sizeof(float), &bytes) != 0)
- return NULL;
- return (float *)malloc(bytes);
-}
-
-int realloc_floats_2d(float **ptr, int rows, int cols) {
- if (rows < 0 || cols < 0)
- return -1;
-
- size_t count;
- if (mul_size((size_t)rows, (size_t)cols, &count) != 0)
- return -1;
- return realloc_floats(ptr, count);
-}
-
-int realloc_ints(int **ptr, size_t count) {
- size_t bytes;
- if (mul_size(count, sizeof(int), &bytes) != 0)
- return -1;
-
- void *p = realloc(*ptr, bytes);
- if (!p && bytes != 0)
- return -1;
- *ptr = (int *)p;
- return 0;
-}
diff --git a/src/alloc.h b/src/alloc.h
deleted file mode 100644
index bc40e8b..0000000
--- a/src/alloc.h
+++ /dev/null
@@ -1,34 +0,0 @@
-#ifndef EMBED_ALLOC_H
-#define EMBED_ALLOC_H
-
-#include <stddef.h>
-
-/* Overflow-checked size arithmetic and typed (re)allocation helpers shared by
- * the model loader, the forward pass, and the late-interaction code. Every
- * helper returns -1 (or NULL) on overflow or allocation failure so callers can
- * bail without a partial buffer. realloc_* keep the original pointer intact on
- * failure; the caller still owns it. */
-
-/* out = a * b, or -1 if the product overflows size_t. */
-int mul_size(size_t a, size_t b, size_t *out);
-
-/* out = a + b, or -1 if the sum overflows size_t. */
-int add_size(size_t a, size_t b, size_t *out);
-
-/* Round a requested sequence capacity up to the workspace growth granularity,
- * never shrinking below the current capacity. Returns -1 on bad input/overflow. */
-int grow_cap(int current, int needed, int *out);
-
-/* realloc *ptr to hold `count` floats; *ptr unchanged on failure. */
-int realloc_floats(float **ptr, size_t count);
-
-/* malloc `count` floats, NULL on overflow/failure. */
-float *malloc_floats(size_t count);
-
-/* realloc *ptr to hold rows*cols floats; -1 on negative dims/overflow/failure. */
-int realloc_floats_2d(float **ptr, int rows, int cols);
-
-/* realloc *ptr to hold `count` ints; *ptr unchanged on failure. */
-int realloc_ints(int **ptr, size_t count);
-
-#endif /* EMBED_ALLOC_H */
diff --git a/src/base64.c b/src/base64.c
deleted file mode 100644
index 082101f..0000000
--- a/src/base64.c
+++ /dev/null
@@ -1,47 +0,0 @@
-#include "base64.h"
-#include "server_util.h"
-
-#include <stdint.h>
-
-char *base64_encode(const unsigned char *src, size_t n) {
- static const char tbl[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/";
- size_t max_groups = (SIZE_MAX - 1) / 4;
- if (n > max_groups * 3)
- die_oom();
-
- size_t out_len = ((n + 2) / 3) * 4;
- char *out = xmalloc(out_len + 1);
- const unsigned char *p = src;
- const unsigned char *end = src + (n / 3) * 3;
- char *q = out;
-
- while (p < end) {
- uint32_t v = ((uint32_t)p[0] << 16) | ((uint32_t)p[1] << 8) | (uint32_t)p[2];
- q[0] = tbl[(v >> 18) & 63];
- q[1] = tbl[(v >> 12) & 63];
- q[2] = tbl[(v >> 6) & 63];
- q[3] = tbl[v & 63];
- p += 3;
- q += 4;
- }
-
- size_t rem = n - (size_t)(p - src);
- if (rem == 1) {
- uint32_t v = (uint32_t)p[0] << 16;
- q[0] = tbl[(v >> 18) & 63];
- q[1] = tbl[(v >> 12) & 63];
- q[2] = '=';
- q[3] = '=';
- q += 4;
- } else if (rem == 2) {
- uint32_t v = ((uint32_t)p[0] << 16) | ((uint32_t)p[1] << 8);
- q[0] = tbl[(v >> 18) & 63];
- q[1] = tbl[(v >> 12) & 63];
- q[2] = tbl[(v >> 6) & 63];
- q[3] = '=';
- q += 4;
- }
-
- *q = '\0';
- return out;
-}
diff --git a/src/base64.h b/src/base64.h
deleted file mode 100644
index 17e2036..0000000
--- a/src/base64.h
+++ /dev/null
@@ -1,10 +0,0 @@
-#ifndef EMBED_BASE64_H
-#define EMBED_BASE64_H
-
-#include <stddef.h>
-
-/* Standard base64 (with '=' padding) of n bytes. Returns a NUL-terminated heap
- * string the caller frees; die-on-OOM, so it never returns NULL. */
-char *base64_encode(const unsigned char *src, size_t n);
-
-#endif /* EMBED_BASE64_H */
diff --git a/src/build.h b/src/build.h
deleted file mode 100644
index dc68405..0000000
--- a/src/build.h
+++ /dev/null
@@ -1,64 +0,0 @@
-/* embed_build.h - build identity for the embed and embed-server front-ends.
- *
- * The real values come from the Makefile's VERSION_CFLAGS at compile time; the
- * fallbacks below apply when those flags are absent. Front-end only: the
- * inference library does not use these. */
-#ifndef EMBED_BUILD_H
-#define EMBED_BUILD_H
-
-#include <stdio.h>
-#include <string.h>
-
-#ifndef EMBED_VERSION
-#define EMBED_VERSION "development"
-#endif
-#ifndef EMBED_BUILD_COMMIT
-#define EMBED_BUILD_COMMIT "unknown"
-#endif
-#ifndef EMBED_BUILD_DATE
-#define EMBED_BUILD_DATE "unknown"
-#endif
-#ifndef EMBED_BUILD_OS
-#define EMBED_BUILD_OS "unknown"
-#endif
-#ifndef EMBED_BUILD_ARCH
-#define EMBED_BUILD_ARCH "unknown"
-#endif
-
-/* Backend label for --build-info, selected by the same macro that picks the
- * compiled inference path. */
-#if defined(USE_MLX)
-#define EMBED_BUILD_BACKENDS "accelerate,mlx"
-#elif defined(USE_CUDA)
-#define EMBED_BUILD_BACKENDS "openblas,cuda"
-#elif defined(__APPLE__)
-#define EMBED_BUILD_BACKENDS "accelerate"
-#else
-#define EMBED_BUILD_BACKENDS "openblas"
-#endif
-
-/* Program name with any leading directory removed. */
-static inline const char *embed_prog_name(const char *argv0)
-{
- const char *slash = strrchr(argv0, '/');
- return slash ? slash + 1 : argv0;
-}
-
-static inline void embed_print_version(const char *prog)
-{
- printf("%s %s (%s/%s)\n", prog, EMBED_VERSION,
- EMBED_BUILD_OS, EMBED_BUILD_ARCH);
-}
-
-static inline void embed_print_build_info(const char *prog)
-{
- embed_print_version(prog);
- printf("backend: %s\n"
- "compiler: %s\n"
- "built: %s\n"
- "commit: %s\n",
- EMBED_BUILD_BACKENDS, __VERSION__,
- EMBED_BUILD_DATE, EMBED_BUILD_COMMIT);
-}
-
-#endif /* EMBED_BUILD_H */
diff --git a/src/cli.c b/src/cli.c
deleted file mode 100644
index 0e3ed7d..0000000
--- a/src/cli.c
+++ /dev/null
@@ -1,911 +0,0 @@
-/* embed_cli.c - embed command-line tool */
-
-#include "embed.h"
-#include "build.h"
-#include "kernels.h"
-#include "tokenizer_bpe.h"
-#include "tokenizer_sentencepiece.h"
-#include "tokenizer_wordpiece.h"
-
-#ifdef USE_MLX
-#include "mlx.h"
-#endif
-#ifdef USE_CUDA
-#include "cuda.h"
-#endif
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <time.h>
-#include <limits.h>
-#include <unistd.h>
-
-static double now_ms(void) {
- struct timespec ts;
- clock_gettime(CLOCK_MONOTONIC, &ts);
- return (double)ts.tv_sec * 1000.0 + (double)ts.tv_nsec / 1e6;
-}
-
-static void print_usage(const char *prog) {
- fprintf(stderr,
- "Usage: %s -d <model_dir> [options] [text...]\n"
- "\n"
- "Options:\n"
- " -d <dir> Model directory (required)\n"
-#ifdef USE_MLX
- " --mlx Use Apple MLX GPU backend\n"
-#endif
-#ifdef USE_CUDA
- " --cuda Use CUDA/cuBLAS GPU backend\n"
-#endif
- " --stream Read lines from stdin, write one JSON embedding per line\n"
- " --json Emit JSON instead of plain text (embedding or matrix)\n"
-#ifdef USE_MLX
- " --mlx-quant-bits N\n"
- " Quantize MLX linear weights to 8 bits at load time\n"
- " --mlx-quant-group-size N\n"
- " MLX quantization group size (default: 64)\n"
-#endif
-#ifdef USE_CUDA
- " --cuda-gemm-mode MODE\n"
- " CUDA GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
- " --cuda-weight-dtype DTYPE\n"
- " CUDA weight storage: f32 or bf16 (default: f32). bf16 halves\n"
- " weight memory and uses BF16 tensor cores\n"
-#endif
- " -b, --batch-size N\n"
- " Max texts per engine batch (default: all; --stream: 1)\n"
- " -t, --threads N\n"
- " CPU threads (default: available cores, cgroup-aware)\n"
- " -e, --embeddings\n"
- " Print raw embeddings (with multiple texts)\n"
- " -v, --verbose\n"
- " Verbose (-vv for debug)\n"
- " -V, --version\n"
- " Print version and exit\n"
- " --build-info Print build details and exit\n"
- " -h Show this help\n"
- "\n"
- "Modes:\n"
- " 1 text arg Embedding as space-separated floats (JSON with --json)\n"
- " 2+ text args Cosine similarity matrix, one row per line (JSON with --json)\n"
- " no args Batch: read all stdin lines, then similarity matrix\n"
- " --stream Streaming: read stdin lines, write JSON per line\n"
- "\n"
- "Examples:\n"
- " %s -d ./model \"what is AI?\"\n"
-#ifdef USE_MLX
- " %s -d ./model --mlx --stream < texts.txt\n",
- prog, prog, prog);
-#else
- ,
- prog, prog);
-#endif
-}
-
-/* ========================================================================
- * Embed one text: tokenize > forward > return float[dim]
- * ======================================================================== */
-
-typedef struct {
- embed_model_t *model;
- embed_workspace_t *workspace;
-#ifdef USE_MLX
- embed_mlx_ctx_t *mlx_ctx;
-#endif
-#ifdef USE_CUDA
- embed_cuda_ctx_t *cuda_ctx;
-#endif
- embed_tokenizer_t *tok;
- embed_tokenizer_workspace_t *tok_ws;
- /* BERT-family WordPiece tokenizer, selected by a vocab.txt probe at load.
- * When wp_tok is set, tokenize_text wraps the ids with [CLS]/[SEP] and the
- * Qwen byte-level BPE fields above stay NULL. */
- wordpiece_tokenizer_t *wp_tok;
- wordpiece_workspace_t *wp_tok_ws;
- /* SentencePiece/XLM-R tokenizer. When sp_tok is set, tokenize_text wraps
- * core ids with <s> ... </s>. */
- sentencepiece_tokenizer_t *sp_tok;
- sentencepiece_workspace_t *sp_tok_ws;
- int cls_id;
- int sep_id;
- int dim;
- int terminal_token_id;
- int append_terminal_token;
-} engine_t;
-
-static void engine_free_tokenizers(engine_t *e) {
- embed_tokenizer_workspace_free(e->tok_ws);
- embed_tokenizer_free(e->tok);
- wordpiece_workspace_free(e->wp_tok_ws);
- wordpiece_tokenizer_free(e->wp_tok);
- sentencepiece_workspace_free(e->sp_tok_ws);
- sentencepiece_tokenizer_free(e->sp_tok);
-}
-
-typedef struct {
- int *ids;
- int n_tokens;
-} token_buf_t;
-
-static int
-engine_embed_batch(engine_t *e, const embed_input_t *inputs, int batch, float *out_embeddings) {
-#ifdef USE_MLX
- if (e->mlx_ctx)
- return embed_mlx_encode_batch(e->mlx_ctx, inputs, batch, out_embeddings);
-#endif
-#ifdef USE_CUDA
- if (e->cuda_ctx)
- return embed_cuda_encode_batch(e->cuda_ctx, inputs, batch, out_embeddings);
-#endif
- return embed_model_encode_batch(e->model, e->workspace, inputs, batch, out_embeddings);
-}
-
-static int tokenize_text(engine_t *e, const char *text, token_buf_t *out) {
- memset(out, 0, sizeof(*out));
- if (e->wp_tok) {
- /* BERT: WordPiece encode, then wrap [CLS] ... [SEP] (the same contract
- * as the BPE terminal-token append below). An empty clean yields the
- * bare pair, matching Hugging Face. Grow the buffer in place. */
- int n = 0;
- int *core = wordpiece_tokenizer_encode_with_workspace(e->wp_tok, e->wp_tok_ws, text, &n);
- if (n < 0 || n > INT_MAX - 2) {
- free(core);
- return -1;
- }
- int *ids = (int *)realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- return -1;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = e->cls_id;
- ids[n + 1] = e->sep_id;
- out->ids = ids;
- out->n_tokens = n + 2;
- } else if (e->sp_tok) {
- int n = 0;
- int *core =
- sentencepiece_tokenizer_encode_with_workspace(e->sp_tok, e->sp_tok_ws, text, &n);
- if (n < 0 || n > INT_MAX - 2) {
- free(core);
- return -1;
- }
- int *ids = (int *)realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- return -1;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = e->cls_id;
- ids[n + 1] = e->sep_id;
- out->ids = ids;
- out->n_tokens = n + 2;
- } else {
- out->ids = embed_tokenizer_encode_with_workspace(e->tok, e->tok_ws, text, &out->n_tokens);
- if (!out->ids || out->n_tokens == 0) {
- fprintf(stderr, "tokenization failed: %s\n", text);
- free(out->ids);
- out->ids = NULL;
- out->n_tokens = 0;
- return -1;
- }
- if (e->append_terminal_token) {
- if (out->n_tokens == INT_MAX) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- int *ids = (int *)realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
- if (!ids) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- out->ids = ids;
- out->ids[out->n_tokens++] = e->terminal_token_id;
- }
- }
-
- if (embed_verbose >= 1) {
- fprintf(stderr, "tokens (%d): ", out->n_tokens);
- for (int i = 0; i < out->n_tokens && i < 20; i++)
- fprintf(stderr, "%d ", out->ids[i]);
- if (out->n_tokens > 20)
- fprintf(stderr, "...");
- fprintf(stderr, "\n");
- }
- return 0;
-}
-
-static void free_tokens(token_buf_t *tokens, int n) {
- if (!tokens)
- return;
- for (int i = 0; i < n; i++)
- free(tokens[i].ids);
-}
-
-/* ========================================================================
- * Output helpers
- * ======================================================================== */
-
-static void print_embedding_raw(const float *emb, int dim) {
- for (int i = 0; i < dim; i++) {
- if (i > 0)
- putchar(' ');
- printf("%.8f", (double)emb[i]);
- }
- putchar('\n');
-}
-
-/* Single embedding as a JSON array of floats. */
-static void print_embedding_array(const float *emb, int dim) {
- putchar('[');
- for (int i = 0; i < dim; i++) {
- if (i)
- putchar(',');
- printf("%.8f", (double)emb[i]);
- }
- puts("]");
-}
-
-static size_t engine_workspace_nbytes(const engine_t *e) {
- return (e && e->workspace) ? embed_workspace_nbytes(e->workspace) : 0;
-}
-
-static void
-print_embedding_json(const float *emb, int dim, int n_tokens, double ms, size_t workspace_bytes) {
- printf("{\"embedding\":[");
- for (int i = 0; i < dim; i++) {
- if (i > 0)
- putchar(',');
- printf("%.8f", (double)emb[i]);
- }
- printf("],\"dim\":%d,\"tokens\":%d,\"ms\":%.1f,\"workspace_bytes\":%zu}\n", dim, n_tokens, ms,
- workspace_bytes);
- fflush(stdout);
-}
-
-/* ========================================================================
- * Stdin mode: read lines from stdin, write JSON to stdout
- * ======================================================================== */
-
-static int process_stdin_batch(engine_t *e, char **lines, int n_lines) {
- if (n_lines <= 0)
- return 0;
-
- token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_lines, sizeof(token_buf_t));
- embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_lines * sizeof(embed_input_t));
- int *input_to_line = (int *)malloc((size_t)n_lines * sizeof(int));
- float *embs = (float *)malloc((size_t)n_lines * e->dim * sizeof(float));
- if (!tokens || !inputs || !input_to_line || !embs) {
- fprintf(stderr, "OOM\n");
- free(tokens);
- free(inputs);
- free(input_to_line);
- free(embs);
- for (int i = 0; i < n_lines; i++)
- printf("{\"error\":\"embedding failed\"}\n");
- fflush(stdout);
- return 1;
- }
-
- int n_inputs = 0;
- for (int i = 0; i < n_lines; i++) {
- if (tokenize_text(e, lines[i], &tokens[i]) == 0) {
- inputs[n_inputs].ids = tokens[i].ids;
- inputs[n_inputs].n_tokens = tokens[i].n_tokens;
- input_to_line[n_inputs] = i;
- n_inputs++;
- }
- }
-
- double ms = 0.0;
- int rc = 0;
- if (n_inputs > 0) {
- double t0 = now_ms();
- rc = engine_embed_batch(e, inputs, n_inputs, embs);
- ms = now_ms() - t0;
- if (embed_verbose >= 1)
- fprintf(stderr, "embed batch: %d texts in %.1f ms\n", n_inputs, ms);
- }
-
- int next_valid = 0;
- for (int i = 0; i < n_lines; i++) {
- if (!tokens[i].ids) {
- printf("{\"error\":\"tokenization failed\"}\n");
- continue;
- }
- if (rc != 0) {
- printf("{\"error\":\"embedding failed\"}\n");
- continue;
- }
- while (next_valid < n_inputs && input_to_line[next_valid] != i)
- next_valid++;
- if (next_valid >= n_inputs) {
- printf("{\"error\":\"embedding failed\"}\n");
- continue;
- }
- print_embedding_json(embs + (size_t)next_valid * e->dim, e->dim, tokens[i].n_tokens, ms,
- engine_workspace_nbytes(e));
- next_valid++;
- }
- fflush(stdout);
-
- free_tokens(tokens, n_lines);
- free(tokens);
- free(inputs);
- free(input_to_line);
- free(embs);
- return rc == 0 ? 0 : 1;
-}
-
-static int run_stdin(engine_t *e, int batch_size) {
- char line[65536];
- if (batch_size <= 0)
- batch_size = 1;
-
- if (embed_verbose >= 1)
- fprintf(stderr, "stdin: ready, reading from stdin (batch_size=%d)\n", batch_size);
-
- char **batch_lines = (char **)calloc((size_t)batch_size, sizeof(char *));
- if (!batch_lines) {
- fprintf(stderr, "OOM\n");
- return 1;
- }
- int n_batch = 0;
- int rc = 0;
-
- while (fgets(line, sizeof(line), stdin)) {
- /* Strip newline */
- size_t len = strlen(line);
- while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r'))
- line[--len] = '\0';
- if (len == 0)
- continue;
-
- if (embed_verbose >= 1)
- fprintf(stderr, "stdin: \"%.*s%s\"\n", (int)(len > 60 ? 60 : len), line,
- len > 60 ? "..." : "");
-
- batch_lines[n_batch] = strdup(line);
- if (!batch_lines[n_batch]) {
- fprintf(stderr, "OOM\n");
- rc = 1;
- break;
- }
- n_batch++;
-
- if (n_batch == batch_size) {
- if (process_stdin_batch(e, batch_lines, n_batch) != 0)
- rc = 1;
- for (int i = 0; i < n_batch; i++) {
- free(batch_lines[i]);
- batch_lines[i] = NULL;
- }
- n_batch = 0;
- }
- }
-
- if (n_batch > 0) {
- if (process_stdin_batch(e, batch_lines, n_batch) != 0)
- rc = 1;
- for (int i = 0; i < n_batch; i++)
- free(batch_lines[i]);
- }
- free(batch_lines);
-
- if (embed_verbose >= 1)
- fprintf(stderr, "stdin: EOF\n");
-
- return rc;
-}
-
-/* ========================================================================
- * Batch mode: embed args or stdin lines, then print similarity or vectors
- * ======================================================================== */
-
-static int append_text(char ***texts, int *n_texts, int *cap, const char *s) {
- if (*n_texts == *cap) {
- int new_cap = *cap ? *cap * 2 : 8;
- char **new_texts = (char **)realloc(*texts, (size_t)new_cap * sizeof(char *));
- if (!new_texts)
- return -1;
- *texts = new_texts;
- *cap = new_cap;
- }
-
- (*texts)[*n_texts] = strdup(s);
- if (!(*texts)[*n_texts])
- return -1;
- (*n_texts)++;
- return 0;
-}
-
-/* Cosine similarity matrix. --json emits a JSON array of rows; otherwise one
- * row of space-separated values per line. With print_embs the raw embeddings
- * follow, one per line after a blank line (plain-text output only). */
-static void print_matrix(const float *embs, int n, int dim, int json, int print_embs) {
- if (json) {
- putchar('[');
- for (int i = 0; i < n; i++) {
- if (i)
- putchar(',');
- putchar('[');
- for (int j = 0; j < n; j++) {
- if (j)
- putchar(',');
- printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim,
- embs + (size_t)j * dim, dim));
- }
- putchar(']');
- }
- puts("]");
- return;
- }
-
- for (int i = 0; i < n; i++) {
- for (int j = 0; j < n; j++) {
- if (j)
- putchar(' ');
- printf("%.6f", (double)embed_cosine_similarity(embs + (size_t)i * dim,
- embs + (size_t)j * dim, dim));
- }
- putchar('\n');
- }
- if (print_embs) {
- putchar('\n');
- for (int i = 0; i < n; i++)
- print_embedding_raw(embs + (size_t)i * dim, dim);
- }
-}
-
-static int run_batch(
- engine_t *e, int argc, char **argv, int arg_start, int print_embs, int batch_size, int json) {
- int n_texts = 0, cap = 0;
- char **texts = NULL;
- int rc = 0;
-
- if (arg_start < argc) {
- for (int i = arg_start; i < argc; i++) {
- if (append_text(&texts, &n_texts, &cap, argv[i]) != 0)
- goto oom_texts;
- }
- } else {
- char line[65536];
- while (fgets(line, sizeof(line), stdin)) {
- size_t l = strlen(line);
- while (l > 0 && (line[l - 1] == '\n' || line[l - 1] == '\r'))
- line[--l] = '\0';
- if (l == 0)
- continue;
- if (append_text(&texts, &n_texts, &cap, line) != 0)
- goto oom_texts;
- }
- }
-
- if (n_texts == 0) {
- fprintf(stderr, "no input texts\n");
- free(texts);
- return 1;
- }
-
- int dim = e->dim;
- token_buf_t *tokens = (token_buf_t *)calloc((size_t)n_texts, sizeof(token_buf_t));
- embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_texts * sizeof(embed_input_t));
- float *embs = (float *)malloc((size_t)n_texts * dim * sizeof(float));
- if (!tokens || !inputs || !embs) {
- fprintf(stderr, "OOM\n");
- free(tokens);
- free(inputs);
- free(embs);
- for (int i = 0; i < n_texts; i++)
- free(texts[i]);
- free(texts);
- return 1;
- }
-
- for (int i = 0; i < n_texts; i++) {
- if (embed_verbose >= 1)
- fprintf(stderr, "[%d/%d] \"%s\"\n", i + 1, n_texts, texts[i]);
- if (tokenize_text(e, texts[i], &tokens[i]) != 0) {
- rc = 1;
- goto done;
- }
- inputs[i].ids = tokens[i].ids;
- inputs[i].n_tokens = tokens[i].n_tokens;
- }
-
- int max_batch = batch_size > 0 ? batch_size : n_texts;
- for (int start = 0; start < n_texts; start += max_batch) {
- int cur = n_texts - start;
- if (cur > max_batch)
- cur = max_batch;
-
- double t0 = now_ms();
- if (engine_embed_batch(e, inputs + start, cur, embs + (size_t)start * dim) != 0) {
- fprintf(stderr, "forward pass failed\n");
- rc = 1;
- goto done;
- }
- if (embed_verbose >= 1)
- fprintf(stderr, "embed batch: [%d..%d] %d texts in %.1f ms\n", start, start + cur - 1,
- cur, now_ms() - t0);
- }
-
- if (n_texts == 1) {
- if (json)
- print_embedding_array(embs, dim);
- else
- print_embedding_raw(embs, dim);
- } else {
- print_matrix(embs, n_texts, dim, json, print_embs);
- }
-
-done:
- free_tokens(tokens, n_texts);
- for (int i = 0; i < n_texts; i++)
- free(texts[i]);
- free(texts);
- free(tokens);
- free(inputs);
- free(embs);
- return rc;
-
-oom_texts:
- fprintf(stderr, "OOM\n");
- for (int i = 0; i < n_texts; i++)
- free(texts[i]);
- free(texts);
- return 1;
-}
-
-/* ========================================================================
- * Main
- * ======================================================================== */
-
-int main(int argc, char *argv[]) {
- const char *model_dir = NULL;
- int n_threads = 0;
- int print_embs = 0;
- int verbose = 0;
- int stdin_mode = 0;
- int batch_size = 0;
- int json = 0;
- const char *prog = embed_prog_name(argv[0]);
-#ifdef USE_MLX
- int use_mlx = 0;
- int mlx_quantize_bits = 0;
- int mlx_quantize_group_size = 64;
-#endif
-#ifdef USE_CUDA
- int use_cuda = 0;
- const char *cuda_fast_gemm = NULL;
- const char *cuda_weights = NULL;
-#endif
-
- int arg_start = 1;
- while (arg_start < argc && argv[arg_start][0] == '-') {
- const char *f = argv[arg_start];
- if (!strcmp(f, "-d")) {
- model_dir = argv[++arg_start];
- } else if (!strcmp(f, "-V") || !strcmp(f, "--version")) {
- embed_print_version(prog);
- return 0;
- } else if (!strcmp(f, "--build-info")) {
- embed_print_build_info(prog);
- return 0;
- } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
- n_threads = atoi(argv[++arg_start]);
- } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
- batch_size = atoi(argv[++arg_start]);
- } else if (!strcmp(f, "--json")) {
- json = 1;
- }
-#ifdef USE_MLX
- else if (!strcmp(f, "--mlx-quant-bits")) {
- mlx_quantize_bits = atoi(argv[++arg_start]);
- if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
- fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
- return 1;
- }
- } else if (!strcmp(f, "--mlx-quant-group-size")) {
- mlx_quantize_group_size = atoi(argv[++arg_start]);
- if (mlx_quantize_group_size <= 0) {
- fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
- return 1;
- }
- }
-#endif
-#ifdef USE_CUDA
- else if (!strcmp(f, "--cuda-gemm-mode")) {
- cuda_fast_gemm = argv[++arg_start];
- } else if (!strcmp(f, "--cuda-weight-dtype")) {
- cuda_weights = argv[++arg_start];
- }
-#endif
- else if (!strcmp(f, "-e") || !strcmp(f, "--embeddings")) {
- print_embs = 1;
- } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) {
- verbose++;
- } else if (!strcmp(f, "-vv")) {
- verbose = 2;
- } else if (!strcmp(f, "--stream")) {
- stdin_mode = 1;
- }
-#ifdef USE_MLX
- else if (!strcmp(f, "--mlx")) {
- use_mlx = 1;
- }
-#endif
-#ifdef USE_CUDA
- else if (!strcmp(f, "--cuda")) {
- use_cuda = 1;
- }
-#endif
- else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
- print_usage(prog);
- return 0;
- } else if (!strcmp(f, "--")) {
- /* End of options: everything after is positional text, even if it
- * starts with a dash. */
- arg_start++;
- break;
- } else if (f[0] == '-' && f[1] == '-') {
- /* An unrecognized --option is a hard error, not silently-embedded
- * text. Catches typos and stale flags (e.g. the removed --backend)
- * loudly instead of running the default path or embedding the flag. */
- fprintf(stderr, "unknown option: %s\n", f);
- print_usage(prog);
- return 1;
- } else
- break;
- arg_start++;
- }
-
- embed_verbose = verbose;
-
-#ifdef USE_CUDA
- if (cuda_fast_gemm) {
- if (!use_cuda) {
- fprintf(stderr, "--cuda-gemm-mode requires --cuda\n");
- return 1;
- }
- if (embed_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
- fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n");
- return 1;
- }
- }
- if (cuda_weights) {
- if (!use_cuda) {
- fprintf(stderr, "--cuda-weight-dtype requires --cuda\n");
- return 1;
- }
- if (!strcmp(cuda_weights, "bf16")) {
- embed_cuda_set_weights_bf16(1);
- } else if (!strcmp(cuda_weights, "f32")) {
- embed_cuda_set_weights_bf16(0);
- } else {
- fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n");
- return 1;
- }
- }
-#endif
-#ifdef USE_MLX
- if (mlx_quantize_bits && !use_mlx) {
- fprintf(stderr, "--mlx-quant-bits requires --mlx\n");
- return 1;
- }
-#endif
-
- if (!model_dir || !model_dir[0]) {
- fprintf(stderr, "model directory required (-d <dir>)\n");
- print_usage(prog);
- return 1;
- }
-
-#ifdef USE_MLX
- if (use_mlx) {
- if (verbose >= 1)
- fprintf(stderr, "Using MLX GPU backend\n");
- } else
-#endif
-#ifdef USE_CUDA
- if (use_cuda) {
- if (verbose >= 1)
- fprintf(stderr, "Using CUDA GPU backend\n");
- } else
-#endif
- {
- if (n_threads <= 0)
- n_threads = embed_get_num_cpus();
- embed_set_threads(n_threads);
- if (verbose >= 1)
- fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
- }
-
- /* Tokenizer selection by files present: WordPiece vocab.txt for BERT,
- * SentencePiece .model for XLM-R/RoBERTa, otherwise Qwen byte-level BPE. */
- engine_t e = {0};
- e.cls_id = -1;
- e.sep_id = -1;
- double t0 = now_ms();
- char wp_path[1024];
- snprintf(wp_path, sizeof(wp_path), "%s/vocab.txt", model_dir);
- if (access(wp_path, R_OK) == 0) {
- e.wp_tok = wordpiece_tokenizer_load(model_dir);
- if (e.wp_tok) {
- e.cls_id = wordpiece_tokenizer_token_id(e.wp_tok, "[CLS]");
- e.sep_id = wordpiece_tokenizer_token_id(e.wp_tok, "[SEP]");
- }
- if (!e.wp_tok || e.cls_id < 0 || e.sep_id < 0) {
- fprintf(stderr, "failed to load WordPiece tokenizer: %s\n", wp_path);
- wordpiece_tokenizer_free(e.wp_tok);
- return 1;
- }
- } else {
- char sp_path[1024];
- snprintf(sp_path, sizeof(sp_path), "%s/sentencepiece.bpe.model", model_dir);
- if (access(sp_path, R_OK) != 0) {
- snprintf(sp_path, sizeof(sp_path), "%s/tokenizer.model", model_dir);
- if (access(sp_path, R_OK) != 0)
- snprintf(sp_path, sizeof(sp_path), "%s/spiece.model", model_dir);
- }
- if (access(sp_path, R_OK) == 0) {
- e.sp_tok = sentencepiece_tokenizer_load(model_dir);
- if (e.sp_tok) {
- e.cls_id = sentencepiece_tokenizer_token_id(e.sp_tok, "<s>");
- e.sep_id = sentencepiece_tokenizer_token_id(e.sp_tok, "</s>");
- }
- if (!e.sp_tok || e.cls_id < 0 || e.sep_id < 0) {
- fprintf(stderr, "failed to load SentencePiece tokenizer: %s\n", sp_path);
- sentencepiece_tokenizer_free(e.sp_tok);
- return 1;
- }
- }
- }
- if (!e.wp_tok && !e.sp_tok) {
- char vocab_path[1024];
- snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", model_dir);
- e.tok = embed_tokenizer_load(vocab_path);
- if (!e.tok) {
- fprintf(stderr, "failed to load tokenizer: %s\n", vocab_path);
- return 1;
- }
- }
- if (verbose >= 1)
- fprintf(stderr, "Tokenizer: %.0f ms\n", now_ms() - t0);
-
- /* Model */
- t0 = now_ms();
- const embed_config_t *config = NULL;
-
-#ifdef USE_MLX
- if (use_mlx) {
- embed_mlx_options_t mlx_opts = {
- .quantize_bits = mlx_quantize_bits,
- .quantize_group_size = mlx_quantize_group_size,
- };
- e.mlx_ctx = embed_mlx_load_with_options(model_dir, &mlx_opts);
- if (!e.mlx_ctx) {
- fprintf(stderr, "failed to load model\n");
- engine_free_tokenizers(&e);
- return 1;
- }
- config = embed_mlx_config(e.mlx_ctx);
- e.dim = config->hidden_size;
- } else
-#endif
-#ifdef USE_CUDA
- if (use_cuda) {
- e.cuda_ctx = embed_cuda_load(model_dir);
- if (!e.cuda_ctx) {
- fprintf(stderr, "failed to load CUDA model\n");
- engine_free_tokenizers(&e);
- return 1;
- }
- config = embed_cuda_config(e.cuda_ctx);
- e.dim = config->hidden_size;
- } else
-#endif
- {
- e.model = embed_model_load(model_dir);
- if (!e.model) {
- fprintf(stderr, "failed to load model\n");
- engine_free_tokenizers(&e);
- return 1;
- }
- e.workspace = embed_workspace_new(e.model);
- if (!e.workspace) {
- fprintf(stderr, "failed to allocate workspace\n");
- embed_model_free(e.model);
- engine_free_tokenizers(&e);
- return 1;
- }
- config = embed_model_config(e.model);
- e.dim = config->hidden_size;
- }
- e.append_terminal_token = config->append_terminal_token;
- /* The appended terminal token is the tokenizer's <|endoftext|>, resolved
- * like the contextual separator: a vocab that defines it (test models)
- * wins; otherwise the released-model family constant applies. */
- if (e.append_terminal_token) {
- if (!e.tok) {
- fprintf(stderr, "terminal-token appending requires the BPE tokenizer\n");
- if (e.workspace)
- embed_workspace_free(e.workspace);
- if (e.model)
- embed_model_free(e.model);
-#ifdef USE_MLX
- if (e.mlx_ctx)
- embed_mlx_free(e.mlx_ctx);
-#endif
-#ifdef USE_CUDA
- if (e.cuda_ctx)
- embed_cuda_free(e.cuda_ctx);
-#endif
- engine_free_tokenizers(&e);
- return 1;
- }
- int eot = embed_tokenizer_token_id(e.tok, "<|endoftext|>");
- e.terminal_token_id = eot >= 0 ? eot : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
- }
- if (verbose >= 1)
- fprintf(stderr, "Model: %d-dim, %.0f ms%s\n", e.dim, now_ms() - t0,
-#ifdef USE_MLX
- use_mlx ? " (MLX)" :
-#endif
-#ifdef USE_CUDA
- use_cuda ? " (CUDA)"
- :
-#endif
- "");
-
- if (e.wp_tok)
- e.wp_tok_ws = wordpiece_workspace_new();
- else if (e.sp_tok)
- e.sp_tok_ws = sentencepiece_workspace_new();
- else
- e.tok_ws = embed_tokenizer_workspace_new();
- if (!e.tok_ws && !e.wp_tok_ws && !e.sp_tok_ws) {
- fprintf(stderr, "failed to allocate tokenizer workspace\n");
- if (e.workspace)
- embed_workspace_free(e.workspace);
- if (e.model)
- embed_model_free(e.model);
-#ifdef USE_MLX
- if (e.mlx_ctx)
- embed_mlx_free(e.mlx_ctx);
-#endif
-#ifdef USE_CUDA
- if (e.cuda_ctx)
- embed_cuda_free(e.cuda_ctx);
-#endif
- engine_free_tokenizers(&e);
- return 1;
- }
-
- /* Run */
- int rc;
- if (stdin_mode)
- rc = run_stdin(&e, batch_size > 0 ? batch_size : 1);
- else
- rc = run_batch(&e, argc, argv, arg_start, print_embs, batch_size, json);
-
- /* Cleanup */
- if (e.workspace)
- embed_workspace_free(e.workspace);
- if (e.model)
- embed_model_free(e.model);
-#ifdef USE_MLX
- if (e.mlx_ctx)
- embed_mlx_free(e.mlx_ctx);
-#endif
-#ifdef USE_CUDA
- if (e.cuda_ctx)
- embed_cuda_free(e.cuda_ctx);
-#endif
- engine_free_tokenizers(&e);
- return rc;
-}
diff --git a/src/config.c b/src/config.c
deleted file mode 100644
index a33a135..0000000
--- a/src/config.c
+++ /dev/null
@@ -1,321 +0,0 @@
-#include "config.h"
-
-#include <ctype.h>
-#include <errno.h>
-#include <limits.h>
-#include <math.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-typedef enum {
- JSON_BOOL_MISSING = 0,
- JSON_BOOL_FALSE = 1,
- JSON_BOOL_TRUE = 2,
- JSON_BOOL_INVALID = -1,
-} json_bool_t;
-
-static const char *skip_ws(const char *p) {
- while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')
- p++;
- return p;
-}
-
-static const char *json_find_key(const char *json, const char *key) {
- size_t klen = strlen(key);
- const char *p = json;
- while ((p = strstr(p, key)) != NULL) {
- if (p > json && p[-1] == '"') {
- const char *after = p + klen;
- if (*after == '"') {
- after = skip_ws(after + 1);
- if (*after == ':')
- return skip_ws(after + 1);
- }
- }
- p += klen;
- }
- return NULL;
-}
-
-static int json_get_int(const char *json, const char *key, int fallback) {
- const char *v = json_find_key(json, key);
- return v ? atoi(v) : fallback;
-}
-
-static double json_get_double(const char *json, const char *key, double fallback) {
- const char *v = json_find_key(json, key);
- return v ? atof(v) : fallback;
-}
-
-static int json_string_equals(const char *json, const char *key, const char *expected) {
- const char *v = json_find_key(json, key);
- if (!v || *v != '"')
- return 0;
- v++;
- size_t len = strlen(expected);
- return strncmp(v, expected, len) == 0 && v[len] == '"';
-}
-
-static json_bool_t json_get_bool(const char *json, const char *key) {
- const char *v = json_find_key(json, key);
- if (!v)
- return JSON_BOOL_MISSING;
- if (strncmp(v, "true", 4) == 0 && !isalnum((unsigned char)v[4]) && v[4] != '_')
- return JSON_BOOL_TRUE;
- if (strncmp(v, "false", 5) == 0 && !isalnum((unsigned char)v[5]) && v[5] != '_')
- return JSON_BOOL_FALSE;
- return JSON_BOOL_INVALID;
-}
-
-/* Return 1 with a NUL-terminated buffer, 0 when an optional file is absent,
- * and -1 for an existing file that could not be read. */
-static int read_optional_file(const char *model_dir, const char *relative, char **out) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative);
- if (n < 0 || (size_t)n >= sizeof(path)) {
- fprintf(stderr, "embed_config: model path too long\n");
- return -1;
- }
-
- FILE *f = fopen(path, "rb");
- if (!f && errno == ENOENT)
- return 0;
- if (!f) {
- fprintf(stderr, "embed_config: cannot open %s\n", path);
- return -1;
- }
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return -1;
- }
- long sz = ftell(f);
- if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return -1;
- }
- char *buf = (char *)malloc((size_t)sz + 1);
- if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
- fclose(f);
- free(buf);
- return -1;
- }
- fclose(f);
- buf[sz] = '\0';
- *out = buf;
- return 1;
-}
-
-static int parse_sentence_transformers_metadata(embed_config_t *cfg, const char *model_dir) {
- char *buf = NULL;
- int rc = read_optional_file(model_dir, "1_Pooling/config.json", &buf);
- if (rc < 0)
- return -1;
- if (rc > 0) {
- json_bool_t mean = json_get_bool(buf, "pooling_mode_mean_tokens");
- json_bool_t last = json_get_bool(buf, "pooling_mode_lasttoken");
- json_bool_t cls = json_get_bool(buf, "pooling_mode_cls_token");
- json_bool_t max = json_get_bool(buf, "pooling_mode_max_tokens");
- json_bool_t sqrt_len = json_get_bool(buf, "pooling_mode_mean_sqrt_len_tokens");
- json_bool_t weighted = json_get_bool(buf, "pooling_mode_weightedmean_tokens");
- if (mean < 0 || last < 0 || cls < 0 || max < 0 || sqrt_len < 0 || weighted < 0) {
- fprintf(stderr, "embed_config: invalid pooling metadata\n");
- free(buf);
- return -1;
- }
- int supported =
- (mean == JSON_BOOL_TRUE) + (last == JSON_BOOL_TRUE) + (cls == JSON_BOOL_TRUE);
- int unsupported =
- max == JSON_BOOL_TRUE || sqrt_len == JSON_BOOL_TRUE || weighted == JSON_BOOL_TRUE;
- if (supported != 1 || unsupported) {
- fprintf(stderr, "embed_config: unsupported Sentence Transformers pooling mode\n");
- free(buf);
- return -1;
- }
- cfg->pooling_mode = cls == JSON_BOOL_TRUE ? EMBED_POOL_CLS
- : last == JSON_BOOL_TRUE ? EMBED_POOL_LAST_TOKEN
- : EMBED_POOL_MEAN;
- free(buf);
- buf = NULL;
- }
-
- rc = read_optional_file(model_dir, "modules.json", &buf);
- if (rc < 0)
- return -1;
- if (rc > 0) {
- cfg->normalize_embeddings =
- strstr(buf, "\"sentence_transformers.models.Normalize\"") != NULL;
- free(buf);
- buf = NULL;
- }
-
- rc = read_optional_file(model_dir, "tokenizer_config.json", &buf);
- if (rc < 0)
- return -1;
- if (rc > 0) {
- json_bool_t add_eos = json_get_bool(buf, "add_eos_token");
- if (add_eos < 0) {
- fprintf(stderr, "embed_config: invalid add_eos_token metadata\n");
- free(buf);
- return -1;
- }
- if (add_eos != JSON_BOOL_MISSING)
- cfg->append_terminal_token = add_eos == JSON_BOOL_TRUE;
- free(buf);
- }
- return 0;
-}
-
-int embed_config_parse(embed_config_t *cfg, const char *model_dir) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/config.json", model_dir);
- if (n < 0 || (size_t)n >= sizeof(path)) {
- fprintf(stderr, "embed_config: model path too long\n");
- return -1;
- }
-
- FILE *f = fopen(path, "rb");
- if (!f) {
- fprintf(stderr, "embed_config: cannot open %s\n", path);
- return -1;
- }
-
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return -1;
- }
- long sz = ftell(f);
- if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return -1;
- }
- char *buf = (char *)malloc((size_t)sz + 1);
- if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
- fclose(f);
- free(buf);
- return -1;
- }
- fclose(f);
- buf[sz] = '\0';
-
- cfg->hidden_size = json_get_int(buf, "hidden_size", 0);
- cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0);
- cfg->n_heads = json_get_int(buf, "num_attention_heads", 0);
- cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0);
- cfg->head_dim = json_get_int(buf, "head_dim", EMBED_HEAD_DIM);
- cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0);
- cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE);
- cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6);
- cfg->layer_norm_eps = (float)json_get_double(buf, "layer_norm_eps", 1e-12);
- cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0);
- cfg->max_position_embeddings = json_get_int(buf, "max_position_embeddings", 0);
- cfg->position_id_offset = 0;
- cfg->type_vocab_size = json_get_int(buf, "type_vocab_size", 2);
- cfg->family = EMBED_FAMILY_QWEN3;
- cfg->qk_norm = 1;
- cfg->qkv_bias = 0;
- cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
- cfg->pooling_mode = EMBED_POOL_MEAN;
- cfg->normalize_embeddings = 0;
- cfg->append_terminal_token = 0;
-
- if (json_string_equals(buf, "model_type", "qwen3")) {
- cfg->attention_mode = EMBED_ATTENTION_CAUSAL;
- cfg->pooling_mode = EMBED_POOL_LAST_TOKEN;
- cfg->normalize_embeddings = 1;
- cfg->append_terminal_token = 1;
- } else if (json_string_equals(buf, "model_type", "qwen2")) {
- cfg->qk_norm = 0;
- cfg->qkv_bias = 1;
- cfg->attention_mode = EMBED_ATTENTION_CAUSAL;
- } else if (json_string_equals(buf, "model_type", "bert") ||
- json_string_equals(buf, "model_type", "roberta") ||
- json_string_equals(buf, "model_type", "xlm-roberta")) {
- cfg->family = EMBED_FAMILY_BERT;
- cfg->qk_norm = 0;
- cfg->qkv_bias = 1; /* q/k/v/o and both dense layers carry bias */
- cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
- cfg->n_kv_heads = cfg->n_heads; /* full multi-head attention, no GQA */
- cfg->head_dim = cfg->n_heads > 0 ? cfg->hidden_size / cfg->n_heads : 0;
- if (json_string_equals(buf, "model_type", "roberta") ||
- json_string_equals(buf, "model_type", "xlm-roberta")) {
- int pad_id = json_get_int(buf, "pad_token_id", 1);
- if (pad_id < 0 || pad_id == INT_MAX) {
- fprintf(stderr, "embed_config: invalid RoBERTa pad_token_id\n");
- free(buf);
- return -1;
- }
- cfg->position_id_offset = pad_id + 1;
- }
- /* hidden_act selects the feed-forward GeLU curve. gelu_new and
- * gelu_pytorch_tanh are the same tanh approximation; everything else
- * (gelu, absent) is the exact erf GeLU the released encoders use. */
- if (json_string_equals(buf, "hidden_act", "gelu_new") ||
- json_string_equals(buf, "hidden_act", "gelu_pytorch_tanh")) {
- cfg->ffn_act = EMBED_ACT_GELU_TANH;
- }
- if (cfg->max_position_embeddings <= 0 ||
- cfg->position_id_offset >= cfg->max_position_embeddings ||
- cfg->type_vocab_size <= 0 || !isfinite(cfg->layer_norm_eps) ||
- cfg->layer_norm_eps <= 0.0f) {
- fprintf(stderr, "embed_config: invalid BERT layer_norm_eps/max_position_embeddings\n");
- free(buf);
- return -1;
- }
- }
-
- json_bool_t is_causal = json_get_bool(buf, "is_causal");
- if (is_causal < 0) {
- fprintf(stderr, "embed_config: invalid is_causal in %s\n", path);
- free(buf);
- return -1;
- }
- if (is_causal != JSON_BOOL_MISSING) {
- cfg->attention_mode =
- is_causal == JSON_BOOL_TRUE ? EMBED_ATTENTION_CAUSAL : EMBED_ATTENTION_BIDIRECTIONAL;
- }
-
- cfg->q_dim = 0;
- cfg->kv_dim = 0;
- if (cfg->n_heads > 0 && cfg->head_dim > 0 && cfg->n_heads <= INT_MAX / cfg->head_dim)
- cfg->q_dim = cfg->n_heads * cfg->head_dim;
- if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && cfg->n_kv_heads <= INT_MAX / cfg->head_dim)
- cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim;
-
- free(buf);
-
- if (parse_sentence_transformers_metadata(cfg, model_dir) != 0)
- return -1;
-
- if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 ||
- cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 ||
- cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) ||
- cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) ||
- cfg->rms_norm_eps <= 0.0f || !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
- fprintf(stderr,
- "embed_config: invalid config in %s "
- "(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n",
- path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->head_dim,
- cfg->intermediate_size);
- return -1;
- }
- if (cfg->n_layers > EMBED_MAX_LAYERS) {
- fprintf(stderr, "embed_config: too many layers (%d > %d)\n", cfg->n_layers,
- EMBED_MAX_LAYERS);
- return -1;
- }
-
- if (embed_verbose >= 1)
- fprintf(stderr,
- "config: hidden=%d, layers=%d, heads=%d/%d, "
- "inter=%d, head_dim=%d, attention=%s, pooling=%s, qk_norm=%s, qkv_bias=%s\n",
- cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads,
- cfg->intermediate_size, cfg->head_dim,
- cfg->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "bidirectional",
- cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token"
- : cfg->pooling_mode == EMBED_POOL_CLS ? "cls"
- : "mean",
- cfg->qk_norm ? "yes" : "no", cfg->qkv_bias ? "yes" : "no");
-
- return 0;
-}
diff --git a/src/config.h b/src/config.h
deleted file mode 100644
index e8353d3..0000000
--- a/src/config.h
+++ /dev/null
@@ -1,8 +0,0 @@
-#ifndef EMBED_CONFIG_H
-#define EMBED_CONFIG_H
-
-#include "embed.h"
-
-int embed_config_parse(embed_config_t *cfg, const char *model_dir);
-
-#endif /* EMBED_CONFIG_H */
diff --git a/src/cuda.cu b/src/cuda.cu
deleted file mode 100644
index 872834b..0000000
--- a/src/cuda.cu
+++ /dev/null
@@ -1,2056 +0,0 @@
-extern "C" {
-#include "cuda.h"
-#include "internal.h"
-}
-
-#include <cuda_runtime.h>
-#include <cuda_bf16.h>
-#include <cublas_v2.h>
-
-#include <limits.h>
-#include <math.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-typedef struct {
- float *d; // device buffer; holds __nv_bfloat16 elements when bf16
- int rows;
- int cols;
- int bf16; // 1 if d holds __nv_bfloat16, 0 if F32
-} cuda_matrix_t;
-
-typedef struct {
- cuda_matrix_t qkv, wo;
- float *qkv_bias;
- float *q_norm;
- float *k_norm;
- float *input_norm;
- float *post_attn_norm;
- cuda_matrix_t gate_up_proj, down_proj;
- /* BERT family (NULL for Qwen3). The two block LayerNorm weights reuse
- * input_norm (post-attention) and post_attn_norm (post-FFN); gate_up_proj
- * holds the single BERT up_proj. These add the biases those slots lack. */
- float *o_bias;
- float *attn_ln_bias;
- float *ffn_inter_bias;
- float *ffn_out_bias;
- float *ffn_ln_bias;
-} cuda_layer_t;
-
-struct embed_cuda_ctx {
- embed_model_t *cpu;
- int own_cpu; // free ctx->cpu in embed_cuda_free only when this context owns it
- embed_config_t config;
- cudaStream_t stream;
- cublasHandle_t blas;
- cuda_matrix_t embed_tokens;
- cuda_layer_t *layers;
- float *norm;
- /* BERT family (NULL for Qwen3): learned absolute position embeddings, the
- * token-type[0] embedding (row 0 only, [hidden]), and the embedding
- * LayerNorm weight/bias. */
- float *position_embeddings;
- float *token_type_embedding;
- float *embed_ln_w;
- float *embed_ln_b;
-
- int seq_cap;
- int batch_cap;
- int max_seq_cap;
- float *x;
- float *x_norm;
- float *qkv;
- float *attn_out;
- float *ffn_gate_up;
- void *act_bf16; // BF16 cast of a GEMM activation operand (bf16 weights)
- float *weight_f32; // F32 widen scratch for one BF16 weight (memory-only bf16)
- size_t weight_f32_elems;
- float *pooled_out;
- float *rope_cos;
- float *rope_sin;
- int *token_ids;
- int *offsets;
- int *offsets_host;
- int *positions;
- float *kexp;
- float *vexp;
- float *attn_scores;
- void *attn_probs; // BF16 softmax output (bf16 weights)
- long long attn_scores_elems;
- long long attn_probs_elems;
- // Equal-length micro-batch attention: pointer arrays for batched GEMMs,
- // built once per forward pass (layer-invariant buffer addresses).
- // attn_G is the number of sequences whose score tensors fit the scratch
- // budget at once; 0 disables the batched path for this forward.
- const void **attn_ptrs; // device: [K|Q|S|V|P|O] x batch x n_heads
- const void **attn_ptrs_host;
- int attn_ptrs_cap; // capacity in pointer entries
- int attn_G;
- int attn_L;
- int *span_starts;
- int *span_lens;
- int pooled_rows_cap;
- int span_cap;
-};
-
-// Scratch budget for the batched-attention score tensors (F32 elements).
-enum { CUDA_ATTN_SCORES_BUDGET = 64 * 1024 * 1024 };
-// Above this sequence length the per-sequence strided-batched loop beats the
-// pointer-array batched GEMMs (L4: batched wins <=80tok, ties ~140, loses
-// >=256; both paths are identical precision).
-enum { CUDA_ATTN_BATCHED_MAX_LEN = 192 };
-
-#define CUDA_CHECK(expr) \
- do { \
- cudaError_t _e = (expr); \
- if (_e != cudaSuccess) { \
- fprintf(stderr, "cuda: %s failed: %s\n", #expr, cudaGetErrorString(_e)); \
- return -1; \
- } \
- } while (0)
-
-#define CUBLAS_CHECK(expr) \
- do { \
- cublasStatus_t _s = (expr); \
- if (_s != CUBLAS_STATUS_SUCCESS) { \
- fprintf(stderr, "cuda: %s failed: cublas status %d\n", #expr, (int)_s); \
- return -1; \
- } \
- } while (0)
-
-static int parse_gemm_compute(const char *mode, cublasComputeType_t *out) {
- if (!mode || !mode[0] || !strcmp(mode, "f32")) {
- *out = CUBLAS_COMPUTE_32F;
- return 0;
- }
- if (!strcmp(mode, "tf32")) {
- *out = CUBLAS_COMPUTE_32F_FAST_TF32;
- return 0;
- }
- if (!strcmp(mode, "bf16")) {
- *out = CUBLAS_COMPUTE_32F_FAST_16BF;
- return 0;
- }
- if (!strcmp(mode, "16f")) {
- *out = CUBLAS_COMPUTE_32F_FAST_16F;
- return 0;
- }
- return -1;
-}
-
-static cublasComputeType_t g_gemm_compute = CUBLAS_COMPUTE_32F;
-static int g_gemm_compute_set = 0;
-
-int embed_cuda_set_fast_gemm(const char *mode) {
- cublasComputeType_t ct;
- if (parse_gemm_compute(mode, &ct) != 0)
- return -1;
- g_gemm_compute = ct;
- g_gemm_compute_set = 1;
- return 0;
-}
-
-static void cuda_matrix_free(cuda_matrix_t *m) {
- if (!m)
- return;
- cudaFree(m->d);
- memset(m, 0, sizeof(*m));
-}
-
-static float bf16_to_f32(uint16_t v) {
- uint32_t u = ((uint32_t)v) << 16;
- float f;
- memcpy(&f, &u, sizeof(f));
- return f;
-}
-
-// Round F32 to BF16 (truncate the low 16 bits, round to nearest even). NaNs are
-// preserved. BF16 shares F32's 8-bit exponent, so this never overflows.
-static uint16_t f32_to_bf16(float f) {
- uint32_t x;
- memcpy(&x, &f, sizeof(x));
- if ((x & 0x7fffffffu) > 0x7f800000u) // NaN
- return (uint16_t)((x >> 16) | 0x0040u);
- x += 0x7fffu + ((x >> 16) & 1u); // round to nearest even
- return (uint16_t)(x >> 16);
-}
-
-// Store weights as native BF16 on the device when set (halves weight memory and
-// feeds BF16 tensor cores); default keeps exact F32 for F32 model files. When
-// the flag was never set explicitly, the loader follows the model file dtype,
-// matching the CPU and MLX backends.
-static int g_weights_bf16 = 0;
-static int g_weights_bf16_set = 0;
-
-int embed_cuda_set_weights_bf16(int on) {
- g_weights_bf16 = on ? 1 : 0;
- g_weights_bf16_set = 1;
- return 0;
-}
-
-static int copy_weight_host_f32(float *dst, const embed_weight_ref_t *w, size_t count) {
- if (!dst || !w || !w->data)
- return -1;
- if (w->dtype == DTYPE_F32) {
- memcpy(dst, w->data, count * sizeof(float));
- return 0;
- }
- if (w->dtype == DTYPE_BF16) {
- const uint16_t *src = (const uint16_t *)w->data;
- for (size_t i = 0; i < count; i++)
- dst[i] = bf16_to_f32(src[i]);
- return 0;
- }
- return -1;
-}
-
-// Upload a host F32 weight buffer to the device, as BF16 when g_weights_bf16 is
-// set (rounding each value) or as F32 otherwise. Sets m->d, rows, cols, bf16.
-static int upload_weight(cuda_matrix_t *m, const float *src, size_t count, int rows, int cols) {
- cudaError_t e;
- if (g_weights_bf16) {
- uint16_t *tmp16 = (uint16_t *)malloc(count * sizeof(uint16_t));
- if (!tmp16)
- return -1;
- for (size_t i = 0; i < count; i++)
- tmp16[i] = f32_to_bf16(src[i]);
- e = cudaMalloc((void **)&m->d, count * sizeof(__nv_bfloat16));
- if (e != cudaSuccess) {
- fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e));
- free(tmp16);
- return -1;
- }
- e = cudaMemcpy(m->d, tmp16, count * sizeof(__nv_bfloat16), cudaMemcpyHostToDevice);
- free(tmp16);
- m->bf16 = 1;
- } else {
- e = cudaMalloc((void **)&m->d, count * sizeof(float));
- if (e != cudaSuccess) {
- fprintf(stderr, "cudaMalloc weight failed: %s\n", cudaGetErrorString(e));
- return -1;
- }
- e = cudaMemcpy(m->d, src, count * sizeof(float), cudaMemcpyHostToDevice);
- m->bf16 = 0;
- }
- if (e != cudaSuccess) {
- fprintf(stderr, "cudaMemcpy weight failed: %s\n", cudaGetErrorString(e));
- cuda_matrix_free(m);
- return -1;
- }
- m->rows = rows;
- m->cols = cols;
- return 0;
-}
-
-static int load_matrix(cuda_matrix_t *m, const embed_weight_ref_t *w, int rows, int cols) {
- if (!m || !w || rows <= 0 || cols <= 0)
- return -1;
- size_t count = (size_t)rows * (size_t)cols;
- float *tmp = (float *)malloc(count * sizeof(float));
- if (!tmp)
- return -1;
- if (copy_weight_host_f32(tmp, w, count) != 0) {
- free(tmp);
- return -1;
- }
- int r = upload_weight(m, tmp, count, rows, cols);
- free(tmp);
- return r;
-}
-
-static int load_qkv_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) {
- if (!m || !src || !c)
- return -1;
- int rows = c->q_dim + 2 * c->kv_dim;
- int cols = c->hidden_size;
- size_t q_count = (size_t)c->q_dim * cols;
- size_t kv_count = (size_t)c->kv_dim * cols;
- size_t total = q_count + 2 * kv_count;
- float *tmp = (float *)malloc(total * sizeof(float));
- if (!tmp)
- return -1;
- if (copy_weight_host_f32(tmp, &src->wq, q_count) != 0 ||
- copy_weight_host_f32(tmp + q_count, &src->wk, kv_count) != 0 ||
- copy_weight_host_f32(tmp + q_count + kv_count, &src->wv, kv_count) != 0) {
- free(tmp);
- return -1;
- }
- int r = upload_weight(m, tmp, total, rows, cols);
- free(tmp);
- return r;
-}
-
-static int
-load_gate_up_matrix(cuda_matrix_t *m, const embed_layer_t *src, const embed_config_t *c) {
- if (!m || !src || !c)
- return -1;
- int rows = 2 * c->intermediate_size;
- int cols = c->hidden_size;
- size_t proj_count = (size_t)c->intermediate_size * cols;
- size_t total = 2 * proj_count;
- float *tmp = (float *)malloc(total * sizeof(float));
- if (!tmp)
- return -1;
- if (copy_weight_host_f32(tmp, &src->gate_proj, proj_count) != 0 ||
- copy_weight_host_f32(tmp + proj_count, &src->up_proj, proj_count) != 0) {
- free(tmp);
- return -1;
- }
- int r = upload_weight(m, tmp, total, rows, cols);
- free(tmp);
- return r;
-}
-
-static int load_vector(float **out, const float *src, int n) {
- if (!out || !src || n <= 0)
- return -1;
- cudaError_t e = cudaMalloc((void **)out, (size_t)n * sizeof(float));
- if (e != cudaSuccess) {
- fprintf(stderr, "cudaMalloc vector failed: %s\n", cudaGetErrorString(e));
- return -1;
- }
- e = cudaMemcpy(*out, src, (size_t)n * sizeof(float), cudaMemcpyHostToDevice);
- if (e != cudaSuccess) {
- fprintf(stderr, "cudaMemcpy vector failed: %s\n", cudaGetErrorString(e));
- cudaFree(*out);
- *out = NULL;
- return -1;
- }
- return 0;
-}
-
-static int load_qkv_bias(float **out, const embed_layer_t *src, const embed_config_t *c) {
- int n = c->q_dim + 2 * c->kv_dim;
- float *host = (float *)malloc((size_t)n * sizeof(float));
- if (!host)
- return -1;
- memcpy(host, src->q_bias, (size_t)c->q_dim * sizeof(float));
- memcpy(host + c->q_dim, src->k_bias, (size_t)c->kv_dim * sizeof(float));
- memcpy(host + c->q_dim + c->kv_dim, src->v_bias, (size_t)c->kv_dim * sizeof(float));
- int rc = load_vector(out, host, n);
- free(host);
- return rc;
-}
-
-__global__ static void add_row_bias_kernel(float *x, const float *bias, int rows, int dim) {
- int idx = blockIdx.x * blockDim.x + threadIdx.x;
- int count = rows * dim;
- if (idx < count)
- x[idx] += bias[idx % dim];
-}
-
-__global__ static void embed_lookup_kernel(
- float *x, const int *ids, const void *emb, int emb_bf16, int total, int hidden, int vocab) {
- int idx = blockIdx.x * blockDim.x + threadIdx.x;
- int count = total * hidden;
- if (idx >= count)
- return;
- int tok = idx / hidden;
- int d = idx - tok * hidden;
- int id = ids[tok];
- if (id < 0 || id >= vocab) {
- x[idx] = 0.0f;
- return;
- }
- size_t off = (size_t)id * hidden + d;
- x[idx] =
- emb_bf16 ? __bfloat162float(((const __nv_bfloat16 *)emb)[off]) : ((const float *)emb)[off];
-}
-
-// Cast n F32 values to BF16 (used to feed BF16-weight projection GEMMs).
-__global__ static void cast_f32_to_bf16_kernel(__nv_bfloat16 *out, const float *in, size_t n) {
- size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
- if (i < n)
- out[i] = __float2bfloat16(in[i]);
-}
-
-// Widen n BF16 values to F32 (used to run an exact F32 GEMM on BF16-stored
-// weights - memory-only BF16, no precision loss from the operands).
-__global__ static void cast_bf16_to_f32_kernel(float *out, const __nv_bfloat16 *in, size_t n) {
- size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
- if (i < n)
- out[i] = __bfloat162float(in[i]);
-}
-
-__device__ static float block_sum(float v) {
- __shared__ float partial[8];
- __shared__ float total;
- int lane = threadIdx.x & 31;
- int warp = threadIdx.x >> 5;
- int warps = blockDim.x >> 5;
-
- for (int offset = 16; offset > 0; offset >>= 1)
- v += __shfl_down_sync(0xffffffff, v, offset);
- if (lane == 0)
- partial[warp] = v;
- __syncthreads();
-
- if (warp == 0) {
- v = lane < warps ? partial[lane] : 0.0f;
- for (int offset = 16; offset > 0; offset >>= 1)
- v += __shfl_down_sync(0xffffffff, v, offset);
- if (lane == 0)
- total = v;
- }
- __syncthreads();
- return total;
-}
-
-__device__ static inline void store_act(float *out, size_t i, float v) { out[i] = v; }
-
-__device__ static inline void store_act(__nv_bfloat16 *out, size_t i, float v) {
- out[i] = __float2bfloat16(v);
-}
-
-// OUT_T is float, or __nv_bfloat16 when the result feeds a BF16-weight
-// projection GEMM directly (same rounding as a separate cast kernel).
-template <typename OUT_T>
-__global__ static void
-rms_norm_kernel(OUT_T *out, const float *x, const float *weight, int rows, int dim, float eps) {
- int row = blockIdx.x;
- int tid = threadIdx.x;
- float sum = 0.0f;
- for (int d = tid; d < dim; d += blockDim.x) {
- float v = x[(size_t)row * dim + d];
- sum += v * v;
- }
- float total = block_sum(sum);
- float inv = rsqrtf(total / (float)dim + eps);
- for (int d = tid; d < dim; d += blockDim.x)
- store_act(out, (size_t)row * dim + d, x[(size_t)row * dim + d] * inv * weight[d]);
-}
-
-// One grid covers Q and K: blockIdx.y < n_heads selects the Q head with
-// q_norm/q_offset, the remaining blocks the K head with k_norm/k_offset.
-__global__ static void rms_norm_rope_qk_kernel(float *x,
- const float *q_norm,
- const float *k_norm,
- const int *positions,
- const float *cosv,
- const float *sinv,
- int rows,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- int row_stride,
- int q_offset,
- int k_offset,
- int qk_norm,
- float eps) {
- __shared__ float partial[4];
- __shared__ float total;
- int row = blockIdx.x;
- int head = blockIdx.y;
- int tid = threadIdx.x;
- int lane = tid & 31;
- int warp = tid >> 5;
- int half = head_dim / 2;
- int pair_d = tid < half ? tid + half : tid - half;
- float sign = tid < half ? -1.0f : 1.0f;
- const float *weight = head < n_heads ? q_norm : k_norm;
- int base_offset = head < n_heads ? q_offset : k_offset;
- if (head >= n_heads)
- head -= n_heads;
- size_t base = (size_t)row * row_stride + base_offset + head * head_dim;
- float v = tid < head_dim ? x[base + tid] : 0.0f;
- float pair_v = tid < head_dim ? x[base + pair_d] : 0.0f;
- if (qk_norm) {
- float sum = v * v;
- for (int offset = 16; offset > 0; offset >>= 1)
- sum += __shfl_down_sync(0xffffffff, sum, offset);
- if (lane == 0)
- partial[warp] = sum;
- __syncthreads();
- if (warp == 0) {
- sum = lane < 4 ? partial[lane] : 0.0f;
- for (int offset = 16; offset > 0; offset >>= 1)
- sum += __shfl_down_sync(0xffffffff, sum, offset);
- if (lane == 0)
- total = sum;
- }
- __syncthreads();
- }
-
- if (tid < head_dim) {
- int pos = positions[row];
- float c = cosv[(size_t)pos * head_dim + tid];
- float s = sinv[(size_t)pos * head_dim + tid];
- float a = v;
- float b = pair_v;
- if (qk_norm) {
- float inv = rsqrtf(total / (float)head_dim + eps);
- a *= inv * weight[tid];
- b *= inv * weight[pair_d];
- }
- x[base + tid] = a * c + sign * b * s;
- }
-}
-
-// Writes SiLU(gate)*up to out with out_stride. The F32 path writes in place
-// over the gate half (out = gate_up, out_stride = 2*intermediate); the BF16
-// path writes a packed [rows x intermediate] BF16 matrix for the down GEMM.
-template <typename OUT_T>
-__global__ static void silu_mul_packed_kernel(
- OUT_T *out, int out_stride, const float *gate_up, int rows, int intermediate) {
- int i = blockIdx.x * blockDim.x + threadIdx.x;
- int n = rows * intermediate;
- if (i < n) {
- int row = i / intermediate;
- int d = i - row * intermediate;
- size_t base = (size_t)row * (2 * intermediate);
- float g = gate_up[base + d];
- float u = gate_up[base + intermediate + d];
- store_act(out, (size_t)row * out_stride + d, (g / (1.0f + expf(-g))) * u);
- }
-}
-
-// Expand K and V from the n_kv_heads layout to a contiguous [total x q_dim]
-// per-query-head layout (GQA: each kv head repeated n_heads/n_kv_heads times)
-// so the attention GEMMs can stride uniformly over query heads.
-// V_T is float, or __nv_bfloat16 when V feeds a BF16 @V GEMM directly.
-template <typename V_T>
-__global__ static void attn_expand_kv_kernel(float *kexp,
- V_T *vexp,
- const float *qkv,
- int total,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- int qkv_dim,
- int k_offset,
- int v_offset) {
- int q_dim = n_heads * head_dim;
- size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
- size_t count = (size_t)total * q_dim;
- if (idx >= count)
- return;
- int t = (int)(idx / q_dim);
- int rem = (int)(idx - (size_t)t * q_dim);
- int h = rem / head_dim;
- int e = rem - h * head_dim;
- int kv = h / (n_heads / n_kv_heads);
- const float *base = qkv + (size_t)t * qkv_dim;
- kexp[idx] = base[k_offset + kv * head_dim + e];
- store_act(vexp, idx, base[v_offset + kv * head_dim + e]);
-}
-
-// Row softmax over the key dimension for one sequence's score tensor laid out
-// as scores[head * L * L + i * L + j]; normalizes across j for each (head, i).
-// One block per (head, query) row; scores are pre-scaled by the QK^T GEMM.
-// P_T is float with probs == scores (in place), or __nv_bfloat16 when the
-// probabilities feed a BF16 @V GEMM.
-template <typename P_T>
-__global__ static void attn_softmax_kernel(float *scores, P_T *probs, int L) {
- float *s = scores + (size_t)blockIdx.x * L;
- P_T *p = probs + (size_t)blockIdx.x * L;
- int tid = threadIdx.x;
- __shared__ float red[256];
-
- float m = -3.402823466e+38F;
- for (int j = tid; j < L; j += blockDim.x)
- m = fmaxf(m, s[j]);
- red[tid] = m;
- __syncthreads();
- for (int o = blockDim.x / 2; o > 0; o >>= 1) {
- if (tid < o)
- red[tid] = fmaxf(red[tid], red[tid + o]);
- __syncthreads();
- }
- m = red[0];
- __syncthreads();
-
- float sum = 0.0f;
- for (int j = tid; j < L; j += blockDim.x) {
- float e = __expf(s[j] - m);
- s[j] = e;
- sum += e;
- }
- red[tid] = sum;
- __syncthreads();
- for (int o = blockDim.x / 2; o > 0; o >>= 1) {
- if (tid < o)
- red[tid] += red[tid + o];
- __syncthreads();
- }
- float inv = 1.0f / red[0];
- for (int j = tid; j < L; j += blockDim.x)
- store_act(p, (size_t)j, s[j] * inv);
-}
-
-template <typename P_T>
-__global__ static void attn_causal_softmax_kernel(float *scores, P_T *probs, int L) {
- float *s = scores + (size_t)blockIdx.x * L;
- P_T *p = probs + (size_t)blockIdx.x * L;
- int query = blockIdx.x % L;
- int keys = query + 1;
- int tid = threadIdx.x;
- __shared__ float red[256];
-
- float m = -3.402823466e+38F;
- for (int j = tid; j < keys; j += blockDim.x)
- m = fmaxf(m, s[j]);
- red[tid] = m;
- __syncthreads();
- for (int o = blockDim.x / 2; o > 0; o >>= 1) {
- if (tid < o)
- red[tid] = fmaxf(red[tid], red[tid + o]);
- __syncthreads();
- }
- m = red[0];
- __syncthreads();
-
- float sum = 0.0f;
- for (int j = tid; j < keys; j += blockDim.x) {
- float e = __expf(s[j] - m);
- s[j] = e;
- sum += e;
- }
- red[tid] = sum;
- __syncthreads();
- for (int o = blockDim.x / 2; o > 0; o >>= 1) {
- if (tid < o)
- red[tid] += red[tid + o];
- __syncthreads();
- }
- float inv = 1.0f / red[0];
- for (int j = tid; j < L; j += blockDim.x)
- store_act(p, (size_t)j, j < keys ? s[j] * inv : 0.0f);
-}
-
-__global__ static void
-mean_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) {
- int b = blockIdx.x;
- int tid = threadIdx.x;
- int start = offsets[b];
- int end = offsets[b + 1];
- int len = end - start;
- for (int d = tid; d < hidden; d += blockDim.x) {
- float sum = 0.0f;
- for (int t = start; t < end; t++)
- sum += x[(size_t)t * hidden + d];
- out[(size_t)b * hidden + d] = sum / (float)len;
- }
-}
-
-__global__ static void
-last_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) {
- int b = blockIdx.x;
- int tid = threadIdx.x;
- int last = offsets[b + 1] - 1;
- for (int d = tid; d < hidden; d += blockDim.x)
- out[(size_t)b * hidden + d] = x[(size_t)last * hidden + d];
-}
-
-/* CLS pooling: gather the first token of each packed sequence. */
-__global__ static void
-first_pool_kernel(float *out, const float *x, const int *offsets, int batch, int hidden) {
- int b = blockIdx.x;
- int tid = threadIdx.x;
- int first = offsets[b];
- for (int d = tid; d < hidden; d += blockDim.x)
- out[(size_t)b * hidden + d] = x[(size_t)first * hidden + d];
-}
-
-__global__ static void span_pool_kernel(
- float *out, const float *x, const int *starts, const int *lens, int n_spans, int hidden) {
- int s = blockIdx.x;
- int tid = threadIdx.x;
- int start = starts[s];
- int len = lens[s];
- for (int d = tid; d < hidden; d += blockDim.x) {
- float sum = 0.0f;
- for (int t = 0; t < len; t++)
- sum += x[(size_t)(start + t) * hidden + d];
- out[(size_t)s * hidden + d] = sum / (float)len;
- }
-}
-
-// BERT LayerNorm: mean-subtract, biased variance (/dim), scale by gamma, shift
-// by beta. One block per row, two block reductions before any write, so it is
-// safe in place (out == x). Matches the CPU embed_layer_norm.
-__global__ static void layer_norm_kernel(float *out,
- const float *x,
- const float *gamma,
- const float *beta,
- int rows,
- int dim,
- float eps) {
- int row = blockIdx.x;
- int tid = threadIdx.x;
- const float *xr = x + (size_t)row * dim;
- float s = 0.0f;
- for (int d = tid; d < dim; d += blockDim.x)
- s += xr[d];
- float mean = block_sum(s) / (float)dim;
- float vs = 0.0f;
- for (int d = tid; d < dim; d += blockDim.x) {
- float c = xr[d] - mean;
- vs += c * c;
- }
- float inv = rsqrtf(block_sum(vs) / (float)dim + eps);
- float *o = out + (size_t)row * dim;
- for (int d = tid; d < dim; d += blockDim.x)
- o[d] = (xr[d] - mean) * inv * gamma[d] + beta[d];
-}
-
-// Exact-erf GeLU in place: 0.5 x (1 + erf(x / sqrt(2))). The released encoders
-// use this; the tanh approximation below is for gelu_new / gelu_pytorch_tanh.
-__global__ static void gelu_kernel(float *x, size_t n) {
- size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
- if (i < n) {
- float v = x[i];
- x[i] = 0.5f * v * (1.0f + erff(v * 0.70710678118654752f));
- }
-}
-
-// Tanh-approximation GeLU in place: 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))).
-__global__ static void gelu_tanh_kernel(float *x, size_t n) {
- size_t i = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
- if (i < n) {
- float v = x[i];
- float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v);
- x[i] = 0.5f * v * (1.0f + tanhf(inner));
- }
-}
-
-// Add the learned absolute position embedding (indexed by the within-sequence
-// position) and the token-type[0] embedding to the token embeddings already in
-// x. token_type holds row 0 only; every input token has segment id 0.
-__global__ static void bert_pos_type_add_kernel(float *x,
- const int *positions,
- const float *pos_emb,
- const float *token_type,
- int total,
- int hidden) {
- size_t idx = (size_t)blockIdx.x * blockDim.x + threadIdx.x;
- size_t count = (size_t)total * hidden;
- if (idx >= count)
- return;
- int tok = (int)(idx / hidden);
- int d = (int)(idx - (size_t)tok * hidden);
- int pos = positions[tok];
- x[idx] += pos_emb[(size_t)pos * hidden + d] + token_type[d];
-}
-
-static int launch_check(void) {
- cudaError_t e = cudaGetLastError();
- if (e != cudaSuccess) {
- fprintf(stderr, "cuda kernel failed: %s\n", cudaGetErrorString(e));
- return -1;
- }
- return 0;
-}
-
-// Compute type for CUDA GEMMs. Default is exact F32 (== cublasSgemm).
-// EMBED_CUDA_GEMM_MODE={bf16,tf32,16f} selects a reduced-precision tensor-core
-// matmul that keeps F32 inputs/outputs and F32 accumulation but rounds operands
-// internally.
-static cublasComputeType_t gemm_compute(void) {
- static int env_init = 0;
- if (!g_gemm_compute_set && !env_init) {
- const char *e = getenv("EMBED_CUDA_GEMM_MODE");
- if (e && parse_gemm_compute(e, &g_gemm_compute) == 0)
- g_gemm_compute_set = 1;
- env_init = 1;
- }
- return g_gemm_compute;
-}
-
-// Grow the F32 weight-widen scratch to hold `elems` floats. The memory-only
-// BF16 path materializes one F32 weight at a time into this buffer.
-static int ensure_weight_f32(embed_cuda_ctx_t *ctx, size_t elems) {
- if (ctx->weight_f32_elems >= elems)
- return 0;
- cudaFree(ctx->weight_f32);
- ctx->weight_f32 = NULL;
- ctx->weight_f32_elems = 0;
- CUDA_CHECK(cudaMalloc((void **)&ctx->weight_f32, elems * sizeof(float)));
- ctx->weight_f32_elems = elems;
- return 0;
-}
-
-// y = w @ x (+ beta*y). With BF16 weights and an already-BF16 activation
-// (x_is_bf16 - the Qwen fused path) both inputs are BF16 with F32 accumulation.
-// For the BERT family the activation stays F32 and the BF16 weight is widened
-// back to F32 here, so the GEMM is exact (honoring gemm_compute): BF16 is then a
-// storage choice that costs no quality, which the post-norm LayerNorm needs.
-// With F32 weights this is the exact path.
-static int linear_ex(embed_cuda_ctx_t *ctx,
- const cuda_matrix_t *w,
- const void *x,
- int x_is_bf16,
- float *y,
- int rows,
- int in_dim,
- int out_dim,
- int x_stride,
- float beta) {
- const float alpha = 1.0f;
- if (w->bf16) {
- if (!x_is_bf16 && ctx->config.family == EMBED_FAMILY_BERT) {
- size_t welems = (size_t)out_dim * (size_t)in_dim;
- if (ensure_weight_f32(ctx, welems) != 0)
- return -1;
- int wt = 256;
- cast_bf16_to_f32_kernel<<<(welems + wt - 1) / wt, wt, 0, ctx->stream>>>(
- ctx->weight_f32, (const __nv_bfloat16 *)w->d, welems);
- if (launch_check() != 0)
- return -1;
- CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim,
- &alpha, ctx->weight_f32, CUDA_R_32F, in_dim, x, CUDA_R_32F,
- x_stride, &beta, y, CUDA_R_32F, out_dim, gemm_compute(),
- CUBLAS_GEMM_DEFAULT));
- return 0;
- }
- const void *xb = x;
- if (!x_is_bf16) {
- size_t n = (size_t)rows * (size_t)x_stride;
- int threads = 256;
- cast_f32_to_bf16_kernel<<<(n + threads - 1) / threads, threads, 0, ctx->stream>>>(
- (__nv_bfloat16 *)ctx->act_bf16, (const float *)x, n);
- if (launch_check() != 0)
- return -1;
- xb = ctx->act_bf16;
- }
- CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim,
- &alpha, w->d, CUDA_R_16BF, in_dim, xb, CUDA_R_16BF, x_stride,
- &beta, y, CUDA_R_32F, out_dim, CUBLAS_COMPUTE_32F,
- CUBLAS_GEMM_DEFAULT));
- return 0;
- }
- CUBLAS_CHECK(cublasGemmEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, out_dim, rows, in_dim, &alpha,
- w->d, CUDA_R_32F, in_dim, x, CUDA_R_32F, x_stride, &beta, y,
- CUDA_R_32F, out_dim, gemm_compute(), CUBLAS_GEMM_DEFAULT));
- return 0;
-}
-
-static int linear(embed_cuda_ctx_t *ctx,
- const cuda_matrix_t *w,
- const float *x,
- float *y,
- int rows,
- int in_dim,
- int out_dim) {
- return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 0.0f);
-}
-
-// x points at activations already stored as BF16 (e.g. ctx->act_bf16).
-static int linear_bf16x(embed_cuda_ctx_t *ctx,
- const cuda_matrix_t *w,
- const void *x,
- float *y,
- int rows,
- int in_dim,
- int out_dim,
- int x_stride,
- float beta) {
- return linear_ex(ctx, w, x, 1, y, rows, in_dim, out_dim, x_stride, beta);
-}
-
-static int linear_accum(embed_cuda_ctx_t *ctx,
- const cuda_matrix_t *w,
- const float *x,
- float *y,
- int rows,
- int in_dim,
- int out_dim) {
- return linear_ex(ctx, w, x, 0, y, rows, in_dim, out_dim, in_dim, 1.0f);
-}
-
-static int gemm_strided_batched(cublasHandle_t blas,
- cublasOperation_t transa,
- cublasOperation_t transb,
- int m,
- int n,
- int k,
- const float *alpha,
- const float *A,
- int lda,
- long long strideA,
- const float *B,
- int ldb,
- long long strideB,
- const float *beta,
- float *C,
- int ldc,
- long long strideC,
- int batch_count) {
- cublasComputeType_t ct = gemm_compute();
- if (ct == CUBLAS_COMPUTE_32F) {
- CUBLAS_CHECK(cublasSgemmStridedBatched(blas, transa, transb, m, n, k, alpha, A, lda,
- strideA, B, ldb, strideB, beta, C, ldc, strideC,
- batch_count));
- } else {
- CUBLAS_CHECK(cublasGemmStridedBatchedEx(
- blas, transa, transb, m, n, k, alpha, A, CUDA_R_32F, lda, strideA, B, CUDA_R_32F, ldb,
- strideB, beta, C, CUDA_R_32F, ldc, strideC, batch_count, ct, CUBLAS_GEMM_DEFAULT));
- }
- return 0;
-}
-
-// Attention for one micro-batch. Expands K/V to the per-query-head layout,
-// then for each sequence runs batched-over-heads Q@K^T (scaled) -> row softmax
-// -> @V via cuBLAS. Requires ctx->kexp, ctx->vexp ([total x q_dim]) and
-// ctx->attn_scores ([n_heads x max_seq x max_seq]) to be allocated. h_offsets
-// is the host copy of the packed sequence boundaries.
-// Prepare the equal-length batched-attention path for one forward pass:
-// grow the score/probability scratch to hold attn_G sequences at once and
-// upload the pointer arrays for the two batched GEMMs (buffer addresses are
-// layer-invariant). attn_G stays 0 - per-sequence fallback - for ragged
-// batches, single sequences, or when one sequence's scores exceed the
-// budget.
-static int attn_batched_setup(embed_cuda_ctx_t *ctx, int batch, int q_offset, int qkv_dim) {
- const embed_config_t *c = &ctx->config;
- ctx->attn_G = 0;
- if (batch < 2)
- return 0;
- int L = ctx->offsets_host[1] - ctx->offsets_host[0];
- if (L > CUDA_ATTN_BATCHED_MAX_LEN)
- return 0;
- for (int b = 1; b < batch; b++)
- if (ctx->offsets_host[b + 1] - ctx->offsets_host[b] != L)
- return 0;
- int H = c->n_heads, hd = c->head_dim, q_dim = c->q_dim;
- long long per = (long long)H * L * L;
- if (per <= 0 || per > CUDA_ATTN_SCORES_BUDGET / 2)
- return 0;
- int G = (int)(CUDA_ATTN_SCORES_BUDGET / per);
- if (G > batch)
- G = batch;
- if (G < 2)
- return 0;
-
- int bf16 = g_weights_bf16;
- long long want = (long long)G * per;
- if (want > ctx->attn_scores_elems) {
- cudaFree(ctx->attn_scores);
- ctx->attn_scores = NULL;
- ctx->attn_scores_elems = 0;
- CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)want * sizeof(float)));
- ctx->attn_scores_elems = want;
- }
- if (bf16 && want > ctx->attn_probs_elems) {
- cudaFree(ctx->attn_probs);
- ctx->attn_probs = NULL;
- ctx->attn_probs_elems = 0;
- CUDA_CHECK(cudaMalloc(&ctx->attn_probs, (size_t)want * sizeof(__nv_bfloat16)));
- ctx->attn_probs_elems = want;
- }
-
- int entries = 6 * batch * H;
- if (entries > ctx->attn_ptrs_cap) {
- cudaFree((void *)ctx->attn_ptrs);
- free((void *)ctx->attn_ptrs_host);
- ctx->attn_ptrs = NULL;
- ctx->attn_ptrs_cap = 0;
- ctx->attn_ptrs_host = (const void **)malloc((size_t)entries * sizeof(void *));
- if (!ctx->attn_ptrs_host)
- return -1;
- CUDA_CHECK(cudaMalloc((void **)&ctx->attn_ptrs, (size_t)entries * sizeof(void *)));
- ctx->attn_ptrs_cap = entries;
- }
-
- const void **K = ctx->attn_ptrs_host;
- const void **Q = K + (size_t)batch * H;
- const void **S = Q + (size_t)batch * H;
- const void **V = S + (size_t)batch * H;
- const void **P = V + (size_t)batch * H;
- const void **O = P + (size_t)batch * H;
- const __nv_bfloat16 *vb = (const __nv_bfloat16 *)ctx->vexp;
- const __nv_bfloat16 *pb = (const __nv_bfloat16 *)ctx->attn_probs;
- const __nv_bfloat16 *ab = (const __nv_bfloat16 *)ctx->act_bf16;
- for (int b = 0; b < batch; b++) {
- // Score/probability rows cycle every G sequences (chunked reuse).
- long long srow = (long long)(b % G) * per;
- size_t tok = (size_t)b * L;
- for (int h = 0; h < H; h++) {
- size_t i = (size_t)b * H + h;
- K[i] = ctx->kexp + tok * q_dim + (size_t)h * hd;
- Q[i] = ctx->qkv + tok * qkv_dim + q_offset + (size_t)h * hd;
- S[i] = ctx->attn_scores + srow + (long long)h * L * L;
- if (bf16) {
- V[i] = vb + tok * q_dim + (size_t)h * hd;
- P[i] = pb + srow + (long long)h * L * L;
- O[i] = ab + tok * q_dim + (size_t)h * hd;
- } else {
- V[i] = ctx->vexp + tok * q_dim + (size_t)h * hd;
- P[i] = S[i];
- O[i] = ctx->attn_out + tok * q_dim + (size_t)h * hd;
- }
- }
- }
- CUDA_CHECK(cudaMemcpy((void *)ctx->attn_ptrs, ctx->attn_ptrs_host,
- (size_t)entries * sizeof(void *), cudaMemcpyHostToDevice));
- ctx->attn_G = G;
- ctx->attn_L = L;
- return 0;
-}
-
-// out_bf16 != NULL selects the BF16-weight layout: V and the softmax
-// probabilities are stored BF16 and the @V GEMM emits BF16 attention output
-// at out_bf16 (stride q_dim), ready for the wo projection without a cast.
-static int cuda_attention_gemm(embed_cuda_ctx_t *ctx,
- const int *h_offsets,
- int batch,
- int q_offset,
- int k_offset,
- int v_offset,
- int qkv_dim,
- float scale,
- __nv_bfloat16 *out_bf16) {
- const embed_config_t *c = &ctx->config;
- int q_dim = c->q_dim;
- int hd = c->head_dim;
- int H = c->n_heads;
- int total = h_offsets[batch];
- int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
- const float alpha = scale, beta = 0.0f, one = 1.0f;
-
- int threads = 256;
- size_t exp_count = (size_t)total * q_dim;
- if (out_bf16) {
- attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
- ctx->kexp, (__nv_bfloat16 *)ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim,
- k_offset, v_offset);
- } else {
- attn_expand_kv_kernel<<<(exp_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
- ctx->kexp, ctx->vexp, ctx->qkv, total, H, c->n_kv_heads, hd, qkv_dim, k_offset,
- v_offset);
- }
- if (launch_check() != 0)
- return -1;
-
- if (ctx->attn_G >= 2) {
- int G = ctx->attn_G, L = ctx->attn_L;
- const void *const *K = (const void *const *)ctx->attn_ptrs;
- const void *const *Q = K + (size_t)batch * H;
- void *const *S = (void *const *)(Q + (size_t)batch * H);
- const void *const *V = (const void *const *)S + (size_t)batch * H;
- const void *const *P = V + (size_t)batch * H;
- void *const *O = (void *const *)(P + (size_t)batch * H);
- cudaDataType pv = out_bf16 ? CUDA_R_16BF : CUDA_R_32F;
- cublasComputeType_t av_ct = out_bf16 ? CUBLAS_COMPUTE_32F : gemm_compute();
- for (int s = 0; s < batch; s += G) {
- int g = batch - s < G ? batch - s : G;
- int n = g * H;
- size_t off = (size_t)s * H;
- /* scores = scale * Q @ K^T, batched over (seq, head) */
- CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha,
- K + off, CUDA_R_32F, q_dim, Q + off, CUDA_R_32F,
- qkv_dim, &beta, S + off, CUDA_R_32F, L, n,
- gemm_compute(), CUBLAS_GEMM_DEFAULT));
- if (out_bf16) {
- if (causal)
- attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L);
- else
- attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(
- ctx->attn_scores, (__nv_bfloat16 *)ctx->attn_probs, L);
- } else {
- if (causal)
- attn_causal_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
- ctx->attn_scores, L);
- else
- attn_softmax_kernel<<<n * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
- ctx->attn_scores, L);
- }
- if (launch_check() != 0)
- return -1;
- /* O = P @ V, batched over (seq, head) */
- CUBLAS_CHECK(cublasGemmBatchedEx(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one,
- V + off, pv, q_dim, P + off, pv, L, &beta, O + off, pv,
- q_dim, n, av_ct, CUBLAS_GEMM_DEFAULT));
- }
- return 0;
- }
-
- for (int b = 0; b < batch; b++) {
- int start = h_offsets[b];
- int L = h_offsets[b + 1] - start;
- const float *Q = ctx->qkv + (size_t)start * qkv_dim + q_offset;
- const float *K = ctx->kexp + (size_t)start * q_dim;
- /* scores[h] = scale * Q[h] @ K[h]^T (row-major C = A @ B^T) */
- if (gemm_strided_batched(ctx->blas, CUBLAS_OP_T, CUBLAS_OP_N, L, L, hd, &alpha, K, q_dim,
- hd, Q, qkv_dim, hd, &beta, ctx->attn_scores, L, (long long)L * L,
- H) != 0)
- return -1;
- if (out_bf16) {
- __nv_bfloat16 *P = (__nv_bfloat16 *)ctx->attn_probs;
- const __nv_bfloat16 *V = (const __nv_bfloat16 *)ctx->vexp + (size_t)start * q_dim;
- __nv_bfloat16 *O = out_bf16 + (size_t)start * q_dim;
- if (causal)
- attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores, P, L);
- else
- attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores, P, L);
- if (launch_check() != 0)
- return -1;
- /* O[h] = P[h] @ V[h] (row-major C = A @ B) */
- CUBLAS_CHECK(cublasGemmStridedBatchedEx(
- ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, CUDA_R_16BF, q_dim, hd, P,
- CUDA_R_16BF, L, (long long)L * L, &beta, O, CUDA_R_16BF, q_dim, hd, H,
- CUBLAS_COMPUTE_32F, CUBLAS_GEMM_DEFAULT));
- } else {
- const float *V = ctx->vexp + (size_t)start * q_dim;
- float *O = ctx->attn_out + (size_t)start * q_dim;
- if (causal)
- attn_causal_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
- ctx->attn_scores, L);
- else
- attn_softmax_kernel<<<H * L, 256, 0, ctx->stream>>>(ctx->attn_scores,
- ctx->attn_scores, L);
- if (launch_check() != 0)
- return -1;
- /* O[h] = P[h] @ V[h] (row-major C = A @ B) */
- if (gemm_strided_batched(ctx->blas, CUBLAS_OP_N, CUBLAS_OP_N, hd, L, L, &one, V, q_dim,
- hd, ctx->attn_scores, L, (long long)L * L, &beta, O, q_dim, hd,
- H) != 0)
- return -1;
- }
- }
- return 0;
-}
-
-static int ensure_buffers(embed_cuda_ctx_t *ctx, int total, int batch, int max_seq) {
- const embed_config_t *c = &ctx->config;
- if (total > ctx->seq_cap) {
- cudaFree(ctx->x);
- cudaFree(ctx->x_norm);
- cudaFree(ctx->qkv);
- cudaFree(ctx->attn_out);
- cudaFree(ctx->ffn_gate_up);
- cudaFree(ctx->token_ids);
- cudaFree(ctx->positions);
- cudaFree(ctx->kexp);
- cudaFree(ctx->vexp);
- cudaFree(ctx->act_bf16);
- ctx->x = ctx->x_norm = ctx->qkv = NULL;
- ctx->attn_out = ctx->ffn_gate_up = NULL;
- ctx->token_ids = ctx->positions = NULL;
- ctx->kexp = ctx->vexp = NULL;
- ctx->act_bf16 = NULL;
- CUDA_CHECK(cudaMalloc((void **)&ctx->x, (size_t)total * c->hidden_size * sizeof(float)));
- CUDA_CHECK(
- cudaMalloc((void **)&ctx->x_norm, (size_t)total * c->hidden_size * sizeof(float)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->qkv,
- (size_t)total * (c->q_dim + 2 * c->kv_dim) * sizeof(float)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->attn_out, (size_t)total * c->q_dim * sizeof(float)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->ffn_gate_up,
- (size_t)total * 2 * c->intermediate_size * sizeof(float)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->token_ids, (size_t)total * sizeof(int)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->positions, (size_t)total * sizeof(int)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->kexp, (size_t)total * c->q_dim * sizeof(float)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->vexp, (size_t)total * c->q_dim * sizeof(float)));
- if (g_weights_bf16)
- CUDA_CHECK(cudaMalloc(&ctx->act_bf16, (size_t)total * 2 * c->intermediate_size *
- sizeof(__nv_bfloat16)));
- ctx->seq_cap = total;
- }
- if (batch + 1 > ctx->batch_cap) {
- cudaFree(ctx->offsets);
- free(ctx->offsets_host);
- CUDA_CHECK(cudaMalloc((void **)&ctx->offsets, (size_t)(batch + 1) * sizeof(int)));
- ctx->offsets_host = (int *)malloc((size_t)(batch + 1) * sizeof(int));
- if (!ctx->offsets_host)
- return -1;
- ctx->batch_cap = batch + 1;
- }
- if (max_seq > ctx->max_seq_cap) {
- cudaFree(ctx->rope_cos);
- cudaFree(ctx->rope_sin);
- long long score_elems = (long long)c->n_heads * max_seq * max_seq;
- if (score_elems > ctx->attn_scores_elems) {
- cudaFree(ctx->attn_scores);
- ctx->attn_scores = NULL;
- ctx->attn_scores_elems = 0;
- CUDA_CHECK(cudaMalloc((void **)&ctx->attn_scores, (size_t)score_elems * sizeof(float)));
- ctx->attn_scores_elems = score_elems;
- }
- if (g_weights_bf16 && score_elems > ctx->attn_probs_elems) {
- cudaFree(ctx->attn_probs);
- ctx->attn_probs = NULL;
- ctx->attn_probs_elems = 0;
- CUDA_CHECK(cudaMalloc(&ctx->attn_probs, (size_t)score_elems * sizeof(__nv_bfloat16)));
- ctx->attn_probs_elems = score_elems;
- }
- float *hcos = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
- float *hsin = (float *)malloc((size_t)max_seq * c->head_dim * sizeof(float));
- if (!hcos || !hsin) {
- free(hcos);
- free(hsin);
- return -1;
- }
- int half = c->head_dim / 2;
- for (int pos = 0; pos < max_seq; pos++) {
- for (int d = 0; d < half; d++) {
- float inv = 1.0f / powf(c->rope_theta, (float)(2 * d) / (float)c->head_dim);
- float angle = (float)pos * inv;
- float cv = cosf(angle);
- float sv = sinf(angle);
- hcos[(size_t)pos * c->head_dim + d] = cv;
- hcos[(size_t)pos * c->head_dim + half + d] = cv;
- hsin[(size_t)pos * c->head_dim + d] = sv;
- hsin[(size_t)pos * c->head_dim + half + d] = sv;
- }
- }
- CUDA_CHECK(
- cudaMalloc((void **)&ctx->rope_cos, (size_t)max_seq * c->head_dim * sizeof(float)));
- CUDA_CHECK(
- cudaMalloc((void **)&ctx->rope_sin, (size_t)max_seq * c->head_dim * sizeof(float)));
- CUDA_CHECK(cudaMemcpyAsync(ctx->rope_cos, hcos,
- (size_t)max_seq * c->head_dim * sizeof(float),
- cudaMemcpyHostToDevice, ctx->stream));
- CUDA_CHECK(cudaMemcpyAsync(ctx->rope_sin, hsin,
- (size_t)max_seq * c->head_dim * sizeof(float),
- cudaMemcpyHostToDevice, ctx->stream));
- free(hcos);
- free(hsin);
- ctx->max_seq_cap = max_seq;
- }
- return 0;
-}
-
-static int ensure_pooled_rows(embed_cuda_ctx_t *ctx, int rows) {
- if (rows <= ctx->pooled_rows_cap)
- return 0;
- const embed_config_t *c = &ctx->config;
- cudaFree(ctx->pooled_out);
- ctx->pooled_out = NULL;
- CUDA_CHECK(
- cudaMalloc((void **)&ctx->pooled_out, (size_t)rows * c->hidden_size * sizeof(float)));
- ctx->pooled_rows_cap = rows;
- return 0;
-}
-
-static int ensure_span_buffers(embed_cuda_ctx_t *ctx, int n_spans) {
- if (n_spans <= ctx->span_cap)
- return 0;
- cudaFree(ctx->span_starts);
- cudaFree(ctx->span_lens);
- ctx->span_starts = NULL;
- ctx->span_lens = NULL;
- CUDA_CHECK(cudaMalloc((void **)&ctx->span_starts, (size_t)n_spans * sizeof(int)));
- CUDA_CHECK(cudaMalloc((void **)&ctx->span_lens, (size_t)n_spans * sizeof(int)));
- ctx->span_cap = n_spans;
- return 0;
-}
-
-static int load_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) {
- return load_qkv_matrix(&dst->qkv, src, c) ||
- load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) ||
- (c->qkv_bias && load_qkv_bias(&dst->qkv_bias, src, c)) ||
- (c->qk_norm && load_vector(&dst->q_norm, src->q_norm, c->head_dim)) ||
- (c->qk_norm && load_vector(&dst->k_norm, src->k_norm, c->head_dim)) ||
- load_vector(&dst->input_norm, src->input_norm, c->hidden_size) ||
- load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) ||
- load_gate_up_matrix(&dst->gate_up_proj, src, c) ||
- load_matrix(&dst->down_proj, &src->down_proj, c->hidden_size, c->intermediate_size);
-}
-
-/* BERT layer: attention reuses the fused qkv matrix and qkv_bias (q_dim ==
- * kv_dim == hidden); the two block LayerNorm weights load into input_norm
- * (post-attention) and post_attn_norm (post-FFN); gate_up_proj holds the single
- * up_proj. The added fields carry the biases the Qwen slots lack. */
-static int load_bert_layer(cuda_layer_t *dst, const embed_layer_t *src, const embed_config_t *c) {
- return load_qkv_matrix(&dst->qkv, src, c) ||
- load_matrix(&dst->wo, &src->wo, c->hidden_size, c->q_dim) ||
- load_qkv_bias(&dst->qkv_bias, src, c) ||
- load_vector(&dst->o_bias, src->o_bias, c->hidden_size) ||
- load_vector(&dst->input_norm, src->input_norm, c->hidden_size) ||
- load_vector(&dst->attn_ln_bias, src->attn_ln_bias, c->hidden_size) ||
- load_vector(&dst->post_attn_norm, src->post_attn_norm, c->hidden_size) ||
- load_vector(&dst->ffn_ln_bias, src->ffn_ln_bias, c->hidden_size) ||
- load_matrix(&dst->gate_up_proj, &src->up_proj, c->intermediate_size, c->hidden_size) ||
- load_vector(&dst->ffn_inter_bias, src->ffn_inter_bias, c->intermediate_size) ||
- load_matrix(&dst->down_proj, &src->down_proj, c->hidden_size, c->intermediate_size) ||
- load_vector(&dst->ffn_out_bias, src->ffn_out_bias, c->hidden_size);
-}
-
-static void free_layer(cuda_layer_t *l) {
- cuda_matrix_free(&l->qkv);
- cuda_matrix_free(&l->wo);
- cudaFree(l->qkv_bias);
- cudaFree(l->q_norm);
- cudaFree(l->k_norm);
- cudaFree(l->input_norm);
- cudaFree(l->post_attn_norm);
- cuda_matrix_free(&l->gate_up_proj);
- cuda_matrix_free(&l->down_proj);
- cudaFree(l->o_bias);
- cudaFree(l->attn_ln_bias);
- cudaFree(l->ffn_inter_bias);
- cudaFree(l->ffn_out_bias);
- cudaFree(l->ffn_ln_bias);
-}
-
-/* Build a device context from an already-loaded CPU model. own_cpu records
- * whether embed_cuda_free should release that CPU model: embed_cuda_load owns
- * the model it loaded; the late path passes a model owned by its CPU late
- * model and sets own_cpu = 0. */
-static embed_cuda_ctx_t *cuda_ctx_from_model(embed_model_t *cpu, int own_cpu) {
- if (!cpu)
- return NULL;
-
- embed_cuda_ctx_t *ctx = (embed_cuda_ctx_t *)calloc(1, sizeof(*ctx));
- if (!ctx) {
- if (own_cpu)
- embed_model_free(cpu);
- return NULL;
- }
- ctx->cpu = cpu;
- ctx->own_cpu = own_cpu;
- ctx->config = cpu->config;
-
- /* Without an explicit --cuda-weight-dtype, store weights in the model
- * file's dtype: BF16 snapshots load as BF16 (bit-exact pass-through),
- * F32 snapshots keep the exact F32 default. */
- if (!g_weights_bf16_set) {
- g_weights_bf16 = cpu->weights.embed_tokens.dtype == DTYPE_BF16;
- if (g_weights_bf16)
- fprintf(stderr, "cuda: BF16 model file; storing weights as BF16 "
- "(use --cuda-weight-dtype f32 to override)\n");
- }
-
- if (cudaStreamCreate(&ctx->stream) != cudaSuccess ||
- cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS ||
- cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) {
- embed_cuda_free(ctx);
- return NULL;
- }
-
- const embed_config_t *c = &ctx->config;
- ctx->layers = (cuda_layer_t *)calloc((size_t)c->n_layers, sizeof(*ctx->layers));
- if (!ctx->layers) {
- embed_cuda_free(ctx);
- return NULL;
- }
- if (load_matrix(&ctx->embed_tokens, &cpu->weights.embed_tokens, c->vocab_size,
- c->hidden_size) != 0) {
- embed_cuda_free(ctx);
- return NULL;
- }
- if (c->family == EMBED_FAMILY_BERT) {
- /* BERT has no final norm; it adds learned position + token-type[0]
- * embeddings and an embedding LayerNorm (only row 0 of token-type is
- * ever used, so load just that row). */
- const embed_weights_t *w = &cpu->weights;
- if (load_vector(&ctx->position_embeddings, w->position_embeddings,
- c->max_position_embeddings * c->hidden_size) != 0 ||
- load_vector(&ctx->token_type_embedding, w->token_type_embeddings, c->hidden_size) != 0 ||
- load_vector(&ctx->embed_ln_w, w->embed_ln_w, c->hidden_size) != 0 ||
- load_vector(&ctx->embed_ln_b, w->embed_ln_b, c->hidden_size) != 0) {
- embed_cuda_free(ctx);
- return NULL;
- }
- } else if (load_vector(&ctx->norm, cpu->weights.norm, c->hidden_size) != 0) {
- embed_cuda_free(ctx);
- return NULL;
- }
- for (int i = 0; i < c->n_layers; i++) {
- int rc = c->family == EMBED_FAMILY_BERT
- ? load_bert_layer(&ctx->layers[i], &cpu->weights.layers[i], c)
- : load_layer(&ctx->layers[i], &cpu->weights.layers[i], c);
- if (rc != 0) {
- fprintf(stderr, "cuda: failed to load layer %d\n", i);
- embed_cuda_free(ctx);
- return NULL;
- }
- }
- cudaStreamSynchronize(ctx->stream);
- return ctx;
-}
-
-embed_cuda_ctx_t *embed_cuda_load(const char *model_dir) {
- return cuda_ctx_from_model(embed_model_load(model_dir), 1);
-}
-
-void embed_cuda_free(embed_cuda_ctx_t *ctx) {
- if (!ctx)
- return;
- cuda_matrix_free(&ctx->embed_tokens);
- if (ctx->layers) {
- for (int i = 0; i < ctx->config.n_layers; i++)
- free_layer(&ctx->layers[i]);
- free(ctx->layers);
- }
- cudaFree(ctx->norm);
- cudaFree(ctx->position_embeddings);
- cudaFree(ctx->token_type_embedding);
- cudaFree(ctx->embed_ln_w);
- cudaFree(ctx->embed_ln_b);
- cudaFree(ctx->x);
- cudaFree(ctx->x_norm);
- cudaFree(ctx->qkv);
- cudaFree(ctx->attn_out);
- cudaFree(ctx->ffn_gate_up);
- cudaFree(ctx->pooled_out);
- cudaFree(ctx->rope_cos);
- cudaFree(ctx->rope_sin);
- cudaFree(ctx->token_ids);
- cudaFree(ctx->offsets);
- free(ctx->offsets_host);
- cudaFree(ctx->positions);
- cudaFree(ctx->kexp);
- cudaFree(ctx->vexp);
- cudaFree(ctx->act_bf16);
- cudaFree(ctx->weight_f32);
- cudaFree(ctx->attn_scores);
- cudaFree(ctx->attn_probs);
- cudaFree((void *)ctx->attn_ptrs);
- free((void *)ctx->attn_ptrs_host);
- cudaFree(ctx->span_starts);
- cudaFree(ctx->span_lens);
- if (ctx->blas)
- cublasDestroy(ctx->blas);
- if (ctx->stream)
- cudaStreamDestroy(ctx->stream);
- if (ctx->own_cpu)
- embed_model_free(ctx->cpu);
- free(ctx);
-}
-
-const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx) {
- return ctx ? &ctx->config : NULL;
-}
-
-// Pack inputs into the device scratch every forward shares: token ids, the
-// per-sequence positions 0..L-1, and the batch offsets. Grows scratch via
-// ensure_buffers. Returns the packed token count in *out_total, or -1 on error.
-static int cuda_upload_packed_inputs(embed_cuda_ctx_t *ctx,
- const embed_input_t *inputs,
- int batch,
- int *out_total) {
- int *h_offsets = (int *)malloc((size_t)(batch + 1) * sizeof(int));
- if (!h_offsets)
- return -1;
- int total = 0, max_seq = 0;
- for (int b = 0; b < batch; b++) {
- if (!inputs[b].ids || inputs[b].n_tokens <= 0) {
- free(h_offsets);
- return -1;
- }
- h_offsets[b] = total;
- total += inputs[b].n_tokens;
- if (inputs[b].n_tokens > max_seq)
- max_seq = inputs[b].n_tokens;
- }
- if (ctx->config.family == EMBED_FAMILY_BERT &&
- max_seq > ctx->config.max_position_embeddings - ctx->config.position_id_offset) {
- free(h_offsets);
- return -1;
- }
- h_offsets[batch] = total;
- int *h_ids = (int *)malloc((size_t)total * sizeof(int));
- int *h_pos = (int *)malloc((size_t)total * sizeof(int));
- if (!h_ids || !h_pos) {
- free(h_offsets);
- free(h_ids);
- free(h_pos);
- return -1;
- }
- for (int b = 0; b < batch; b++) {
- int off = h_offsets[b];
- memcpy(h_ids + off, inputs[b].ids, (size_t)inputs[b].n_tokens * sizeof(int));
- for (int i = 0; i < inputs[b].n_tokens; i++)
- h_pos[off + i] = ctx->config.position_id_offset + i;
- }
- if (ensure_buffers(ctx, total, batch, max_seq) != 0) {
- free(h_offsets);
- free(h_ids);
- free(h_pos);
- return -1;
- }
- memcpy(ctx->offsets_host, h_offsets, (size_t)(batch + 1) * sizeof(int));
- cudaError_t ce = cudaMemcpyAsync(ctx->token_ids, h_ids, (size_t)total * sizeof(int),
- cudaMemcpyHostToDevice, ctx->stream);
- if (ce == cudaSuccess)
- ce = cudaMemcpyAsync(ctx->positions, h_pos, (size_t)total * sizeof(int),
- cudaMemcpyHostToDevice, ctx->stream);
- if (ce == cudaSuccess)
- ce = cudaMemcpyAsync(ctx->offsets, h_offsets, (size_t)(batch + 1) * sizeof(int),
- cudaMemcpyHostToDevice, ctx->stream);
- free(h_offsets);
- free(h_ids);
- free(h_pos);
- if (ce != cudaSuccess) {
- fprintf(stderr, "cuda input copy failed: %s\n", cudaGetErrorString(ce));
- return -1;
- }
- *out_total = total;
- return 0;
-}
-
-// BERT-family forward (post-norm encoder), mirroring the CPU forward_packed_bert.
-// Embeddings: token + learned position + token-type[0], then the embedding
-// LayerNorm. Each layer: QKV(+bias) from x directly -> bidirectional attention
-// -> output dense(+bias) -> residual -> LayerNorm -> dense(+bias) -> GeLU ->
-// dense(+bias) -> residual -> LayerNorm. No RoPE, no qk-norm, no final norm.
-// F32 activations: BERT checkpoints are F32, so the linear/linear_accum F32
-// entry points are used throughout (they still handle bf16 weights via an
-// internal cast, which keeps a future bf16 BERT correct if not yet optimized).
-static int cuda_forward_batch_bert(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) {
- const embed_config_t *c = &ctx->config;
- int total = 0;
- if (cuda_upload_packed_inputs(ctx, inputs, batch, &total) != 0)
- return -1;
-
- int hidden = c->hidden_size;
- int inter = c->intermediate_size;
- int qkv_dim = c->q_dim + 2 * c->kv_dim;
- int q_offset = 0, k_offset = c->q_dim, v_offset = c->q_dim + c->kv_dim;
- float scale = 1.0f / sqrtf((float)c->head_dim);
- float eps = c->layer_norm_eps;
- int threads = 256;
- int blocks_hidden = (total * hidden + threads - 1) / threads;
-
- embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
- ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, hidden,
- c->vocab_size);
- if (launch_check() != 0)
- return -1;
- bert_pos_type_add_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
- ctx->x, ctx->positions, ctx->position_embeddings, ctx->token_type_embedding, total, hidden);
- if (launch_check() != 0)
- return -1;
- layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->embed_ln_w,
- ctx->embed_ln_b, total, hidden, eps);
- if (launch_check() != 0)
- return -1;
-
- if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0)
- return -1;
-
- for (int layer = 0; layer < c->n_layers; layer++) {
- cuda_layer_t *l = &ctx->layers[layer];
-
- // Self-attention (post-norm): QKV read x directly, then +bias.
- if (linear(ctx, &l->qkv, ctx->x, ctx->qkv, total, hidden, qkv_dim) != 0)
- return -1;
- int qkv_count = total * qkv_dim;
- add_row_bias_kernel<<<(qkv_count + 255) / 256, 256, 0, ctx->stream>>>(ctx->qkv, l->qkv_bias,
- total, qkv_dim);
- if (launch_check() != 0)
- return -1;
- if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset,
- qkv_dim, scale, NULL) != 0)
- return -1;
-
- // Output dense(+bias), residual into x, then post-attention LayerNorm.
- if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, total, c->q_dim, hidden) != 0)
- return -1;
- add_row_bias_kernel<<<(total * hidden + 255) / 256, 256, 0, ctx->stream>>>(
- ctx->x, l->o_bias, total, hidden);
- if (launch_check() != 0)
- return -1;
- layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, l->input_norm,
- l->attn_ln_bias, total, hidden, eps);
- if (launch_check() != 0)
- return -1;
-
- // Feed-forward (post-norm): dense(+bias) -> GeLU -> dense(+bias).
- if (linear(ctx, &l->gate_up_proj, ctx->x, ctx->ffn_gate_up, total, hidden, inter) != 0)
- return -1;
- int inter_count = total * inter;
- add_row_bias_kernel<<<(inter_count + 255) / 256, 256, 0, ctx->stream>>>(
- ctx->ffn_gate_up, l->ffn_inter_bias, total, inter);
- if (launch_check() != 0)
- return -1;
- if (c->ffn_act == EMBED_ACT_GELU_TANH)
- gelu_tanh_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
- ctx->ffn_gate_up, (size_t)inter_count);
- else
- gelu_kernel<<<(inter_count + threads - 1) / threads, threads, 0, ctx->stream>>>(
- ctx->ffn_gate_up, (size_t)inter_count);
- if (launch_check() != 0)
- return -1;
- if (linear_accum(ctx, &l->down_proj, ctx->ffn_gate_up, ctx->x, total, inter, hidden) != 0)
- return -1;
- add_row_bias_kernel<<<(total * hidden + 255) / 256, 256, 0, ctx->stream>>>(
- ctx->x, l->ffn_out_bias, total, hidden);
- if (launch_check() != 0)
- return -1;
- layer_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, l->post_attn_norm,
- l->ffn_ln_bias, total, hidden, eps);
- if (launch_check() != 0)
- return -1;
- }
- // No final norm; embed_cuda_encode_batch pools ctx->x directly.
- return 0;
-}
-
-static int cuda_forward_batch(embed_cuda_ctx_t *ctx, const embed_input_t *inputs, int batch) {
- if (!ctx || !inputs || batch <= 0)
- return -1;
- const embed_config_t *c = &ctx->config;
- if (c->family == EMBED_FAMILY_BERT)
- return cuda_forward_batch_bert(ctx, inputs, batch);
- int total = 0;
- if (cuda_upload_packed_inputs(ctx, inputs, batch, &total) != 0)
- return -1;
-
- int threads = 256;
- int blocks_hidden = (total * c->hidden_size + threads - 1) / threads;
- embed_lookup_kernel<<<blocks_hidden, threads, 0, ctx->stream>>>(
- ctx->x, ctx->token_ids, ctx->embed_tokens.d, ctx->embed_tokens.bf16, total, c->hidden_size,
- c->vocab_size);
- if (launch_check() != 0)
- return -1;
-
- float scale = 1.0f / sqrtf((float)c->head_dim);
- int q_offset = 0;
- int k_offset = c->q_dim;
- int v_offset = c->q_dim + c->kv_dim;
- int qkv_dim = c->q_dim + 2 * c->kv_dim;
- // With BF16 weights every producer (norm, SiLU, attention @V GEMM)
- // stores its result as BF16 directly into ctx->act_bf16, which the next
- // projection GEMM consumes before any later producer reuses the buffer
- // (single stream). No cast launches remain in the layer loop.
- __nv_bfloat16 *act = (__nv_bfloat16 *)ctx->act_bf16;
- if (attn_batched_setup(ctx, batch, q_offset, qkv_dim) != 0)
- return -1;
- for (int layer = 0; layer < c->n_layers; layer++) {
- cuda_layer_t *l = &ctx->layers[layer];
- int wbf16 = l->qkv.bf16;
- if (wbf16) {
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(act, ctx->x, l->input_norm, total,
- c->hidden_size, c->rms_norm_eps);
- if (launch_check() != 0)
- return -1;
- if (linear_bf16x(ctx, &l->qkv, act, ctx->qkv, total, c->hidden_size, qkv_dim,
- c->hidden_size, 0.0f) != 0)
- return -1;
- } else {
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->input_norm,
- total, c->hidden_size, c->rms_norm_eps);
- if (launch_check() != 0)
- return -1;
- if (linear(ctx, &l->qkv, ctx->x_norm, ctx->qkv, total, c->hidden_size, qkv_dim) != 0)
- return -1;
- }
-
- if (c->qkv_bias) {
- int count = total * qkv_dim;
- add_row_bias_kernel<<<(count + 255) / 256, 256, 0, ctx->stream>>>(ctx->qkv, l->qkv_bias,
- total, qkv_dim);
- if (launch_check() != 0)
- return -1;
- }
-
- rms_norm_rope_qk_kernel<<<dim3(total, c->n_heads + c->n_kv_heads), 128, 0, ctx->stream>>>(
- ctx->qkv, l->q_norm, l->k_norm, ctx->positions, ctx->rope_cos, ctx->rope_sin, total,
- c->n_heads, c->n_kv_heads, c->head_dim, qkv_dim, q_offset, k_offset, c->qk_norm,
- c->rms_norm_eps);
- if (launch_check() != 0)
- return -1;
-
- if (cuda_attention_gemm(ctx, ctx->offsets_host, batch, q_offset, k_offset, v_offset,
- qkv_dim, scale, wbf16 ? act : NULL) != 0)
- return -1;
-
- if (wbf16) {
- if (linear_bf16x(ctx, &l->wo, act, ctx->x, total, c->q_dim, c->hidden_size, c->q_dim,
- 1.0f) != 0)
- return -1;
- } else {
- if (linear_accum(ctx, &l->wo, ctx->attn_out, ctx->x, total, c->q_dim, c->hidden_size) !=
- 0)
- return -1;
- }
-
- int inter_count = total * c->intermediate_size;
- if (wbf16) {
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(act, ctx->x, l->post_attn_norm, total,
- c->hidden_size, c->rms_norm_eps);
- if (launch_check() != 0)
- return -1;
- if (linear_bf16x(ctx, &l->gate_up_proj, act, ctx->ffn_gate_up, total, c->hidden_size,
- 2 * c->intermediate_size, c->hidden_size, 0.0f) != 0)
- return -1;
- silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, threads, 0,
- ctx->stream>>>(act, c->intermediate_size, ctx->ffn_gate_up,
- total, c->intermediate_size);
- if (launch_check() != 0)
- return -1;
- if (linear_bf16x(ctx, &l->down_proj, act, ctx->x, total, c->intermediate_size,
- c->hidden_size, c->intermediate_size, 1.0f) != 0)
- return -1;
- } else {
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x_norm, ctx->x, l->post_attn_norm,
- total, c->hidden_size, c->rms_norm_eps);
- if (launch_check() != 0)
- return -1;
- if (linear(ctx, &l->gate_up_proj, ctx->x_norm, ctx->ffn_gate_up, total, c->hidden_size,
- 2 * c->intermediate_size) != 0)
- return -1;
- silu_mul_packed_kernel<<<(inter_count + threads - 1) / threads, threads, 0,
- ctx->stream>>>(ctx->ffn_gate_up, 2 * c->intermediate_size,
- ctx->ffn_gate_up, total, c->intermediate_size);
- if (launch_check() != 0)
- return -1;
- if (linear_ex(ctx, &l->down_proj, ctx->ffn_gate_up, 0, ctx->x, total,
- c->intermediate_size, c->hidden_size, 2 * c->intermediate_size,
- 1.0f) != 0)
- return -1;
- }
- }
-
- rms_norm_kernel<<<total, 256, 0, ctx->stream>>>(ctx->x, ctx->x, ctx->norm, total,
- c->hidden_size, c->rms_norm_eps);
- if (launch_check() != 0)
- return -1;
-
- return 0;
-}
-
-int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!ctx || !inputs || batch <= 0 || !out_embeddings)
- return -1;
- const embed_config_t *c = &ctx->config;
- if (cuda_forward_batch(ctx, inputs, batch) != 0)
- return -1;
- if (ensure_pooled_rows(ctx, batch) != 0)
- return -1;
- if (c->pooling_mode == EMBED_POOL_LAST_TOKEN)
- last_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
- batch, c->hidden_size);
- else if (c->pooling_mode == EMBED_POOL_CLS)
- first_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
- batch, c->hidden_size);
- else
- mean_pool_kernel<<<batch, 256, 0, ctx->stream>>>(ctx->pooled_out, ctx->x, ctx->offsets,
- batch, c->hidden_size);
- if (launch_check() != 0)
- return -1;
- cudaError_t ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
- (size_t)batch * c->hidden_size * sizeof(float),
- cudaMemcpyDeviceToHost, ctx->stream);
- if (ce == cudaSuccess)
- ce = cudaStreamSynchronize(ctx->stream);
- if (ce != cudaSuccess) {
- fprintf(stderr, "cuda output copy failed: %s\n", cudaGetErrorString(ce));
- return -1;
- }
- if (c->normalize_embeddings) {
- for (int b = 0; b < batch; b++) {
- if (embed_l2_normalize(out_embeddings + (size_t)b * c->hidden_size, c->hidden_size) !=
- 0)
- return -1;
- }
- }
- return 0;
-}
-
-int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!ctx || !inputs || batch <= 0 || !out_embeddings)
- return -1;
- const embed_config_t *c = &ctx->config;
- int total_spans = 0;
- int total_tokens = 0;
- for (int b = 0; b < batch; b++) {
- const embed_context_input_t *input = &inputs[b];
- int n_tokens = input->input.n_tokens;
- if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
- total_spans > INT_MAX - input->n_spans || total_tokens > INT_MAX - n_tokens)
- return -1;
- for (int s = 0; s < input->n_spans; s++) {
- int start = input->spans[s].start;
- int len = input->spans[s].n_tokens;
- if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start)
- return -1;
- }
- total_spans += input->n_spans;
- total_tokens += n_tokens;
- }
-
- embed_input_t *packed = (embed_input_t *)malloc((size_t)batch * sizeof(*packed));
- int *h_starts = (int *)malloc((size_t)total_spans * sizeof(int));
- int *h_lens = (int *)malloc((size_t)total_spans * sizeof(int));
- if (!packed || !h_starts || !h_lens) {
- free(packed);
- free(h_starts);
- free(h_lens);
- return -1;
- }
-
- int token_offset = 0;
- int span_offset = 0;
- for (int b = 0; b < batch; b++) {
- const embed_context_input_t *input = &inputs[b];
- packed[b] = input->input;
- for (int s = 0; s < input->n_spans; s++) {
- h_starts[span_offset] = token_offset + input->spans[s].start;
- h_lens[span_offset] = input->spans[s].n_tokens;
- span_offset++;
- }
- token_offset += input->input.n_tokens;
- }
-
- int rc = -1;
- cudaError_t ce;
- if (cuda_forward_batch(ctx, packed, batch) != 0)
- goto cleanup;
- if (ensure_pooled_rows(ctx, total_spans) != 0 || ensure_span_buffers(ctx, total_spans) != 0)
- goto cleanup;
- ce = cudaMemcpyAsync(ctx->span_starts, h_starts, (size_t)total_spans * sizeof(int),
- cudaMemcpyHostToDevice, ctx->stream);
- if (ce == cudaSuccess)
- ce = cudaMemcpyAsync(ctx->span_lens, h_lens, (size_t)total_spans * sizeof(int),
- cudaMemcpyHostToDevice, ctx->stream);
- if (ce != cudaSuccess) {
- fprintf(stderr, "cuda span copy failed: %s\n", cudaGetErrorString(ce));
- goto cleanup;
- }
- span_pool_kernel<<<total_spans, 256, 0, ctx->stream>>>(
- ctx->pooled_out, ctx->x, ctx->span_starts, ctx->span_lens, total_spans, c->hidden_size);
- if (launch_check() != 0)
- goto cleanup;
- ce = cudaMemcpyAsync(out_embeddings, ctx->pooled_out,
- (size_t)total_spans * c->hidden_size * sizeof(float),
- cudaMemcpyDeviceToHost, ctx->stream);
- if (ce == cudaSuccess)
- ce = cudaStreamSynchronize(ctx->stream);
- if (ce != cudaSuccess) {
- fprintf(stderr, "cuda span output copy failed: %s\n", cudaGetErrorString(ce));
- goto cleanup;
- }
- rc = 0;
-
-cleanup:
- free(h_lens);
- free(h_starts);
- free(packed);
- return rc;
-}
-
-int embed_cuda_encode_into(embed_cuda_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- float *out_embedding) {
- embed_input_t input = {token_ids, n_tokens};
- return embed_cuda_encode_batch(ctx, &input, 1, out_embedding);
-}
-
-float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens) {
- if (!ctx)
- return NULL;
- int hidden = ctx->config.hidden_size;
- float *out = (float *)malloc((size_t)hidden * sizeof(float));
- if (!out)
- return NULL;
- if (embed_cuda_encode_into(ctx, token_ids, n_tokens, out) != 0) {
- free(out);
- return NULL;
- }
- return out;
-}
-
-/* ========================================================================
- * Late-interaction CUDA path
- *
- * The transformer forward and the 1_Dense projection run on the GPU; MaxSim
- * scoring stays on the host, where the grouped-GEMM scorer is far faster than a
- * device graph for this small op. So the encoders copy the projected token
- * vectors back to host memory and the caller runs embed_late_maxsim_batch there.
- * ======================================================================== */
-
-struct embed_cuda_late_ctx {
- embed_cuda_ctx_t *base; // device base model; does not own its CPU model
- embed_late_model_t *cpu_late; // owns the CPU base model + projection host data
- cuda_matrix_t projection; // device [token_dim, hidden]
- float *proj_dev; // device scratch [total_tokens, token_dim], grown on demand
- size_t proj_cap; // capacity of proj_dev in floats
- int token_dim;
-};
-
-embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir) {
- embed_late_model_t *cpu_late = embed_late_model_load(model_dir);
- if (!cpu_late)
- return NULL;
-
- embed_cuda_late_ctx_t *ctx = (embed_cuda_late_ctx_t *)calloc(1, sizeof(*ctx));
- if (!ctx) {
- embed_late_model_free(cpu_late);
- return NULL;
- }
- ctx->cpu_late = cpu_late;
- ctx->token_dim = embed_late_model_token_dim(cpu_late);
-
- // Upload the base transformer weights; the CPU late model keeps ownership.
- ctx->base = cuda_ctx_from_model(embed_late_model_base(cpu_late), 0);
- if (!ctx->base) {
- embed_cuda_late_free(ctx);
- return NULL;
- }
-
- // Upload the 1_Dense projection [token_dim, hidden].
- const embed_weight_ref_t *proj = embed_late_model_projection(cpu_late);
- if (!proj || ctx->token_dim <= 0 ||
- load_matrix(&ctx->projection, proj, ctx->token_dim, ctx->base->config.hidden_size) != 0) {
- embed_cuda_late_free(ctx);
- return NULL;
- }
- return ctx;
-}
-
-void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx) {
- if (!ctx)
- return;
- cuda_matrix_free(&ctx->projection);
- cudaFree(ctx->proj_dev);
- embed_cuda_free(ctx->base); // own_cpu = 0, so this keeps the CPU model
- embed_late_model_free(ctx->cpu_late); // frees the CPU base model + projection
- free(ctx);
-}
-
-const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx) {
- return ctx ? &ctx->base->config : NULL;
-}
-
-int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; }
-
-static int ensure_late_proj(embed_cuda_late_ctx_t *ctx, int total) {
- size_t need = (size_t)total * (size_t)ctx->token_dim;
- if (need <= ctx->proj_cap)
- return 0;
- cudaFree(ctx->proj_dev);
- ctx->proj_dev = NULL;
- ctx->proj_cap = 0;
- if (cudaMalloc((void **)&ctx->proj_dev, need * sizeof(float)) != cudaSuccess)
- return -1;
- ctx->proj_cap = need;
- return 0;
-}
-
-/* Forward all inputs, project every token to token_dim, and copy the packed
- * [total, token_dim] F32 result to a freshly malloc'd host buffer (caller frees).
- * On success *proj_host and *total_out are set; base->offsets_host holds the
- * packed per-document token offsets. */
-static int cuda_late_forward_project(embed_cuda_late_ctx_t *ctx,
- const embed_input_t *inputs,
- int n_docs,
- float **proj_host,
- int *total_out) {
- embed_cuda_ctx_t *base = ctx->base;
- const embed_config_t *c = &base->config;
- if (cuda_forward_batch(base, inputs, n_docs) != 0)
- return -1;
- int total = base->offsets_host[n_docs];
- if (total <= 0 || ensure_late_proj(ctx, total) != 0)
- return -1;
- if (linear(base, &ctx->projection, base->x, ctx->proj_dev, total, c->hidden_size,
- ctx->token_dim) != 0)
- return -1;
- float *host = (float *)malloc((size_t)total * (size_t)ctx->token_dim * sizeof(float));
- if (!host)
- return -1;
- cudaError_t ce =
- cudaMemcpyAsync(host, ctx->proj_dev, (size_t)total * (size_t)ctx->token_dim * sizeof(float),
- cudaMemcpyDeviceToHost, base->stream);
- if (ce == cudaSuccess)
- ce = cudaStreamSynchronize(base->stream);
- if (ce != cudaSuccess) {
- fprintf(stderr, "cuda late output copy failed: %s\n", cudaGetErrorString(ce));
- free(host);
- return -1;
- }
- *proj_host = host;
- *total_out = total;
- return 0;
-}
-
-int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors) {
- if (!ctx || !token_ids || n_tokens <= 0 || !out_vectors || ctx->token_dim <= 0)
- return -1;
- embed_input_t input = {token_ids, n_tokens};
- float *proj_host = NULL;
- int total = 0;
- if (cuda_late_forward_project(ctx, &input, 1, &proj_host, &total) != 0)
- return -1;
- memcpy(out_vectors, proj_host, (size_t)total * (size_t)ctx->token_dim * sizeof(float));
- free(proj_host);
- if (normalize) {
- for (int i = 0; i < total; i++)
- if (embed_l2_normalize(out_vectors + (size_t)i * ctx->token_dim, ctx->token_dim) != 0)
- return -1;
- }
- return 0;
-}
-
-int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets) {
- if (!ctx || !doc_ids || !n_tokens || !keep || !n_keep || n_docs <= 0 || !out_vectors ||
- !out_offsets || ctx->token_dim <= 0)
- return -1;
- int dim = ctx->token_dim;
-
- embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
- if (!inputs)
- return -1;
-
- int total_keep = 0;
- out_offsets[0] = 0;
- for (int d = 0; d < n_docs; d++) {
- if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
- n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) {
- free(inputs);
- return -1;
- }
- for (int k = 0; k < n_keep[d]; k++) {
- if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) {
- free(inputs);
- return -1;
- }
- }
- inputs[d].ids = doc_ids[d];
- inputs[d].n_tokens = n_tokens[d];
- total_keep += n_keep[d];
- out_offsets[d + 1] = total_keep;
- }
-
- float *proj_host = NULL;
- int total = 0;
- if (cuda_late_forward_project(ctx, inputs, n_docs, &proj_host, &total) != 0) {
- free(inputs);
- return -1;
- }
-
- /* Gather each document's kept token rows (in packed forward order) into the
- * output, the layout MaxSim consumes - matching the CPU/MLX paths. */
- const int *off = ctx->base->offsets_host;
- int pos = 0;
- for (int d = 0; d < n_docs; d++) {
- int start = off[d];
- for (int k = 0; k < n_keep[d]; k++) {
- memcpy(out_vectors + (size_t)pos * dim, proj_host + (size_t)(start + keep[d][k]) * dim,
- (size_t)dim * sizeof(float));
- pos++;
- }
- }
- free(proj_host);
- free(inputs);
-
- if (normalize) {
- for (int i = 0; i < total_keep; i++)
- if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
- return -1;
- }
- return 0;
-}
diff --git a/src/cuda.h b/src/cuda.h
deleted file mode 100644
index aad20ab..0000000
--- a/src/cuda.h
+++ /dev/null
@@ -1,80 +0,0 @@
-/*
- * embed_cuda.h - CUDA/cuBLAS backend
- */
-
-#ifndef EMBED_CUDA_H
-#define EMBED_CUDA_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-#include "embed.h"
-
-typedef struct embed_cuda_ctx embed_cuda_ctx_t;
-typedef struct embed_cuda_late_ctx embed_cuda_late_ctx_t;
-
-EMBED_API int embed_cuda_set_fast_gemm(const char *mode);
-EMBED_API int embed_cuda_set_weights_bf16(int on);
-
-EMBED_API embed_cuda_ctx_t *embed_cuda_load(const char *model_dir);
-EMBED_API void embed_cuda_free(embed_cuda_ctx_t *ctx);
-
-/*
- * Late-interaction CUDA path. Encoding (transformer forward + projection) runs
- * on the GPU; MaxSim scoring stays on the host, where the grouped-GEMM scorer is
- * much faster than a device graph for this small op. The encoders therefore copy
- * the projected token vectors back to host memory.
- */
-EMBED_API embed_cuda_late_ctx_t *embed_cuda_late_load(const char *model_dir);
-EMBED_API void embed_cuda_late_free(embed_cuda_late_ctx_t *ctx);
-EMBED_API const embed_config_t *embed_cuda_late_config(const embed_cuda_late_ctx_t *ctx);
-EMBED_API int embed_cuda_late_token_dim(const embed_cuda_late_ctx_t *ctx);
-
-/* Encode token_ids into out_vectors[n_tokens, token_dim] (host). */
-EMBED_API int embed_cuda_late_encode_tokens(embed_cuda_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors);
-
-/*
- * Encode n_docs documents in one packed forward and pack each document's kept
- * token vectors back-to-back into out_vectors[total_keep, token_dim] (host) in
- * document order. out_offsets[n_docs + 1] receives the prefix sum of kept counts
- * (the layout embed_late_maxsim_batch consumes). Returns 0 on success.
- */
-EMBED_API int embed_cuda_late_encode_docs(embed_cuda_late_ctx_t *ctx,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets);
-
-EMBED_API const embed_config_t *embed_cuda_config(const embed_cuda_ctx_t *ctx);
-
-EMBED_API int embed_cuda_encode_into(embed_cuda_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- float *out_embedding);
-EMBED_API float *embed_cuda_encode(embed_cuda_ctx_t *ctx, const int *token_ids, int n_tokens);
-EMBED_API int embed_cuda_encode_batch(embed_cuda_ctx_t *ctx,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings);
-EMBED_API int embed_cuda_encode_spans_batch(embed_cuda_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings);
-
-#endif /* EMBED_CUDA_H */
diff --git a/src/dtype.h b/src/dtype.h
deleted file mode 100644
index 2a25cae..0000000
--- a/src/dtype.h
+++ /dev/null
@@ -1,55 +0,0 @@
-#ifndef EMBED_DTYPE_H
-#define EMBED_DTYPE_H
-
-#include <stdint.h>
-#include <string.h>
-
-/* BF16 is the upper 16 bits of an IEEE-754 binary32, so widening one value is a
- * left shift reinterpreted as a float. Shared by every host-side caller (the
- * CPU loader, the safetensors reader); the CUDA backend keeps its own
- * __nv_bfloat16 device kernels, and kernels.c keeps the SIMD buffer variant
- * embed_bf16_to_f32_buf for hot, contiguous conversions. */
-static inline float embed_bf16_to_f32(uint16_t bf16) {
- uint32_t bits = (uint32_t)bf16 << 16;
- float f;
- memcpy(&f, &bits, sizeof(f));
- return f;
-}
-
-/* IEEE-754 binary16 -> binary32. Unlike BF16, F16 has a 5-bit exponent and a
- * 10-bit mantissa, so widening is a field re-pack, not a shift: rebias the
- * exponent (15 -> 127), shift the mantissa up by 13, and special-case zeros,
- * subnormals, and inf/NaN. Used by the loaders to accept F16 safetensors
- * snapshots (e.g. the public multilingual-e5 weights); there is no fused F16
- * kernel, so weights are widened to F32 once at load. */
-static inline float embed_f16_to_f32(uint16_t f16) {
- uint32_t sign = (uint32_t)(f16 & 0x8000u) << 16;
- uint32_t exp = (f16 >> 10) & 0x1fu;
- uint32_t mant = f16 & 0x3ffu;
- uint32_t bits;
- if (exp == 0) {
- if (mant == 0) {
- bits = sign; /* signed zero */
- } else {
- /* Subnormal half: renormalize into a normal float. Shift the
- * mantissa left until its leading 1 reaches the hidden-bit
- * position, decrementing the exponent for each shift. */
- exp = 127 - 15 + 1;
- while ((mant & 0x400u) == 0) {
- mant <<= 1;
- exp--;
- }
- mant &= 0x3ffu;
- bits = sign | (exp << 23) | (mant << 13);
- }
- } else if (exp == 0x1fu) {
- bits = sign | 0x7f800000u | (mant << 13); /* inf or NaN */
- } else {
- bits = sign | ((exp + (127 - 15)) << 23) | (mant << 13);
- }
- float f;
- memcpy(&f, &bits, sizeof(f));
- return f;
-}
-
-#endif /* EMBED_DTYPE_H */
diff --git a/src/embed.h b/src/embed.h
deleted file mode 100644
index 5c4f2f3..0000000
--- a/src/embed.h
+++ /dev/null
@@ -1,357 +0,0 @@
-/*
- * embed.h - embed API
- */
-
-#ifndef EMBED_H
-#define EMBED_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-#include <stddef.h>
-#include <stdint.h>
-
-/* ========================================================================
- * Shared constants and family defaults
- * ======================================================================== */
-
-#define EMBED_VOCAB_SIZE 151936
-#define EMBED_HEAD_DIM 128
-#define EMBED_MAX_LAYERS 64 /* upper bound for stack arrays */
-#define EMBED_CONTEXT_SEPARATOR_TOKEN_ID 151643 /* <|endoftext|> */
-
-/* ========================================================================
- * Model Configuration (populated from config.json at load time)
- * ======================================================================== */
-
-typedef enum {
- EMBED_ATTENTION_BIDIRECTIONAL = 0,
- EMBED_ATTENTION_CAUSAL = 1,
-} embed_attention_mode_t;
-
-typedef enum {
- EMBED_POOL_MEAN = 0,
- EMBED_POOL_LAST_TOKEN = 1,
- EMBED_POOL_CLS = 2, /* first token; the encoder-family [CLS] sentence vector */
-} embed_pooling_mode_t;
-
-typedef enum {
- EMBED_FAMILY_QWEN3 = 0, /* RMSNorm + RoPE + SwiGLU decoder block (pplx, Qwen2/3) */
- EMBED_FAMILY_BERT = 1, /* LayerNorm + learned positions + GeLU encoder block */
-} embed_family_t;
-
-/* BERT feed-forward activation. The released MiniLM/BGE encoders use exact erf
- * GeLU; some encoders publish the tanh approximation (hidden_act gelu_new or
- * gelu_pytorch_tanh), which is a different curve and must be matched exactly. */
-typedef enum {
- EMBED_ACT_GELU_ERF = 0, /* 0.5 x (1 + erf(x / sqrt(2))) */
- EMBED_ACT_GELU_TANH = 1, /* 0.5 x (1 + tanh(sqrt(2/pi) (x + 0.044715 x^3))) */
-} embed_act_t;
-
-typedef struct {
- embed_family_t family;
- embed_act_t ffn_act; /* BERT feed-forward GeLU variant; ERF for Qwen/SwiGLU */
- int hidden_size;
- int n_layers;
- int n_heads;
- int n_kv_heads;
- int head_dim;
- int q_dim; /* n_heads * head_dim */
- int kv_dim; /* n_kv_heads * head_dim */
- int intermediate_size;
- int vocab_size;
- int max_position_embeddings; /* BERT learned position table size */
- int position_id_offset; /* RoBERTa/XLM-R positions start at pad_id + 1 */
- int type_vocab_size; /* BERT token-type embedding rows; XLM-R uses 1 */
- float rms_norm_eps;
- float layer_norm_eps; /* BERT LayerNorm epsilon */
- float rope_theta;
- int qk_norm;
- int qkv_bias;
- embed_attention_mode_t attention_mode;
- embed_pooling_mode_t pooling_mode;
- int normalize_embeddings;
- /* Qwen3-Embedding pools the last token, which is the tokenizer's
- * <|endoftext|> suffix added by its post-processor - not the model's chat
- * eos_token_id (<|im_end|>). Text frontends append it and resolve the id
- * from the tokenizer exactly like the contextual separator
- * (EMBED_CONTEXT_SEPARATOR_TOKEN_ID is the released-model fallback). The
- * model forward consumes token ids exactly and never appends. */
- int append_terminal_token;
-} embed_config_t;
-
-/* ========================================================================
- * Model + workspace ownership
- * ======================================================================== */
-
-typedef struct embed_model embed_model_t;
-typedef struct embed_workspace embed_workspace_t;
-typedef struct embed_late_model embed_late_model_t;
-typedef struct embed_late_workspace embed_late_workspace_t;
-
-typedef struct {
- const int *ids;
- int n_tokens;
-} embed_input_t;
-
-typedef struct {
- int start;
- int n_tokens;
-} embed_span_t;
-
-typedef struct {
- embed_input_t input;
- const embed_span_t *spans;
- int n_spans;
-} embed_context_input_t;
-
-/* ========================================================================
- * API
- * ======================================================================== */
-
-/*
- * Load immutable model data from a directory containing:
- * config.json - model hyperparameters
- * model.safetensors (or sharded) - F32 or BF16 weights
- * vocab.json + merges.txt - BPE tokenizer
- *
- * Reads config.json to determine model size (0.6B vs 4B).
- * Returns NULL on error.
- */
-EMBED_API embed_model_t *embed_model_load(const char *model_dir);
-
-EMBED_API void embed_model_free(embed_model_t *model);
-
-/*
- * Allocate/free mutable scratch buffers for a model. The model must outlive
- * workspaces created from it.
- */
-EMBED_API embed_workspace_t *embed_workspace_new(const embed_model_t *model);
-EMBED_API void embed_workspace_free(embed_workspace_t *ws);
-
-/* Config accessor for opaque model ownership. */
-EMBED_API const embed_config_t *embed_model_config(const embed_model_t *model);
-
-/*
- * Current mutable workspace memory owned by this workspace, in bytes.
- *
- * This includes the workspace struct, scratch buffers, and RoPE cache capacity.
- * It does not include immutable mmap'd model weights.
- */
-EMBED_API size_t embed_workspace_nbytes(const embed_workspace_t *ws);
-
-/*
- * Compute embedding for a token sequence.
- *
- * 1. Token embedding lookup
- * 2. N transformer layers (GQA + RoPE + SwiGLU)
- * 3. Final RMSNorm
- * 4. Model-defined pooling and optional L2 normalization
- *
- * pplx-embed-v1 uses bidirectional attention and unnormalized mean pooling.
- * Qwen3-Embedding uses causal attention and normalized last-token pooling.
- * Token IDs are consumed exactly as provided; text frontends append any
- * model-required terminal token before calling this API.
- *
- * Returns malloc'd float[hidden_size] (caller frees). NULL on error.
- */
-EMBED_API float *embed_model_encode(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens);
-
-/*
- * Compute one embedding into caller-provided out[hidden_size].
- * Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_model_encode_into(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_embedding);
-
-/*
- * Compute embeddings for a true packed/ragged batch.
- *
- * out_embeddings is caller-provided [batch, hidden_size].
- * Each sequence attends only to tokens from the same input, and RoPE
- * positions restart at zero for every input.
- *
- * Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_model_encode_batch(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings);
-
-/*
- * Apply the config's pooling mode to packed final hidden states.
- *
- * states is [sum(n_tokens), hidden_size]. seq_lengths contains one positive
- * token count per sequence. The states must already include final RMSNorm.
- */
-EMBED_API int embed_pool_batch(const embed_config_t *cfg,
- const float *states,
- const int *seq_lengths,
- int batch,
- float *out_embeddings);
-
-/*
- * Run the transformer forward pass WITHOUT pooling.
- * Returns the full [n_tokens * hidden_size] output after final RMSNorm.
- * Caller frees the returned buffer. NULL on error.
- *
- * Useful for contextual (late-chunking) models where you need
- * per-token embeddings before splitting by separator positions.
- */
-EMBED_API float *embed_model_forward(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens);
-
-/*
- * Run the transformer forward pass into caller-provided
- * out_states[n_tokens * hidden_size]. Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_model_forward_into(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_states);
-
-/*
- * Pool embeddings from final hidden states.
- *
- * states is [n_tokens, hidden_size], normally produced by
- * embed_model_forward_into(). Each span is mean pooled into
- * out_embeddings[n_spans, hidden_size] without L2 normalization.
- */
-EMBED_API int embed_pool_spans(const embed_config_t *cfg,
- const float *states,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings);
-
-/*
- * Run one contextual sequence and pool selected token spans.
- * Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_model_encode_spans(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings);
-
-/*
- * Run a packed/ragged contextual document batch and pool every selected span.
- *
- * Documents attend independently, RoPE positions restart for each document,
- * and out_embeddings contains spans in document order.
- */
-EMBED_API int embed_model_encode_spans_batch(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings);
-
-/*
- * L2-normalize vec[dim] in place.
- *
- * Returns 0 on success. Returns -1 for invalid arguments or a zero-length
- * vector, which cannot be normalized.
- */
-EMBED_API int embed_l2_normalize(float *vec, int dim);
-
-/*
- * Cosine similarity between two vectors.
- * Returns 0 for invalid arguments or if either vector has zero length.
- */
-EMBED_API float embed_cosine_similarity(const float *a, const float *b, int dim);
-
-/*
- * Late-interaction model API.
- *
- * A late model produces one 128D vector per retained token. The vectors are
- * projected by the snapshot's 1_Dense head and L2-normalized per token by
- * default, matching ColBERT/MaxSim scoring semantics.
- */
-EMBED_API embed_late_model_t *embed_late_model_load(const char *model_dir);
-EMBED_API void embed_late_model_free(embed_late_model_t *model);
-
-EMBED_API embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model);
-EMBED_API void embed_late_workspace_free(embed_late_workspace_t *ws);
-
-EMBED_API const embed_config_t *embed_late_model_config(const embed_late_model_t *model);
-EMBED_API int embed_late_model_token_dim(const embed_late_model_t *model);
-
-/*
- * Encode token_ids into out_vectors[n_tokens, token_dim].
- *
- * If normalize is non-zero, every non-zero token vector is L2-normalized.
- */
-EMBED_API int embed_late_model_encode_tokens(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors);
-
-/*
- * Encode n_docs documents in one block-diagonal forward and pack each
- * document's kept token vectors back-to-back into out_vectors[total_keep,
- * token_dim] in document order, where total_keep is the sum of n_keep.
- * out_offsets[n_docs + 1] receives the prefix sum of kept counts, the layout
- * embed_late_maxsim_batch consumes. keep[d] lists the n_keep[d] token indices to
- * retain from document d. Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_late_model_encode_docs(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets);
-
-/*
- * Sum over query tokens of max dot product against document tokens.
- */
-EMBED_API float embed_late_maxsim(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- int doc_tokens,
- int dim);
-
-/*
- * Score one query against a packed/ragged batch of documents.
- *
- * doc_offsets has docs + 1 entries. Document i is stored in
- * doc_vectors[doc_offsets[i] * dim .. doc_offsets[i + 1] * dim).
- * scores is caller-provided [docs].
- */
-EMBED_API int embed_late_maxsim_batch(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- const int *doc_offsets,
- int docs,
- int dim,
- float *scores);
-
-/* Verbose level: 0=quiet, 1=info, 2=debug */
-EMBED_API extern int embed_verbose;
-
-#endif /* EMBED_H */
diff --git a/src/forward.c b/src/forward.c
deleted file mode 100644
index 28243af..0000000
--- a/src/forward.c
+++ /dev/null
@@ -1,710 +0,0 @@
-#include "embed.h"
-#include "model_internal.h"
-#include "dtype.h"
-#include "kernels.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#include <limits.h>
-
-/* Below these packed-sequence lengths the fused BF16 row kernels beat widening
- * the weight to F32 and dispatching through BLAS; above them, widen once. */
-#define EMBED_BF16_QKV_FUSE_MAX_SEQ 16
-#define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4
-
-/* ========================================================================
- * Linear projections (weight-dtype dispatch) and workspace scratch growth
- * ======================================================================== */
-
-static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) {
- for (int i = 0; i < n; i++)
- dst[i] = embed_bf16_to_f32(src[i]);
-}
-
-static void copy_weight_row(float *dst, const embed_weight_ref_t *w, size_t row, int row_width) {
- if (w->dtype == DTYPE_F32) {
- const float *src = (const float *)w->data + row * (size_t)row_width;
- memcpy(dst, src, (size_t)row_width * sizeof(float));
- } else {
- const uint16_t *src = (const uint16_t *)w->data + row * (size_t)row_width;
- bf16_row_to_f32(dst, src, row_width);
- }
-}
-
-static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) {
- if (ws->bf16_widen_count >= count)
- return 0;
- float *p = (float *)realloc(ws->bf16_widen, count * sizeof(float));
- if (!p)
- return -1;
- ws->bf16_widen = p;
- ws->bf16_widen_count = count;
- return 0;
-}
-
-void linear_nobias_weight(embed_workspace_t *ws,
- float *y,
- const float *x,
- const embed_weight_ref_t *w,
- int seq_len,
- int in_dim,
- int out_dim) {
- if (w->dtype == DTYPE_F32) {
- embed_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim);
- return;
- }
- if (seq_len > 16) {
- /* Widen the weight matrix once into workspace scratch and use the
- * BLAS-backed F32 path; below that the fused BF16 row kernels win. */
- size_t count = (size_t)out_dim * (size_t)in_dim;
- if (ensure_bf16_widen(ws, count) == 0) {
- embed_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count);
- embed_linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim);
- return;
- }
- }
- embed_linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim);
-}
-
-static void linear_qkv_weight(embed_workspace_t *ws,
- float *q,
- float *k,
- float *v,
- const float *x,
- const embed_weight_ref_t *wq,
- const embed_weight_ref_t *wk,
- const embed_weight_ref_t *wv,
- int seq_len,
- int in_dim,
- int q_dim,
- int kv_dim) {
- if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && wv->dtype == DTYPE_BF16 &&
- seq_len <= EMBED_BF16_QKV_FUSE_MAX_SEQ) {
- embed_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data,
- (const uint16_t *)wk->data, (const uint16_t *)wv->data,
- seq_len, in_dim, q_dim, kv_dim);
- return;
- }
-
- linear_nobias_weight(ws, q, x, wq, seq_len, in_dim, q_dim);
- linear_nobias_weight(ws, k, x, wk, seq_len, in_dim, kv_dim);
- linear_nobias_weight(ws, v, x, wv, seq_len, in_dim, kv_dim);
-}
-
-static void add_bias_rows(float *x, const float *bias, int rows, int dim) {
- if (!bias)
- return;
- for (int row = 0; row < rows; row++) {
- float *dst = x + (size_t)row * dim;
- for (int d = 0; d < dim; d++)
- dst[d] += bias[d];
- }
-}
-
-static void linear_pair_weight(embed_workspace_t *ws,
- float *a,
- float *b,
- const float *x,
- const embed_weight_ref_t *wa,
- const embed_weight_ref_t *wb,
- int seq_len,
- int in_dim,
- int a_dim,
- int b_dim) {
- if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 &&
- seq_len <= EMBED_BF16_PAIR_FUSE_MAX_SEQ) {
- embed_linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data,
- (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim);
- return;
- }
-
- linear_nobias_weight(ws, a, x, wa, seq_len, in_dim, a_dim);
- linear_nobias_weight(ws, b, x, wb, seq_len, in_dim, b_dim);
-}
-
-/* ========================================================================
- * Packed forward pass
- *
- * The workspace-growth helpers these call (ensure_buffers, ensure_rope_cache,
- * ensure_attention_scores, ensure_offsets) live in model.c beside the workspace
- * lifecycle and embed_workspace_nbytes, whose size accounting mirrors them.
- * ======================================================================== */
-
-int build_offsets(const embed_input_t *inputs,
- int batch,
- int require_ids,
- int *offsets,
- int *total_out,
- int *max_out) {
- if (!inputs || batch <= 0 || !offsets || !total_out || !max_out)
- return -1;
-
- int total = 0;
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- if ((require_ids && !inputs[b].ids) || inputs[b].n_tokens <= 0)
- return -1;
- if (total > INT32_MAX - inputs[b].n_tokens)
- return -1;
- offsets[b] = total;
- total += inputs[b].n_tokens;
- if (inputs[b].n_tokens > max_seq)
- max_seq = inputs[b].n_tokens;
- }
- offsets[batch] = total;
- *total_out = total;
- *max_out = max_seq;
- return 0;
-}
-
-static int forward_packed_slice_inplace(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
- int batch,
- const int *offsets,
- int total_seq,
- int max_seq,
- const float *input_states,
- int layer_start,
- int layer_end,
- int apply_final_norm) {
- if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets ||
- total_seq <= 0 || max_seq <= 0)
- return -1;
-
- const embed_config_t *cfg = &model->config;
- const embed_weights_t *w = &model->weights;
- if (layer_start < 0 || layer_start > layer_end || layer_end > cfg->n_layers ||
- layer_start < model->layer_start || layer_end > model->layer_end ||
- (layer_start == 0 && input_states) || (layer_start != 0 && !input_states) ||
- (layer_start == 0 && !w->embed_tokens.data) ||
- (apply_final_norm && (layer_end != cfg->n_layers || !w->norm)))
- return -1;
-
- int hidden = cfg->hidden_size;
- int n_heads = cfg->n_heads;
- int n_kv_heads = cfg->n_kv_heads;
- int head_dim = cfg->head_dim;
- int q_dim = cfg->q_dim;
- int kv_dim = cfg->kv_dim;
- int inter = cfg->intermediate_size;
- float eps = cfg->rms_norm_eps;
-
- if (ensure_buffers(ws, cfg, total_seq) != 0)
- return -1;
- if (ensure_rope_cache(ws, cfg, max_seq) != 0)
- return -1;
- ensure_attention_scores(ws, offsets, batch);
-
- const float *rope_cos = ws->rope_cos;
- const float *rope_sin = ws->rope_sin;
-
- float *x = ws->x;
- float *x_norm = ws->x_norm;
- float *q_buf = ws->q;
- float *k_buf = ws->k;
- float *v_buf = ws->v;
- float *attn_out = ws->attn_out;
- float *proj_out = ws->proj_out;
- float *ffn_gate = ws->ffn_gate;
- float *ffn_up = ws->ffn_up;
-
- /* A first stage performs embedding lookup. Later stages resume from a
- * packed hidden-state tensor received from the preceding stage. */
- if (layer_start == 0) {
- for (int b = 0; b < batch; b++) {
- int start = offsets[b];
- for (int i = 0; i < inputs[b].n_tokens; i++) {
- int id = inputs[b].ids[i];
- if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b,
- i);
- return -1;
- }
- copy_weight_row(x + (size_t)(start + i) * hidden, &w->embed_tokens, (size_t)id,
- hidden);
- }
- }
- } else {
- memcpy(x, input_states, (size_t)total_seq * hidden * sizeof(float));
- }
-
- float scale = 1.0f / sqrtf((float)head_dim);
-
- /* 2. Transformer layers. Linear/MLP ops run over all packed tokens;
- * attention is explicitly block-diagonal by sequence offsets. */
- for (int layer = layer_start; layer < layer_end; layer++) {
- const embed_layer_t *l = &w->layers[layer];
-
- embed_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps);
-
- linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq,
- hidden, q_dim, kv_dim);
-
- add_bias_rows(q_buf, l->q_bias, total_seq, q_dim);
- add_bias_rows(k_buf, l->k_bias, total_seq, kv_dim);
- add_bias_rows(v_buf, l->v_bias, total_seq, kv_dim);
- if (cfg->qk_norm) {
- embed_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
- embed_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps);
- }
-
- for (int b = 0; b < batch; b++) {
- int start = offsets[b];
- int len = offsets[b + 1] - start;
- embed_apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads,
- head_dim);
- embed_apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len,
- n_kv_heads, head_dim);
- }
-
- model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads,
- head_dim, scale, ws->attn_scores, ws->attn_scores_bytes);
-
- linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, q_dim, hidden);
- embed_add_inplace(x, proj_out, total_seq * hidden);
-
- embed_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps);
-
- linear_pair_weight(ws, ffn_gate, ffn_up, x_norm, &l->gate_proj, &l->up_proj, total_seq,
- hidden, inter, inter);
- embed_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter);
- linear_nobias_weight(ws, proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden);
- embed_add_inplace(x, proj_out, total_seq * hidden);
- }
-
- if (apply_final_norm)
- embed_rms_norm(x, x, w->norm, total_seq, hidden, eps);
- return 0;
-}
-
-int forward_packed_inplace(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
- int batch,
- const int *offsets,
- int total_seq,
- int max_seq,
- int apply_final_norm) {
- return forward_packed_slice_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, NULL,
- 0, model->config.n_layers, apply_final_norm);
-}
-
-/* BERT-family forward: token + learned-position + token-type embeddings and an
- * embedding LayerNorm, then post-norm encoder layers (bias on every projection,
- * GeLU feed-forward, no RoPE, no per-head norm). Bidirectional attention is
- * block-diagonal over the packed batch, so each sequence's positions restart at
- * cfg->position_id_offset (0 for BERT, pad_id + 1 for RoBERTa/XLM-R). Leaves the
- * final hidden states in ws->x; pooling reads them as-is because the last
- * layer's LayerNorm is already the model's final normalization. */
-static int forward_packed_bert(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
- int batch,
- const int *offsets,
- int total_seq,
- int max_seq) {
- if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets ||
- total_seq <= 0 || max_seq <= 0)
- return -1;
-
- const embed_config_t *cfg = &model->config;
- const embed_weights_t *w = &model->weights;
- int hidden = cfg->hidden_size;
- int n_heads = cfg->n_heads;
- int head_dim = cfg->head_dim;
- int inter = cfg->intermediate_size;
- float eps = cfg->layer_norm_eps;
-
- if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings ||
- !w->embed_ln_w || !w->embed_ln_b)
- return -1;
- if (max_seq > cfg->max_position_embeddings - cfg->position_id_offset) {
- fprintf(stderr,
- "embed: sequence length %d plus position offset %d exceeds max_position_embeddings %d\n",
- max_seq, cfg->position_id_offset, cfg->max_position_embeddings);
- return -1;
- }
- if (ensure_buffers(ws, cfg, total_seq) != 0)
- return -1;
- ensure_attention_scores(ws, offsets, batch);
-
- float *x = ws->x;
- float *q_buf = ws->q, *k_buf = ws->k, *v_buf = ws->v;
- float *attn_out = ws->attn_out, *proj_out = ws->proj_out, *ffn_up = ws->ffn_up;
-
- /* Embeddings: token + learned absolute position + token-type[0], then the
- * embedding LayerNorm over every packed token. */
- for (int b = 0; b < batch; b++) {
- int start = offsets[b];
- for (int i = 0; i < inputs[b].n_tokens; i++) {
- int id = inputs[b].ids[i];
- if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b, i);
- return -1;
- }
- float *row = x + (size_t)(start + i) * hidden;
- copy_weight_row(row, &w->embed_tokens, (size_t)id, hidden);
- int pos = cfg->position_id_offset + i;
- embed_add_inplace(row, w->position_embeddings + (size_t)pos * hidden, hidden);
- embed_add_inplace(row, w->token_type_embeddings, hidden);
- }
- }
- embed_layer_norm(x, x, w->embed_ln_w, w->embed_ln_b, total_seq, hidden, eps);
-
- float scale = 1.0f / sqrtf((float)head_dim);
-
- for (int layer = 0; layer < cfg->n_layers; layer++) {
- const embed_layer_t *l = &w->layers[layer];
-
- /* Self-attention (post-norm): projections read x directly. */
- linear_qkv_weight(ws, q_buf, k_buf, v_buf, x, &l->wq, &l->wk, &l->wv, total_seq, hidden,
- hidden, hidden);
- add_bias_rows(q_buf, l->q_bias, total_seq, hidden);
- add_bias_rows(k_buf, l->k_bias, total_seq, hidden);
- add_bias_rows(v_buf, l->v_bias, total_seq, hidden);
-
- model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_heads, head_dim,
- scale, ws->attn_scores, ws->attn_scores_bytes);
-
- linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, hidden, hidden);
- add_bias_rows(proj_out, l->o_bias, total_seq, hidden);
- embed_add_inplace(x, proj_out, total_seq * hidden);
- embed_layer_norm(x, x, l->input_norm, l->attn_ln_bias, total_seq, hidden, eps);
-
- /* Feed-forward (post-norm): dense -> GeLU -> dense. */
- linear_nobias_weight(ws, ffn_up, x, &l->up_proj, total_seq, hidden, inter);
- add_bias_rows(ffn_up, l->ffn_inter_bias, total_seq, inter);
- if (cfg->ffn_act == EMBED_ACT_GELU_TANH)
- embed_gelu_tanh_inplace(ffn_up, total_seq * inter);
- else
- embed_gelu_inplace(ffn_up, total_seq * inter);
- linear_nobias_weight(ws, proj_out, ffn_up, &l->down_proj, total_seq, inter, hidden);
- add_bias_rows(proj_out, l->ffn_out_bias, total_seq, hidden);
- embed_add_inplace(x, proj_out, total_seq * hidden);
- embed_layer_norm(x, x, l->post_attn_norm, l->ffn_ln_bias, total_seq, hidden, eps);
- }
- return 0;
-}
-
-static int pool_embeddings(const embed_model_t *model,
- const embed_workspace_t *ws,
- const int *offsets,
- int batch,
- float *out_embeddings) {
- const embed_config_t *cfg = &model->config;
- int hidden = cfg->hidden_size;
- float eps = cfg->rms_norm_eps;
- /* Qwen pools pre-final-norm states and applies the final RMSNorm here; BERT
- * pools already-normed states (no final norm), signalled by a NULL norm. */
- const float *norm_weight = model->weights.norm;
- const float *x = ws->x;
- if (hidden <= 0 || !x)
- return -1;
-
- for (int b = 0; b < batch; b++) {
- int start = offsets[b];
- int end = offsets[b + 1];
- int len = end - start;
- float *emb = out_embeddings + (size_t)b * hidden;
-
- if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) {
- /* Single-token pooling: CLS takes the first token, last-token the
- * final one. */
- int idx = cfg->pooling_mode == EMBED_POOL_CLS ? start : end - 1;
- const float *row = x + (size_t)idx * hidden;
- if (norm_weight) {
- float sum_sq = 0.0f;
- for (int d = 0; d < hidden; d++)
- sum_sq += row[d] * row[d];
- float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps);
- for (int d = 0; d < hidden; d++)
- emb[d] = row[d] * rms_inv * norm_weight[d];
- } else {
- memcpy(emb, row, (size_t)hidden * sizeof(float));
- }
- } else {
- memset(emb, 0, (size_t)hidden * sizeof(float));
- for (int i = start; i < end; i++) {
- const float *row = x + (size_t)i * hidden;
- if (norm_weight) {
- float sum_sq = 0.0f;
- for (int d = 0; d < hidden; d++)
- sum_sq += row[d] * row[d];
- float rms_inv = 1.0f / sqrtf(sum_sq / (float)hidden + eps);
- for (int d = 0; d < hidden; d++)
- emb[d] += row[d] * rms_inv * norm_weight[d];
- } else {
- for (int d = 0; d < hidden; d++)
- emb[d] += row[d];
- }
- }
- float inv_len = 1.0f / (float)len;
- for (int d = 0; d < hidden; d++)
- emb[d] *= inv_len;
- }
-
- if (cfg->normalize_embeddings && embed_l2_normalize(emb, hidden) != 0)
- return -1;
- }
- return 0;
-}
-
-/* ========================================================================
- * Public forward/embed APIs
- * ======================================================================== */
-
-int embed_model_forward_into(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_states) {
- if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states)
- return -1;
-
- size_t count, bytes;
- if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &count) != 0 ||
- mul_size(count, sizeof(float), &bytes) != 0)
- return -1;
-
- embed_input_t input = {token_ids, n_tokens};
- int offsets[2] = {0, n_tokens};
- if (forward_packed_inplace(model, ws, &input, 1, offsets, n_tokens, n_tokens, 1) != 0)
- return -1;
-
- memcpy(out_states, ws->x, bytes);
- return 0;
-}
-
-int embed_pool_spans(const embed_config_t *cfg,
- const float *states,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings) {
- if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || !out_embeddings)
- return -1;
-
- int hidden = cfg->hidden_size;
- if (hidden <= 0)
- return -1;
-
- for (int s = 0; s < n_spans; s++) {
- int start = spans[s].start;
- int len = spans[s].n_tokens;
- if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start)
- return -1;
-
- float *emb = out_embeddings + (size_t)s * hidden;
- memset(emb, 0, (size_t)hidden * sizeof(float));
-
- for (int t = 0; t < len; t++) {
- const float *row = states + (size_t)(start + t) * hidden;
- for (int d = 0; d < hidden; d++)
- emb[d] += row[d];
- }
-
- float inv_len = 1.0f / (float)len;
- for (int d = 0; d < hidden; d++) {
- emb[d] *= inv_len;
- }
- }
- return 0;
-}
-
-int embed_model_encode_spans_batch(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!model || !ws || !inputs || batch <= 0 || !out_embeddings)
- return -1;
-
- if (ensure_offsets(ws, batch) != 0)
- return -1;
- if ((size_t)batch > SIZE_MAX / sizeof(embed_input_t))
- return -1;
- embed_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs));
- if (!packed_inputs)
- return -1;
- int *offsets = ws->offsets;
- int total_seq = 0;
- int max_seq = 0;
- int total_spans = 0;
- for (int b = 0; b < batch; b++) {
- const embed_context_input_t *input = &inputs[b];
- int n_tokens = input->input.n_tokens;
- if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
- total_seq > INT_MAX - n_tokens || total_spans > INT_MAX - input->n_spans) {
- free(packed_inputs);
- return -1;
- }
- packed_inputs[b] = input->input;
- offsets[b] = total_seq;
- total_seq += n_tokens;
- total_spans += input->n_spans;
- if (n_tokens > max_seq)
- max_seq = n_tokens;
- for (int s = 0; s < input->n_spans; s++) {
- int start = input->spans[s].start;
- int len = input->spans[s].n_tokens;
- if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) {
- free(packed_inputs);
- return -1;
- }
- }
- }
- offsets[batch] = total_seq;
-
- if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets, total_seq, max_seq, 1) !=
- 0) {
- free(packed_inputs);
- return -1;
- }
- free(packed_inputs);
-
- int span_offset = 0;
- int hidden = model->config.hidden_size;
- for (int b = 0; b < batch; b++) {
- const embed_context_input_t *input = &inputs[b];
- if (embed_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden,
- input->input.n_tokens, input->spans, input->n_spans,
- out_embeddings + (size_t)span_offset * hidden) != 0)
- return -1;
- span_offset += input->n_spans;
- }
- return 0;
-}
-
-int embed_model_encode_spans(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings) {
- embed_context_input_t input = {
- .input = {token_ids, n_tokens},
- .spans = spans,
- .n_spans = n_spans,
- };
- return embed_model_encode_spans_batch(model, ws, &input, 1, out_embeddings);
-}
-
-float *embed_model_forward(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens) {
- if (!model || !ws || !token_ids || n_tokens <= 0)
- return NULL;
-
- size_t n;
- if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &n) != 0)
- return NULL;
- float *out = malloc_floats(n);
- if (!out)
- return NULL;
-
- if (embed_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) {
- free(out);
- return NULL;
- }
- return out;
-}
-
-int embed_model_encode_batch(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!model || !ws || !inputs || batch <= 0 || !out_embeddings)
- return -1;
-
- if (ensure_offsets(ws, batch) != 0)
- return -1;
- int *offsets = ws->offsets;
-
- int total_seq = 0;
- int max_seq = 0;
- if (build_offsets(inputs, batch, 1, offsets, &total_seq, &max_seq) != 0)
- return -1;
-
- int rc = model->config.family == EMBED_FAMILY_BERT
- ? forward_packed_bert(model, ws, inputs, batch, offsets, total_seq, max_seq)
- : forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0);
- if (rc == 0)
- rc = pool_embeddings(model, ws, offsets, batch, out_embeddings);
-
- return rc;
-}
-
-int embed_pool_batch(const embed_config_t *cfg,
- const float *states,
- const int *seq_lengths,
- int batch,
- float *out_embeddings) {
- if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || cfg->hidden_size <= 0)
- return -1;
-
- int offset = 0;
- for (int b = 0; b < batch; b++) {
- int len = seq_lengths[b];
- if (len <= 0 || offset > INT_MAX - len)
- return -1;
-
- float *emb = out_embeddings + (size_t)b * cfg->hidden_size;
- if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN || cfg->pooling_mode == EMBED_POOL_CLS) {
- /* CLS takes the first token, last-token the final one; these states
- * already carry the model's final RMSNorm. */
- int idx = cfg->pooling_mode == EMBED_POOL_CLS ? offset : offset + len - 1;
- const float *row = states + (size_t)idx * cfg->hidden_size;
- memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float));
- } else {
- memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float));
- for (int t = 0; t < len; t++) {
- const float *row = states + (size_t)(offset + t) * cfg->hidden_size;
- for (int d = 0; d < cfg->hidden_size; d++)
- emb[d] += row[d];
- }
- float inv_len = 1.0f / (float)len;
- for (int d = 0; d < cfg->hidden_size; d++)
- emb[d] *= inv_len;
- }
- if (cfg->normalize_embeddings && embed_l2_normalize(emb, cfg->hidden_size) != 0)
- return -1;
- offset += len;
- }
- return 0;
-}
-
-int embed_model_encode_into(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- float *out_embedding) {
- embed_input_t input = {token_ids, n_tokens};
- return embed_model_encode_batch(model, ws, &input, 1, out_embedding);
-}
-
-float *embed_model_encode(const embed_model_t *model,
- embed_workspace_t *ws,
- const int *token_ids,
- int n_tokens) {
- if (!model || !ws || !token_ids || n_tokens <= 0)
- return NULL;
-
- int hidden = model->config.hidden_size;
- if (hidden <= 0)
- return NULL;
- float *emb = malloc_floats((size_t)hidden);
- if (!emb)
- return NULL;
-
- if (embed_model_encode_into(model, ws, token_ids, n_tokens, emb) != 0) {
- free(emb);
- return NULL;
- }
- return emb;
-}
diff --git a/src/internal.h b/src/internal.h
deleted file mode 100644
index c1429de..0000000
--- a/src/internal.h
+++ /dev/null
@@ -1,114 +0,0 @@
-#ifndef EMBED_INTERNAL_H
-#define EMBED_INTERNAL_H
-
-#include "embed.h"
-#include "safetensors.h"
-
-typedef struct {
- safetensor_dtype_t dtype;
- const void *data;
- /* 0: data is a borrowed view into the safetensors mmap (F32/BF16, the common
- * case). 1: data is a heap buffer owned by the model and freed on unload.
- * Set when an F16 tensor is widened to F32 at load (no fused F16 kernel). */
- int owned;
-} embed_weight_ref_t;
-
-typedef struct {
- embed_weight_ref_t wq; /* [q_dim, hidden] */
- embed_weight_ref_t wk; /* [kv_dim, hidden] */
- embed_weight_ref_t wv; /* [kv_dim, hidden] */
- embed_weight_ref_t wo; /* [hidden, q_dim] */
- const float *q_bias; /* [q_dim], optional */
- const float *k_bias; /* [kv_dim], optional */
- const float *v_bias; /* [kv_dim], optional */
- const float *q_norm; /* [head_dim] */
- const float *k_norm; /* [head_dim] */
- const float *input_norm; /* [hidden] */
- const float *post_attn_norm; /* [hidden] */
- embed_weight_ref_t gate_proj; /* [intermediate, hidden] */
- embed_weight_ref_t up_proj; /* [intermediate, hidden] */
- embed_weight_ref_t down_proj; /* [hidden, intermediate] */
- /* BERT family (all NULL for Qwen3). The attention projections reuse
- * wq/wk/wv/wo and q/k/v_bias; the two FFN matrices reuse up_proj
- * (intermediate.dense) and down_proj (output.dense); the two block
- * LayerNorm weights reuse input_norm (post-attention) and post_attn_norm
- * (post-FFN). These fields add the biases those reused slots lack. */
- const float *o_bias; /* attention output dense bias [hidden] */
- const float *attn_ln_bias; /* post-attention LayerNorm bias [hidden] */
- const float *ffn_inter_bias; /* intermediate.dense bias [intermediate] */
- const float *ffn_out_bias; /* output.dense bias [hidden] */
- const float *ffn_ln_bias; /* post-FFN LayerNorm bias [hidden] */
-} embed_layer_t;
-
-typedef struct {
- embed_weight_ref_t embed_tokens; /* [vocab_size, hidden] */
- embed_layer_t *layers; /* [n_layers] heap-allocated */
- const float *norm; /* [hidden]; NULL for BERT (no final norm) */
- /* BERT family (NULL for Qwen3): learned absolute position embeddings,
- * token-type (segment) embeddings, and the embedding LayerNorm. */
- const float *position_embeddings; /* [max_position_embeddings, hidden] */
- const float *token_type_embeddings; /* [type_vocab_size, hidden] */
- const float *embed_ln_w; /* embedding LayerNorm weight [hidden] */
- const float *embed_ln_b; /* embedding LayerNorm bias [hidden] */
-} embed_weights_t;
-
-typedef void (*embed_attention_fn)(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes);
-
-struct embed_model {
- embed_config_t config;
- embed_weights_t weights;
- void *safetensors; /* multi_safetensors_t*, keeps mmap alive */
- int layer_start; /* loaded transformer range, inclusive */
- int layer_end; /* loaded transformer range, exclusive */
- embed_attention_fn attention;
-};
-
-struct embed_workspace {
- const embed_model_t *model; /* immutable model this workspace belongs to */
- int buf_seq_cap;
- float *x; /* [seq, hidden] */
- float *x_norm; /* [seq, hidden] */
- float *q; /* [seq, q_dim] */
- float *k; /* [seq, kv_dim] */
- float *v; /* [seq, kv_dim] */
- float *attn_out; /* [seq, q_dim] */
- float *proj_out; /* [seq, hidden] */
- float *ffn_gate; /* [seq, intermediate] */
- float *ffn_up; /* [seq, intermediate] */
- float *attn_scores; /* reusable BLAS attention score tiles */
- size_t attn_scores_bytes;
-
- int *offsets; /* [batch + 1] packed sequence offsets */
- int offsets_cap;
-
- /* RoPE cosine/sine cache [n_pos, head_dim], grown lazily. */
- float *rope_cos;
- float *rope_sin;
- int rope_cache_cap;
-
- /* F32 view of one BF16 weight matrix for SGEMM dispatch, grown to the
- * largest projection. Workspace-owned so concurrent workspaces on one
- * model never share conversion scratch. */
- float *bf16_widen;
- size_t bf16_widen_count;
-};
-
-/* Internal accessors for the late-interaction model. GPU backends build a device
- * context from an already-loaded CPU late model because the model dir is a late
- * snapshot that the pooled loader rejects; these expose the base model and the
- * projection weight without making struct embed_late_model public. */
-embed_model_t *embed_late_model_base(const embed_late_model_t *model);
-const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model);
-
-#endif /* EMBED_INTERNAL_H */
diff --git a/src/kernels.h b/src/kernels.h
deleted file mode 100644
index 17c45c0..0000000
--- a/src/kernels.h
+++ /dev/null
@@ -1,216 +0,0 @@
-/*
- * embed_kernels.h - Math kernels for Qwen3-architecture inference
- *
- * Low-level math operations. All operate on float32 tensors in row-major order.
- */
-
-#ifndef EMBED_KERNELS_H
-#define EMBED_KERNELS_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-#include <stddef.h>
-#include <stdint.h>
-
-/* ========================================================================
- * Basic Operations
- * ======================================================================== */
-
-void embed_add_inplace(float *a, const float *b, int n);
-
-/* ========================================================================
- * Matrix Operations
- * ======================================================================== */
-
-/* C = A @ B^T: A[M,K], B[N,K], C[M,N] */
-void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int N);
-
-/* y = x @ W^T + b: x[seq,in], W[out,in], b[out], y[seq,out] */
-void embed_linear(
- float *y, const float *x, const float *W, const float *b, int seq_len, int in_dim, int out_dim);
-
-void embed_linear_nobias(
- float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim);
-
-/* bf16 weight variants */
-
-void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n);
-void embed_linear_nobias_bf16(
- float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim);
-
-/* Small-sequence BF16 path: compute two projections with one threaded dispatch. */
-void embed_linear_nobias_bf16_pair(float *a,
- float *b,
- const float *x,
- const uint16_t *Wa_bf16,
- const uint16_t *Wb_bf16,
- int seq_len,
- int in_dim,
- int a_dim,
- int b_dim);
-
-/* Small-sequence BF16 path: compute Q/K/V matvecs with one threaded dispatch. */
-void embed_linear_nobias_bf16_qkv(float *q,
- float *k,
- float *v,
- const float *x,
- const uint16_t *Wq_bf16,
- const uint16_t *Wk_bf16,
- const uint16_t *Wv_bf16,
- int seq_len,
- int in_dim,
- int q_dim,
- int kv_dim);
-
-/* Dot product using the best available local SIMD implementation. */
-float embed_dot_f32(const float *a, const float *b, int n);
-
-/* ========================================================================
- * Normalization
- * ======================================================================== */
-
-/* RMS Normalization: out = x / rms(x) * weight */
-void embed_rms_norm(
- float *out, const float *x, const float *weight, int seq_len, int hidden, float eps);
-
-/* Per-head RMS Normalization for Q/K norms in decoder
- * x: [seq, n_heads, head_dim], weight: [head_dim]
- * Normalizes each head independently */
-void embed_rms_norm_per_head(
- float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps);
-
-/* Layer Normalization (BERT family): out = gamma * (x - mean) / sqrt(var + eps)
- * + beta, with mean and biased (population) variance over the hidden axis per
- * row. Unlike RMSNorm this subtracts the mean and adds a bias. */
-void embed_layer_norm(float *out,
- const float *x,
- const float *gamma,
- const float *beta,
- int seq_len,
- int hidden,
- float eps);
-
-/* ========================================================================
- * Activation Functions
- * ======================================================================== */
-
-void embed_softmax(float *x, int rows, int cols);
-/* gate = SiLU(gate) * up */
-void embed_silu_mul_inplace(float *gate, const float *up, int n);
-
-/* Exact (erf) GeLU in place: x = 0.5 * x * (1 + erf(x / sqrt(2))). This is the
- * "gelu" activation HF BERT/BGE/MiniLM use, not the tanh approximation. */
-void embed_gelu_inplace(float *x, int n);
-
-/* Tanh-approximation GeLU in place (HF gelu_new / gelu_pytorch_tanh):
- * x = 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). */
-void embed_gelu_tanh_inplace(float *x, int n);
-
-/* ========================================================================
- * Attention Operations
- * ======================================================================== */
-
-/*
- * Bidirectional GQA attention over a packed/ragged batch.
- *
- * Q: [total_seq, n_heads * head_dim]
- * K,V: [total_seq, n_kv_heads * head_dim]
- * offsets: [batch + 1], token spans for each sequence.
- *
- * Attention is block-diagonal: tokens only attend within their own
- * offsets[b]..offsets[b+1] span.
- */
-void embed_bidirectional_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale);
-
-/* Optional reusable scratch for the BLAS-tiled packed attention path. */
-size_t embed_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch);
-void embed_bidirectional_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes);
-
-/*
- * Causal GQA attention over a packed/ragged batch. Query position i attends
- * only to positions at or before i within its sequence.
- */
-void embed_causal_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale);
-void embed_causal_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes);
-
-/* ========================================================================
- * Position Embeddings
- * ======================================================================== */
-
-/*
- * NeoX-style RoPE: compute cos/sin for positions.
- * cos_out, sin_out: [seq, head_dim]
- * cos[d] and cos[half+d] are the same (duplicated for full head_dim).
- */
-void embed_compute_rope_neox(
- float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta);
-
-/*
- * Apply NeoX-style RoPE to Q or K.
- * x: [seq, n_heads * head_dim] (in-place)
- * cos_vals, sin_vals: [seq, head_dim]
- */
-void embed_apply_rope_neox(
- float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim);
-
-/* ========================================================================
- * Threading
- * ======================================================================== */
-
-/* The persistent worker pool lives in threadpool.{c,h}; included here so kernel
- * callers keep getting embed_set_threads / embed_get_num_cpus from one header. */
-#include "threadpool.h"
-
-/* Global verbose flag */
-EMBED_API extern int embed_verbose;
-
-#endif /* EMBED_KERNELS_H */
diff --git a/src/kernels_act.c b/src/kernels_act.c
deleted file mode 100644
index abbade4..0000000
--- a/src/kernels_act.c
+++ /dev/null
@@ -1,95 +0,0 @@
-#include "kernels.h"
-#include "kernels_impl.h"
-
-#include <math.h>
-#include <stdlib.h>
-#include <string.h>
-
-#if (defined(__AVX512F__) || defined(__AVX2__)) && \
- (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
-#include <immintrin.h>
-#endif
-#if defined(__ARM_NEON) || defined(__ARM_NEON__)
-#include <arm_neon.h>
-#endif
-
-#ifdef USE_BLAS
-#ifdef __APPLE__
-#include <Accelerate/Accelerate.h>
-#else
-#include <cblas.h>
-#endif
-#endif
-
-#ifndef M_PI
-#define M_PI 3.14159265358979323846
-#endif
-
-/* Activation kernels: softmax, SiLU-gated multiply, and the erf and tanh GeLU
- * variants. */
-
-/* ========================================================================
- * Activation Functions
- * ======================================================================== */
-
-void embed_silu_mul_inplace(float *gate, const float *up, int n) {
-#if defined(__APPLE__) && defined(USE_BLAS)
- enum { CHUNK = 4096 };
- float exp_neg[CHUNK];
-
- for (int base = 0; base < n; base += CHUNK) {
- int len = n - base;
- if (len > CHUNK)
- len = CHUNK;
-
- for (int i = 0; i < len; i++)
- exp_neg[i] = -gate[base + i];
- vvexpf(exp_neg, exp_neg, &len);
-
- for (int i = 0; i < len; i++) {
- float g = gate[base + i];
- gate[base + i] = (g / (1.0f + exp_neg[i])) * up[base + i];
- }
- }
-#else
- for (int i = 0; i < n; i++) {
- float g = gate[i];
- gate[i] = (g / (1.0f + expf(-g))) * up[i];
- }
-#endif
-}
-
-void embed_gelu_inplace(float *x, int n) {
- /* Exact erf GeLU; 0.70710678... = 1/sqrt(2). */
- for (int i = 0; i < n; i++)
- x[i] = 0.5f * x[i] * (1.0f + erff(x[i] * 0.70710678118654752f));
-}
-
-void embed_gelu_tanh_inplace(float *x, int n) {
- /* Tanh-approximation GeLU; 0.79788456... = sqrt(2/pi). */
- for (int i = 0; i < n; i++) {
- float v = x[i];
- float inner = 0.79788456080286536f * (v + 0.044715f * v * v * v);
- x[i] = 0.5f * v * (1.0f + tanhf(inner));
- }
-}
-
-void embed_softmax(float *x, int rows, int cols) {
- for (int r = 0; r < rows; r++) {
- float *row = x + r * cols;
- float max_val = row[0];
- for (int c = 1; c < cols; c++) {
- if (row[c] > max_val)
- max_val = row[c];
- }
- float sum = 0.0f;
- for (int c = 0; c < cols; c++) {
- row[c] = expf(row[c] - max_val);
- sum += row[c];
- }
- float inv_sum = 1.0f / sum;
- for (int c = 0; c < cols; c++) {
- row[c] *= inv_sum;
- }
- }
-}
diff --git a/src/kernels_attn.c b/src/kernels_attn.c
deleted file mode 100644
index 66f5ca5..0000000
--- a/src/kernels_attn.c
+++ /dev/null
@@ -1,469 +0,0 @@
-#include "kernels.h"
-#include "kernels_impl.h"
-
-#include <math.h>
-#include <stdlib.h>
-#include <string.h>
-
-#if (defined(__AVX512F__) || defined(__AVX2__)) && \
- (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
-#include <immintrin.h>
-#endif
-#if defined(__ARM_NEON) || defined(__ARM_NEON__)
-#include <arm_neon.h>
-#endif
-
-#ifdef USE_BLAS
-#ifdef __APPLE__
-#include <Accelerate/Accelerate.h>
-#else
-#include <cblas.h>
-#endif
-#endif
-
-#ifndef M_PI
-#define M_PI 3.14159265358979323846
-#endif
-
-/* Packed block-diagonal attention: causal (Qwen) and bidirectional (BERT/GTE)
- * GQA, each with an online-softmax and a BLAS-GEMM path. */
-
-/* ========================================================================
- * Attention Operations
- * ======================================================================== */
-
-float embed_dot_f32(const float *a, const float *b, int n) { return embed_dot_f32_impl(a, b, n); }
-
-static inline float embed_dot_f32_fast(const float *a, const float *b, int n) {
- return embed_dot_f32_impl(a, b, n);
-}
-
-/* dst = dst * scale */
-static inline void embed_vec_scale_inplace(float *dst, float scale, int n) {
- embed_vec_scale_inplace_impl(dst, scale, n);
-}
-
-/* dst += alpha * src */
-static inline void embed_vec_axpy_inplace(float *dst, const float *src, float alpha, int n) {
- embed_vec_axpy_inplace_impl(dst, src, alpha, n);
-}
-
-/* dst = dst * correction + src */
-static inline void embed_vec_scale_add(float *dst, const float *src, float correction, int n) {
- embed_vec_scale_add_impl(dst, src, correction, n);
-}
-
-#define EMBED_PACKED_ATTN_QUERY_TILE 32
-#define EMBED_PACKED_ATTN_BLAS_MIN_SEQ 128
-
-static void embed_bidirectional_gqa_attention_packed_online_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int end,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
-
- for (int i = query_start; i < query_end; i++) {
- const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
- float *o_row = out + (size_t)i * q_hidden + h * head_dim;
-
- float max_score = -1e30f;
- float sum_exp = 0.0f;
- for (int d = 0; d < head_dim; d++)
- o_row[d] = 0.0f;
-
- for (int j = start; j < end; j++) {
- const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
- const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
-
- float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale;
-
- if (score > max_score) {
- float correction = expf(max_score - score);
- sum_exp = sum_exp * correction + 1.0f;
- embed_vec_scale_add(o_row, v_row, correction, head_dim);
- max_score = score;
- } else {
- float wt = expf(score - max_score);
- sum_exp += wt;
- embed_vec_axpy_inplace(o_row, v_row, wt, head_dim);
- }
- }
-
- if (sum_exp > 0.0f) {
- float inv_sum = 1.0f / sum_exp;
- embed_vec_scale_inplace(o_row, inv_sum, head_dim);
- }
- }
-}
-
-#ifdef USE_BLAS
-static void embed_bidirectional_gqa_attention_packed_blas_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int end,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end,
- float *scores) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
- int rows = query_end - query_start;
- int keys = end - start;
-
- const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
- const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
- const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
- float *o = out + (size_t)query_start * q_hidden + h * head_dim;
-
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden,
- k, kv_hidden, 0.0f, scores, keys);
- embed_softmax(scores, rows, keys);
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
- v, kv_hidden, 0.0f, o, q_hidden);
-}
-#endif
-
-typedef struct {
- float *out;
- const float *Q;
- const float *K;
- const float *V;
- const int *offsets;
- int batch;
- int n_heads, n_kv_heads;
- int head_dim;
- float scale;
- float *scores;
- int max_seq;
-} packed_gqa_attn_task_t;
-
-static void packed_gqa_attn_worker(int tid, int n_threads, void *arg) {
- packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
- int task_id = 0;
-#ifdef USE_BLAS
- float *scores =
- t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
-#endif
-
- for (int h = 0; h < t->n_heads; h++) {
- for (int b = 0; b < t->batch; b++) {
- int start = t->offsets[b];
- int end = t->offsets[b + 1];
- for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) {
- int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE;
- if (q1 > end)
- q1 = end;
- if (task_id++ % n_threads != tid)
- continue;
-#ifdef USE_BLAS
- if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) {
- embed_bidirectional_gqa_attention_packed_blas_rows(
- t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads,
- t->head_dim, t->scale, h, q0, q1, scores);
- continue;
- }
-#endif
- embed_bidirectional_gqa_attention_packed_online_rows(
- t->out, t->Q, t->K, t->V, start, end, t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1);
- }
- }
- }
-}
-
-size_t embed_bidirectional_gqa_attention_packed_scratch_bytes(const int *offsets, int batch) {
-#ifdef USE_BLAS
- if (!offsets || batch <= 0)
- return 0;
-
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- if (len > max_seq)
- max_seq = len;
- }
- if (max_seq < EMBED_PACKED_ATTN_BLAS_MIN_SEQ)
- return 0;
- if ((size_t)max_seq >
- SIZE_MAX / (sizeof(float) * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)embed_num_threads()))
- return 0;
- return (size_t)embed_num_threads() * EMBED_PACKED_ATTN_QUERY_TILE * (size_t)max_seq * sizeof(float);
-#else
- (void)offsets;
- (void)batch;
- return 0;
-#endif
-}
-
-void embed_bidirectional_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes) {
- long long qk_work = 0;
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- qk_work += (long long)len * len * n_heads;
- if (len > max_seq)
- max_seq = len;
- }
-
- float *scores = NULL;
-#ifdef USE_BLAS
- size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- if (required != 0 && scratch && scratch_bytes >= required)
- scores = scratch;
-#else
- (void)scratch;
- (void)scratch_bytes;
-#endif
- packed_gqa_attn_task_t task = {.out = out,
- .Q = Q,
- .K = K,
- .V = V,
- .offsets = offsets,
- .batch = batch,
- .n_heads = n_heads,
- .n_kv_heads = n_kv_heads,
- .head_dim = head_dim,
- .scale = scale,
- .scores = scores,
- .max_seq = max_seq};
- if (embed_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096) {
- parallel_for(packed_gqa_attn_worker, &task);
- } else {
- packed_gqa_attn_worker(0, 1, &task);
- }
-}
-
-void embed_bidirectional_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale) {
- size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
- embed_bidirectional_gqa_attention_packed_with_scratch(
- out, Q, K, V, offsets, batch, n_heads, n_kv_heads, head_dim, scale, scores, scratch_bytes);
- free(scores);
-}
-
-static void embed_causal_gqa_attention_packed_online_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
-
- for (int i = query_start; i < query_end; i++) {
- const float *q_row = Q + (size_t)i * q_hidden + h * head_dim;
- float *o_row = out + (size_t)i * q_hidden + h * head_dim;
-
- float max_score = -1e30f;
- float sum_exp = 0.0f;
- for (int d = 0; d < head_dim; d++)
- o_row[d] = 0.0f;
-
- for (int j = start; j <= i; j++) {
- const float *k_row = K + (size_t)j * kv_hidden + kv_h * head_dim;
- const float *v_row = V + (size_t)j * kv_hidden + kv_h * head_dim;
- float score = embed_dot_f32_fast(q_row, k_row, head_dim) * scale;
-
- if (score > max_score) {
- float correction = expf(max_score - score);
- sum_exp = sum_exp * correction + 1.0f;
- embed_vec_scale_add(o_row, v_row, correction, head_dim);
- max_score = score;
- } else {
- float wt = expf(score - max_score);
- sum_exp += wt;
- embed_vec_axpy_inplace(o_row, v_row, wt, head_dim);
- }
- }
-
- if (sum_exp > 0.0f)
- embed_vec_scale_inplace(o_row, 1.0f / sum_exp, head_dim);
- }
-}
-
-#ifdef USE_BLAS
-static void embed_causal_gqa_attention_packed_blas_rows(float *out,
- const float *Q,
- const float *K,
- const float *V,
- int start,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- int h,
- int query_start,
- int query_end,
- float *scores) {
- int heads_per_kv = n_heads / n_kv_heads;
- int q_hidden = n_heads * head_dim;
- int kv_hidden = n_kv_heads * head_dim;
- int kv_h = h / heads_per_kv;
- int rows = query_end - query_start;
- int keys = query_end - start;
-
- const float *q = Q + (size_t)query_start * q_hidden + h * head_dim;
- const float *k = K + (size_t)start * kv_hidden + kv_h * head_dim;
- const float *v = V + (size_t)start * kv_hidden + kv_h * head_dim;
- float *o = out + (size_t)query_start * q_hidden + h * head_dim;
-
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, rows, keys, head_dim, scale, q, q_hidden,
- k, kv_hidden, 0.0f, scores, keys);
- for (int r = 0; r < rows; r++) {
- int allowed = query_start + r - start + 1;
- float *row = scores + (size_t)r * keys;
- for (int j = allowed; j < keys; j++)
- row[j] = -INFINITY;
- }
- embed_softmax(scores, rows, keys);
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasNoTrans, rows, head_dim, keys, 1.0f, scores, keys,
- v, kv_hidden, 0.0f, o, q_hidden);
-}
-#endif
-
-static void packed_causal_gqa_attn_worker(int tid, int n_threads, void *arg) {
- packed_gqa_attn_task_t *t = (packed_gqa_attn_task_t *)arg;
- int task_id = 0;
-#ifdef USE_BLAS
- float *scores =
- t->scores ? t->scores + (size_t)tid * EMBED_PACKED_ATTN_QUERY_TILE * t->max_seq : NULL;
-#endif
-
- for (int h = 0; h < t->n_heads; h++) {
- for (int b = 0; b < t->batch; b++) {
- int start = t->offsets[b];
- int end = t->offsets[b + 1];
- for (int q0 = start; q0 < end; q0 += EMBED_PACKED_ATTN_QUERY_TILE) {
- int q1 = q0 + EMBED_PACKED_ATTN_QUERY_TILE;
- if (q1 > end)
- q1 = end;
- if (task_id++ % n_threads != tid)
- continue;
-#ifdef USE_BLAS
- if (scores && end - start >= EMBED_PACKED_ATTN_BLAS_MIN_SEQ) {
- embed_causal_gqa_attention_packed_blas_rows(
- t->out, t->Q, t->K, t->V, start, t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1, scores);
- continue;
- }
-#endif
- embed_causal_gqa_attention_packed_online_rows(t->out, t->Q, t->K, t->V, start,
- t->n_heads, t->n_kv_heads, t->head_dim,
- t->scale, h, q0, q1);
- }
- }
- }
-}
-
-void embed_causal_gqa_attention_packed_with_scratch(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale,
- float *scratch,
- size_t scratch_bytes) {
- long long qk_work = 0;
- int max_seq = 0;
- for (int b = 0; b < batch; b++) {
- int len = offsets[b + 1] - offsets[b];
- qk_work += (long long)len * (len + 1) / 2 * n_heads;
- if (len > max_seq)
- max_seq = len;
- }
-
- float *scores = NULL;
-#ifdef USE_BLAS
- size_t required = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- if (required != 0 && scratch && scratch_bytes >= required)
- scores = scratch;
-#else
- (void)scratch;
- (void)scratch_bytes;
-#endif
- packed_gqa_attn_task_t task = {.out = out,
- .Q = Q,
- .K = K,
- .V = V,
- .offsets = offsets,
- .batch = batch,
- .n_heads = n_heads,
- .n_kv_heads = n_kv_heads,
- .head_dim = head_dim,
- .scale = scale,
- .scores = scores,
- .max_seq = max_seq};
- if (embed_num_threads() > 1 && n_heads >= 2 && qk_work >= 4096)
- parallel_for(packed_causal_gqa_attn_worker, &task);
- else
- packed_causal_gqa_attn_worker(0, 1, &task);
-}
-
-void embed_causal_gqa_attention_packed(float *out,
- const float *Q,
- const float *K,
- const float *V,
- const int *offsets,
- int batch,
- int n_heads,
- int n_kv_heads,
- int head_dim,
- float scale) {
- size_t scratch_bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- float *scores = scratch_bytes ? malloc(scratch_bytes) : NULL;
- embed_causal_gqa_attention_packed_with_scratch(out, Q, K, V, offsets, batch, n_heads, n_kv_heads,
- head_dim, scale, scores, scratch_bytes);
- free(scores);
-}
diff --git a/src/kernels_avx.c b/src/kernels_avx.c
deleted file mode 100644
index 920fecf..0000000
--- a/src/kernels_avx.c
+++ /dev/null
@@ -1,439 +0,0 @@
-/*
- * embed_kernels_avx.c - x86 SIMD hot kernels (AVX2+FMA, with AVX-512 when available)
- *
- * bf16 to f32 conversion: load 16 uint16, zero-extend to 32-bit, shift left 16.
- * Uses AVX-512F+BW for 16-wide bf16 matvec (dominant for seq_len==1), and
- * AVX2+FMA / AVX-512F for f32 attention helpers (which operate on cache-resident data).
- *
- * The bf16 matvec processes 4 output rows simultaneously to reduce instruction
- * overhead and improve out-of-order execution on memory-bound workloads.
- */
-
-#include "kernels_impl.h"
-
-#if defined(__AVX2__) && defined(__FMA__)
-
-#include <immintrin.h>
-#include <string.h>
-
-/* =====================================================================
- * BF16 matvec - uses AVX-512F+BW when available, AVX2 fallback otherwise
- * ===================================================================== */
-
-#if defined(__AVX512F__) && defined(__AVX512BW__)
-
-void embed_bf16_matvec_fused_avx(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
- int o = 0;
-
- /* Process 4 output rows at a time */
- for (; o + 3 < out_dim; o += 4) {
- const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
- const uint16_t *w1 = w0 + in_dim;
- const uint16_t *w2 = w1 + in_dim;
- const uint16_t *w3 = w2 + in_dim;
-
- __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();
- __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();
- __m512 a4 = _mm512_setzero_ps(), a5 = _mm512_setzero_ps();
- __m512 a6 = _mm512_setzero_ps(), a7 = _mm512_setzero_ps();
- int k = 0;
-
- for (; k + 32 <= in_dim; k += 32) {
- __m512 xv0 = _mm512_loadu_ps(x + k);
- __m512 xv1 = _mm512_loadu_ps(x + k + 16);
-
- /* Row 0 */
- __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k));
- __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16));
- a0 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0);
- a1 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1);
-
- /* Row 1 */
- __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k));
- __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16));
- a2 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2);
- a3 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3);
-
- /* Row 2 */
- __m256i r2a = _mm256_loadu_si256((const __m256i *)(w2 + k));
- __m256i r2b = _mm256_loadu_si256((const __m256i *)(w2 + k + 16));
- a4 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2a), 16)), xv0, a4);
- a5 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2b), 16)), xv1, a5);
-
- /* Row 3 */
- __m256i r3a = _mm256_loadu_si256((const __m256i *)(w3 + k));
- __m256i r3b = _mm256_loadu_si256((const __m256i *)(w3 + k + 16));
- a6 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3a), 16)), xv0, a6);
- a7 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3b), 16)), xv1, a7);
- }
-
- for (; k + 16 <= in_dim; k += 16) {
- __m512 xv = _mm512_loadu_ps(x + k);
- __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k));
- __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k));
- __m256i r2 = _mm256_loadu_si256((const __m256i *)(w2 + k));
- __m256i r3 = _mm256_loadu_si256((const __m256i *)(w3 + k));
- a0 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0);
- a2 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2);
- a4 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r2), 16)), xv, a4);
- a6 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r3), 16)), xv, a6);
- }
-
- float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f);
- float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o + 1] : 0.0f);
- float s2 = _mm512_reduce_add_ps(_mm512_add_ps(a4, a5)) + (bias ? bias[o + 2] : 0.0f);
- float s3 = _mm512_reduce_add_ps(_mm512_add_ps(a6, a7)) + (bias ? bias[o + 3] : 0.0f);
-
- for (; k < in_dim; k++) {
- float xk = x[k];
- uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16;
- uint32_t b2 = ((uint32_t)w2[k]) << 16, b3 = ((uint32_t)w3[k]) << 16;
- float v0, v1, v2, v3;
- memcpy(&v0, &b0, 4);
- memcpy(&v1, &b1, 4);
- memcpy(&v2, &b2, 4);
- memcpy(&v3, &b3, 4);
- s0 += v0 * xk;
- s1 += v1 * xk;
- s2 += v2 * xk;
- s3 += v3 * xk;
- }
- y[o] = s0;
- y[o + 1] = s1;
- y[o + 2] = s2;
- y[o + 3] = s3;
- }
-
- /* Handle 2-row remainder */
- for (; o + 1 < out_dim; o += 2) {
- const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
- const uint16_t *w1 = w0 + in_dim;
- __m512 a0 = _mm512_setzero_ps(), a1 = _mm512_setzero_ps();
- __m512 a2 = _mm512_setzero_ps(), a3 = _mm512_setzero_ps();
- int k = 0;
-
- for (; k + 32 <= in_dim; k += 32) {
- __m512 xv0 = _mm512_loadu_ps(x + k);
- __m512 xv1 = _mm512_loadu_ps(x + k + 16);
- __m256i r0a = _mm256_loadu_si256((const __m256i *)(w0 + k));
- __m256i r0b = _mm256_loadu_si256((const __m256i *)(w0 + k + 16));
- __m256i r1a = _mm256_loadu_si256((const __m256i *)(w1 + k));
- __m256i r1b = _mm256_loadu_si256((const __m256i *)(w1 + k + 16));
- a0 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0a), 16)), xv0, a0);
- a1 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0b), 16)), xv1, a1);
- a2 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1a), 16)), xv0, a2);
- a3 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1b), 16)), xv1, a3);
- }
- for (; k + 16 <= in_dim; k += 16) {
- __m512 xv = _mm512_loadu_ps(x + k);
- __m256i r0 = _mm256_loadu_si256((const __m256i *)(w0 + k));
- __m256i r1 = _mm256_loadu_si256((const __m256i *)(w1 + k));
- a0 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r0), 16)), xv, a0);
- a2 = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(r1), 16)), xv, a2);
- }
-
- float s0 = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1)) + (bias ? bias[o] : 0.0f);
- float s1 = _mm512_reduce_add_ps(_mm512_add_ps(a2, a3)) + (bias ? bias[o + 1] : 0.0f);
-
- for (; k < in_dim; k++) {
- uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16;
- float v0, v1;
- memcpy(&v0, &b0, 4);
- memcpy(&v1, &b1, 4);
- s0 += v0 * x[k];
- s1 += v1 * x[k];
- }
- y[o] = s0;
- y[o + 1] = s1;
- }
-
- /* Single remaining row */
- for (; o < out_dim; o++) {
- const uint16_t *w = W_bf16 + (size_t)o * in_dim;
- __m512 acc = _mm512_setzero_ps();
- int k = 0;
- for (; k + 16 <= in_dim; k += 16) {
- __m256i raw = _mm256_loadu_si256((const __m256i *)(w + k));
- acc = _mm512_fmadd_ps(
- _mm512_castsi512_ps(_mm512_slli_epi32(_mm512_cvtepu16_epi32(raw), 16)),
- _mm512_loadu_ps(x + k), acc);
- }
- float sum = _mm512_reduce_add_ps(acc) + (bias ? bias[o] : 0.0f);
- for (; k < in_dim; k++) {
- uint32_t bits = ((uint32_t)w[k]) << 16;
- float wv;
- memcpy(&wv, &bits, 4);
- sum += wv * x[k];
- }
- y[o] = sum;
- }
-}
-
-#else /* AVX2 only (or AVX-512F without BW) */
-
-/* Helper: bf16 to f32 for 8 elements */
-static inline __m256 bf16x8_to_f32(__m128i raw) {
- return _mm256_castsi256_ps(_mm256_slli_epi32(_mm256_cvtepu16_epi32(raw), 16));
-}
-
-static inline void bf16x16_to_f32(const uint16_t *src, __m256 *lo, __m256 *hi) {
- __m256i raw = _mm256_loadu_si256((const __m256i *)src);
- *lo = bf16x8_to_f32(_mm256_castsi256_si128(raw));
- *hi = bf16x8_to_f32(_mm256_extracti128_si256(raw, 1));
-}
-
-void embed_bf16_matvec_fused_avx(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
- int o = 0;
-
- for (; o + 1 < out_dim; o += 2) {
- const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
- const uint16_t *w1 = w0 + in_dim;
- __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
- __m256 a2 = _mm256_setzero_ps(), a3 = _mm256_setzero_ps();
- int k = 0;
- for (; k + 16 <= in_dim; k += 16) {
- __m256 xlo = _mm256_loadu_ps(x + k);
- __m256 xhi = _mm256_loadu_ps(x + k + 8);
- __m256 wlo, whi;
- bf16x16_to_f32(w0 + k, &wlo, &whi);
- a0 = _mm256_fmadd_ps(wlo, xlo, a0);
- a1 = _mm256_fmadd_ps(whi, xhi, a1);
- bf16x16_to_f32(w1 + k, &wlo, &whi);
- a2 = _mm256_fmadd_ps(wlo, xlo, a2);
- a3 = _mm256_fmadd_ps(whi, xhi, a3);
- }
- a0 = _mm256_add_ps(a0, a1);
- a2 = _mm256_add_ps(a2, a3);
- __m128 r0 = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1));
- __m128 r1 = _mm_add_ps(_mm256_castps256_ps128(a2), _mm256_extractf128_ps(a2, 1));
- r0 = _mm_hadd_ps(r0, r1);
- r0 = _mm_hadd_ps(r0, r0);
- float s0 = _mm_cvtss_f32(r0) + (bias ? bias[o] : 0.0f);
- float s1 = _mm_cvtss_f32(_mm_shuffle_ps(r0, r0, 1)) + (bias ? bias[o + 1] : 0.0f);
- for (; k < in_dim; k++) {
- uint32_t b0 = ((uint32_t)w0[k]) << 16, b1 = ((uint32_t)w1[k]) << 16;
- float v0, v1;
- memcpy(&v0, &b0, 4);
- memcpy(&v1, &b1, 4);
- s0 += v0 * x[k];
- s1 += v1 * x[k];
- }
- y[o] = s0;
- y[o + 1] = s1;
- }
- for (; o < out_dim; o++) {
- const uint16_t *w = W_bf16 + (size_t)o * in_dim;
- __m256 a0 = _mm256_setzero_ps(), a1 = _mm256_setzero_ps();
- int k = 0;
- for (; k + 16 <= in_dim; k += 16) {
- __m256 wlo, whi;
- bf16x16_to_f32(w + k, &wlo, &whi);
- a0 = _mm256_fmadd_ps(wlo, _mm256_loadu_ps(x + k), a0);
- a1 = _mm256_fmadd_ps(whi, _mm256_loadu_ps(x + k + 8), a1);
- }
- a0 = _mm256_add_ps(a0, a1);
- __m128 r = _mm_add_ps(_mm256_castps256_ps128(a0), _mm256_extractf128_ps(a0, 1));
- r = _mm_hadd_ps(r, r);
- r = _mm_hadd_ps(r, r);
- float sum = _mm_cvtss_f32(r) + (bias ? bias[o] : 0.0f);
- for (; k < in_dim; k++) {
- uint32_t bits = ((uint32_t)w[k]) << 16;
- float wv;
- memcpy(&wv, &bits, 4);
- sum += wv * x[k];
- }
- y[o] = sum;
- }
-}
-
-#endif /* AVX-512F+BW vs AVX2 for bf16 */
-
-/* =====================================================================
- * f32 attention helpers - AVX2+FMA, with AVX-512F when available
- * (operates on L1-resident head vectors)
- * ===================================================================== */
-
-float embed_dot_f32_avx(const float *a, const float *b, int n) {
-#if defined(__AVX512F__)
- int i = 0;
- __m512 acc0 = _mm512_setzero_ps();
- __m512 acc1 = _mm512_setzero_ps();
- __m512 acc2 = _mm512_setzero_ps();
- __m512 acc3 = _mm512_setzero_ps();
- for (; i + 64 <= n; i += 64) {
- acc0 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc0);
- acc1 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 16), _mm512_loadu_ps(b + i + 16), acc1);
- acc2 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 32), _mm512_loadu_ps(b + i + 32), acc2);
- acc3 = _mm512_fmadd_ps(_mm512_loadu_ps(a + i + 48), _mm512_loadu_ps(b + i + 48), acc3);
- }
- __m512 acc = _mm512_add_ps(_mm512_add_ps(acc0, acc1), _mm512_add_ps(acc2, acc3));
- for (; i + 16 <= n; i += 16) {
- acc = _mm512_fmadd_ps(_mm512_loadu_ps(a + i), _mm512_loadu_ps(b + i), acc);
- }
- float sum = _mm512_reduce_add_ps(acc);
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-#else
- int i = 0;
- __m256 acc0 = _mm256_setzero_ps();
- __m256 acc1 = _mm256_setzero_ps();
- __m256 acc2 = _mm256_setzero_ps();
- __m256 acc3 = _mm256_setzero_ps();
- for (; i + 32 <= n; i += 32) {
- acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
- acc1 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 8), _mm256_loadu_ps(b + i + 8), acc1);
- acc2 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 16), _mm256_loadu_ps(b + i + 16), acc2);
- acc3 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i + 24), _mm256_loadu_ps(b + i + 24), acc3);
- }
- acc0 = _mm256_add_ps(_mm256_add_ps(acc0, acc1), _mm256_add_ps(acc2, acc3));
- for (; i + 8 <= n; i += 8) {
- acc0 = _mm256_fmadd_ps(_mm256_loadu_ps(a + i), _mm256_loadu_ps(b + i), acc0);
- }
- __m128 r = _mm_add_ps(_mm256_castps256_ps128(acc0), _mm256_extractf128_ps(acc0, 1));
- r = _mm_hadd_ps(r, r);
- r = _mm_hadd_ps(r, r);
- float sum = _mm_cvtss_f32(r);
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-#endif
-}
-
-void embed_vec_scale_inplace_avx(float *dst, float scale, int n) {
-#if defined(__AVX512F__)
- int i = 0;
- __m512 s = _mm512_set1_ps(scale);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
- _mm512_storeu_ps(dst + i + 16, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 16), s));
- _mm512_storeu_ps(dst + i + 32, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 32), s));
- _mm512_storeu_ps(dst + i + 48, _mm512_mul_ps(_mm512_loadu_ps(dst + i + 48), s));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i, _mm512_mul_ps(_mm512_loadu_ps(dst + i), s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-#else
- int i = 0;
- __m256 s = _mm256_set1_ps(scale);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
- _mm256_storeu_ps(dst + i + 8, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 8), s));
- _mm256_storeu_ps(dst + i + 16, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 16), s));
- _mm256_storeu_ps(dst + i + 24, _mm256_mul_ps(_mm256_loadu_ps(dst + i + 24), s));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i, _mm256_mul_ps(_mm256_loadu_ps(dst + i), s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-#endif
-}
-
-void embed_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n) {
-#if defined(__AVX512F__)
- int i = 0;
- __m512 a = _mm512_set1_ps(alpha);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
- _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 16), a,
- _mm512_loadu_ps(dst + i + 16)));
- _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 32), a,
- _mm512_loadu_ps(dst + i + 32)));
- _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(src + i + 48), a,
- _mm512_loadu_ps(dst + i + 48)));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(src + i), a, _mm512_loadu_ps(dst + i)));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-#else
- int i = 0;
- __m256 a = _mm256_set1_ps(alpha);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
- _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 8), a,
- _mm256_loadu_ps(dst + i + 8)));
- _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 16), a,
- _mm256_loadu_ps(dst + i + 16)));
- _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(src + i + 24), a,
- _mm256_loadu_ps(dst + i + 24)));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(src + i), a, _mm256_loadu_ps(dst + i)));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-#endif
-}
-
-void embed_vec_scale_add_avx(float *dst, const float *src, float correction, int n) {
-#if defined(__AVX512F__)
- int i = 0;
- __m512 c = _mm512_set1_ps(correction);
- for (; i + 64 <= n; i += 64) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
- _mm512_storeu_ps(dst + i + 16, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 16), c,
- _mm512_loadu_ps(src + i + 16)));
- _mm512_storeu_ps(dst + i + 32, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 32), c,
- _mm512_loadu_ps(src + i + 32)));
- _mm512_storeu_ps(dst + i + 48, _mm512_fmadd_ps(_mm512_loadu_ps(dst + i + 48), c,
- _mm512_loadu_ps(src + i + 48)));
- }
- for (; i + 16 <= n; i += 16) {
- _mm512_storeu_ps(dst + i,
- _mm512_fmadd_ps(_mm512_loadu_ps(dst + i), c, _mm512_loadu_ps(src + i)));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-#else
- int i = 0;
- __m256 c = _mm256_set1_ps(correction);
- for (; i + 32 <= n; i += 32) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
- _mm256_storeu_ps(dst + i + 8, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 8), c,
- _mm256_loadu_ps(src + i + 8)));
- _mm256_storeu_ps(dst + i + 16, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 16), c,
- _mm256_loadu_ps(src + i + 16)));
- _mm256_storeu_ps(dst + i + 24, _mm256_fmadd_ps(_mm256_loadu_ps(dst + i + 24), c,
- _mm256_loadu_ps(src + i + 24)));
- }
- for (; i + 8 <= n; i += 8) {
- _mm256_storeu_ps(dst + i,
- _mm256_fmadd_ps(_mm256_loadu_ps(dst + i), c, _mm256_loadu_ps(src + i)));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-#endif
-}
-
-#endif /* __AVX2__ && __FMA__ */
diff --git a/src/kernels_gemm.c b/src/kernels_gemm.c
deleted file mode 100644
index aa8dadf..0000000
--- a/src/kernels_gemm.c
+++ /dev/null
@@ -1,434 +0,0 @@
-#include "kernels.h"
-#include "kernels_impl.h"
-
-#include <math.h>
-#include <stdlib.h>
-#include <string.h>
-
-#if (defined(__AVX512F__) || defined(__AVX2__)) && \
- (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
-#include <immintrin.h>
-#endif
-#if defined(__ARM_NEON) || defined(__ARM_NEON__)
-#include <arm_neon.h>
-#endif
-
-#ifdef USE_BLAS
-#ifdef __APPLE__
-#include <Accelerate/Accelerate.h>
-#else
-#include <cblas.h>
-#endif
-#endif
-
-#ifndef M_PI
-#define M_PI 3.14159265358979323846
-#endif
-
-/* Matrix and vector ops: GEMM, the dtype-dispatch linear kernels, and the
- * BF16-fused matvec/QKV/pair workers, plus the elementwise residual add. */
-
-/* ========================================================================
- * Basic Element-wise Operations
- * ======================================================================== */
-
-void embed_add_inplace(float *a, const float *b, int n) {
- for (int i = 0; i < n; i++)
- a[i] += b[i];
-}
-
-/* ========================================================================
- * Matrix Operations
- * ======================================================================== */
-
-void embed_matmul_t(float *C, const float *A, const float *B, int M, int K, int N) {
-#ifdef USE_BLAS
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, M, N, K, 1.0f, A, K, B, K, 0.0f, C, N);
-#else
- for (int m = 0; m < M; m++) {
- for (int n = 0; n < N; n++) {
- float sum = 0.0f;
- for (int k = 0; k < K; k++) {
- sum += A[m * K + k] * B[n * K + k];
- }
- C[m * N + n] = sum;
- }
- }
-#endif
-}
-
-void embed_linear(float *y,
- const float *x,
- const float *W,
- const float *b,
- int seq_len,
- int in_dim,
- int out_dim) {
-#ifdef USE_BLAS
- if (seq_len == 1) {
- cblas_sgemv(CblasRowMajor, CblasNoTrans, out_dim, in_dim, 1.0f, W, in_dim, x, 1, 0.0f, y,
- 1);
- } else {
- cblas_sgemm(CblasRowMajor, CblasNoTrans, CblasTrans, seq_len, out_dim, in_dim, 1.0f, x,
- in_dim, W, in_dim, 0.0f, y, out_dim);
- }
- if (b != NULL) {
- for (int s = 0; s < seq_len; s++) {
- for (int o = 0; o < out_dim; o++) {
- y[s * out_dim + o] += b[o];
- }
- }
- }
-#else
- for (int s = 0; s < seq_len; s++) {
- const float *x_row = x + s * in_dim;
- float *y_row = y + s * out_dim;
- for (int o = 0; o < out_dim; o++) {
- const float *w_row = W + o * in_dim;
- float sum = (b != NULL) ? b[o] : 0.0f;
- for (int i = 0; i < in_dim; i++) {
- sum += x_row[i] * w_row[i];
- }
- y_row[o] = sum;
- }
- }
-#endif
-}
-
-void embed_linear_nobias(
- float *y, const float *x, const float *W, int seq_len, int in_dim, int out_dim) {
- embed_linear(y, x, W, NULL, seq_len, in_dim, out_dim);
-}
-
-/* Convert bf16 buffer to f32 buffer */
-void embed_bf16_to_f32_buf(float *dst, const uint16_t *src, size_t n) {
-#if defined(__ARM_NEON) || defined(__ARM_NEON__)
- size_t i = 0;
- uint32_t *d = (uint32_t *)(void *)dst;
- for (; i + 8 <= n; i += 8) {
- uint16x8_t v = vld1q_u16(src + i);
- uint32x4_t lo = vshlq_n_u32(vmovl_u16(vget_low_u16(v)), 16);
- uint32x4_t hi = vshlq_n_u32(vmovl_u16(vget_high_u16(v)), 16);
- vst1q_u32(d + i, lo);
- vst1q_u32(d + i + 4, hi);
- }
- for (; i < n; i++)
- d[i] = ((uint32_t)src[i]) << 16;
-#else
- uint32_t *d = (uint32_t *)(void *)dst;
- for (size_t i = 0; i < n; i++)
- d[i] = ((uint32_t)src[i]) << 16;
-#endif
-}
-
-/*
- * Fused BF16 matvec: y[out_dim] = W_bf16[out_dim, in_dim] @ x[in_dim] + bias
- * Processes 2 output rows at a time to amortize x vector loads.
- */
-static void bf16_matvec_fused(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
- embed_bf16_matvec_fused_impl(y, x, W_bf16, bias, in_dim, out_dim);
-}
-
-/* Threaded matvec: split output rows across threads */
-typedef struct {
- float *y;
- const float *x;
- const uint16_t *W_bf16;
- const float *bias;
- int in_dim;
- int out_dim;
-} matvec_task_t;
-
-static void matvec_worker(int tid, int n_threads, void *arg) {
- matvec_task_t *t = (matvec_task_t *)arg;
- int chunk = (t->out_dim + n_threads - 1) / n_threads;
- int start = tid * chunk;
- int end = start + chunk;
- if (end > t->out_dim)
- end = t->out_dim;
- if (start >= end)
- return;
-
- bf16_matvec_fused(t->y + start, t->x, t->W_bf16 + (size_t)start * t->in_dim,
- t->bias ? t->bias + start : NULL, t->in_dim, end - start);
-}
-
-static void bf16_matvec_threaded(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
- if (embed_num_threads() <= 1) {
- bf16_matvec_fused(y, x, W_bf16, bias, in_dim, out_dim);
- return;
- }
- matvec_task_t task = {y, x, W_bf16, bias, in_dim, out_dim};
- parallel_for(matvec_worker, &task);
-}
-
-typedef struct {
- float *y;
- const float *x;
- const uint16_t *W_bf16;
- const float *bias;
- int seq_len;
- int in_dim;
- int out_dim;
-} bf16_linear_rows_task_t;
-
-static void bf16_linear_rows_worker(int tid, int n_threads, void *arg) {
- bf16_linear_rows_task_t *t = (bf16_linear_rows_task_t *)arg;
- int chunk = (t->seq_len + n_threads - 1) / n_threads;
- int start = tid * chunk;
- int end = start + chunk;
- if (end > t->seq_len)
- end = t->seq_len;
- if (start >= end)
- return;
-
- for (int s = start; s < end; s++) {
- bf16_matvec_fused(t->y + (size_t)s * t->out_dim, t->x + (size_t)s * t->in_dim, t->W_bf16,
- t->bias, t->in_dim, t->out_dim);
- }
-}
-
-static void bf16_linear_rows(float *y,
- const float *x,
- const uint16_t *W_bf16,
- const float *bias,
- int seq_len,
- int in_dim,
- int out_dim) {
- bf16_linear_rows_task_t task = {
- .y = y,
- .x = x,
- .W_bf16 = W_bf16,
- .bias = bias,
- .seq_len = seq_len,
- .in_dim = in_dim,
- .out_dim = out_dim,
- };
-
- if (embed_num_threads() > 1 && seq_len >= 2)
- parallel_for(bf16_linear_rows_worker, &task);
- else
- bf16_linear_rows_worker(0, 1, &task);
-}
-
-typedef struct {
- float *a;
- float *b;
- const float *x;
- const uint16_t *Wa_bf16;
- const uint16_t *Wb_bf16;
- int seq_len;
- int in_dim;
- int a_dim;
- int b_dim;
- int total_dim;
-} pair_matvec_task_t;
-
-static void pair_matvec_range(const pair_matvec_task_t *t, int row, int start, int end) {
- const float *x_row = t->x + (size_t)row * t->in_dim;
- int a_end = t->a_dim;
- int b_end = a_end + t->b_dim;
-
- if (start < a_end) {
- int s = start;
- int e = end < a_end ? end : a_end;
- if (s < e) {
- bf16_matvec_fused(t->a + (size_t)row * t->a_dim + s, x_row,
- t->Wa_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s);
- }
- }
-
- if (end > a_end && start < b_end) {
- int s = start > a_end ? start - a_end : 0;
- int e_abs = end < b_end ? end : b_end;
- int e = e_abs - a_end;
- if (s < e) {
- bf16_matvec_fused(t->b + (size_t)row * t->b_dim + s, x_row,
- t->Wb_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s);
- }
- }
-}
-
-static void pair_matvec_worker(int tid, int n_threads, void *arg) {
- pair_matvec_task_t *t = (pair_matvec_task_t *)arg;
- int total_work = t->seq_len * t->total_dim;
- int chunk = (total_work + n_threads - 1) / n_threads;
- int start = tid * chunk;
- int end = start + chunk;
- if (end > total_work)
- end = total_work;
- if (start >= end)
- return;
-
- while (start < end) {
- int row = start / t->total_dim;
- int pos = start - row * t->total_dim;
- int row_end = (row + 1) * t->total_dim;
- int stop = end < row_end ? end : row_end;
- pair_matvec_range(t, row, pos, stop - row * t->total_dim);
- start = stop;
- }
-}
-
-void embed_linear_nobias_bf16_pair(float *a,
- float *b,
- const float *x,
- const uint16_t *Wa_bf16,
- const uint16_t *Wb_bf16,
- int seq_len,
- int in_dim,
- int a_dim,
- int b_dim) {
- if (seq_len <= 0)
- return;
-
- if (embed_num_threads() <= 1) {
- for (int s = 0; s < seq_len; s++) {
- const float *x_row = x + (size_t)s * in_dim;
- bf16_matvec_fused(a + (size_t)s * a_dim, x_row, Wa_bf16, NULL, in_dim, a_dim);
- bf16_matvec_fused(b + (size_t)s * b_dim, x_row, Wb_bf16, NULL, in_dim, b_dim);
- }
- return;
- }
-
- pair_matvec_task_t task = {
- .a = a,
- .b = b,
- .x = x,
- .Wa_bf16 = Wa_bf16,
- .Wb_bf16 = Wb_bf16,
- .seq_len = seq_len,
- .in_dim = in_dim,
- .a_dim = a_dim,
- .b_dim = b_dim,
- .total_dim = a_dim + b_dim,
- };
- parallel_for(pair_matvec_worker, &task);
-}
-
-typedef struct {
- float *q;
- float *k;
- float *v;
- const float *x;
- const uint16_t *Wq_bf16;
- const uint16_t *Wk_bf16;
- const uint16_t *Wv_bf16;
- int in_dim;
- int q_dim;
- int kv_dim;
- int seq_len;
- int total_dim;
-} qkv_matvec_task_t;
-
-static void qkv_matvec_range(const qkv_matvec_task_t *t, int row, int start, int end) {
- const float *x_row = t->x + (size_t)row * t->in_dim;
- int q_end = t->q_dim;
- int k_end = q_end + t->kv_dim;
- int v_end = k_end + t->kv_dim;
-
- if (start < q_end) {
- int s = start;
- int e = end < q_end ? end : q_end;
- if (s < e) {
- bf16_matvec_fused(t->q + (size_t)row * t->q_dim + s, x_row,
- t->Wq_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s);
- }
- }
-
- if (end > q_end && start < k_end) {
- int s = start > q_end ? start - q_end : 0;
- int e_abs = end < k_end ? end : k_end;
- int e = e_abs - q_end;
- if (s < e) {
- bf16_matvec_fused(t->k + (size_t)row * t->kv_dim + s, x_row,
- t->Wk_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s);
- }
- }
-
- if (end > k_end && start < v_end) {
- int s = start > k_end ? start - k_end : 0;
- int e_abs = end < v_end ? end : v_end;
- int e = e_abs - k_end;
- if (s < e) {
- bf16_matvec_fused(t->v + (size_t)row * t->kv_dim + s, x_row,
- t->Wv_bf16 + (size_t)s * t->in_dim, NULL, t->in_dim, e - s);
- }
- }
-}
-
-static void qkv_matvec_worker(int tid, int n_threads, void *arg) {
- qkv_matvec_task_t *t = (qkv_matvec_task_t *)arg;
- int total_work = t->seq_len * t->total_dim;
- int chunk = (total_work + n_threads - 1) / n_threads;
- int start = tid * chunk;
- int end = start + chunk;
- if (end > total_work)
- end = total_work;
- if (start >= end)
- return;
-
- while (start < end) {
- int row = start / t->total_dim;
- int pos = start - row * t->total_dim;
- int row_end = (row + 1) * t->total_dim;
- int stop = end < row_end ? end : row_end;
- qkv_matvec_range(t, row, pos, stop - row * t->total_dim);
- start = stop;
- }
-}
-
-void embed_linear_nobias_bf16_qkv(float *q,
- float *k,
- float *v,
- const float *x,
- const uint16_t *Wq_bf16,
- const uint16_t *Wk_bf16,
- const uint16_t *Wv_bf16,
- int seq_len,
- int in_dim,
- int q_dim,
- int kv_dim) {
- if (seq_len <= 0)
- return;
-
- if (embed_num_threads() <= 1) {
- for (int s = 0; s < seq_len; s++) {
- const float *x_row = x + (size_t)s * in_dim;
- bf16_matvec_fused(q + (size_t)s * q_dim, x_row, Wq_bf16, NULL, in_dim, q_dim);
- bf16_matvec_fused(k + (size_t)s * kv_dim, x_row, Wk_bf16, NULL, in_dim, kv_dim);
- bf16_matvec_fused(v + (size_t)s * kv_dim, x_row, Wv_bf16, NULL, in_dim, kv_dim);
- }
- return;
- }
-
- qkv_matvec_task_t task = {
- .q = q,
- .k = k,
- .v = v,
- .x = x,
- .Wq_bf16 = Wq_bf16,
- .Wk_bf16 = Wk_bf16,
- .Wv_bf16 = Wv_bf16,
- .in_dim = in_dim,
- .q_dim = q_dim,
- .kv_dim = kv_dim,
- .seq_len = seq_len,
- .total_dim = q_dim + 2 * kv_dim,
- };
- parallel_for(qkv_matvec_worker, &task);
-}
-
-void embed_linear_nobias_bf16(
- float *y, const float *x, const uint16_t *W_bf16, int seq_len, int in_dim, int out_dim) {
- if (seq_len == 1) {
- bf16_matvec_threaded(y, x, W_bf16, NULL, in_dim, out_dim);
- return;
- }
- /* Callers route longer sequences through an F32-widened weight matrix in
- * caller-owned scratch (see embed.c); this per-row path stays correct for
- * any length. */
- bf16_linear_rows(y, x, W_bf16, NULL, seq_len, in_dim, out_dim);
-}
diff --git a/src/kernels_generic.c b/src/kernels_generic.c
deleted file mode 100644
index 7e85298..0000000
--- a/src/kernels_generic.c
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * embed_kernels_generic.c - architecture-generic hot kernels
- */
-
-#include "kernels_impl.h"
-
-#include <string.h>
-
-void embed_bf16_matvec_fused_generic(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
- for (int o = 0; o < out_dim; o++) {
- const uint16_t *w_row = W_bf16 + (size_t)o * in_dim;
- float sum = bias ? bias[o] : 0.0f;
- for (int k = 0; k < in_dim; k++) {
- uint32_t f32_bits = ((uint32_t)w_row[k]) << 16;
- float w_val;
- memcpy(&w_val, &f32_bits, sizeof(float));
- sum += w_val * x[k];
- }
- y[o] = sum;
- }
-}
-
-float embed_dot_f32_generic(const float *a, const float *b, int n) {
- float sum = 0.0f;
- for (int i = 0; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-}
-
-void embed_vec_scale_inplace_generic(float *dst, float scale, int n) {
- for (int i = 0; i < n; i++)
- dst[i] *= scale;
-}
-
-void embed_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n) {
- for (int i = 0; i < n; i++)
- dst[i] += alpha * src[i];
-}
-
-void embed_vec_scale_add_generic(float *dst, const float *src, float correction, int n) {
- for (int i = 0; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-}
diff --git a/src/kernels_impl.h b/src/kernels_impl.h
deleted file mode 100644
index 4ee7d74..0000000
--- a/src/kernels_impl.h
+++ /dev/null
@@ -1,53 +0,0 @@
-/*
- * embed_kernels_impl.h - internal architecture dispatch for hot kernels
- */
-
-#ifndef EMBED_KERNELS_IMPL_H
-#define EMBED_KERNELS_IMPL_H
-
-#include <stdint.h>
-
-void embed_bf16_matvec_fused_generic(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
-float embed_dot_f32_generic(const float *a, const float *b, int n);
-void embed_vec_scale_inplace_generic(float *dst, float scale, int n);
-void embed_vec_axpy_inplace_generic(float *dst, const float *src, float alpha, int n);
-void embed_vec_scale_add_generic(float *dst, const float *src, float correction, int n);
-
-#ifdef __ARM_NEON
-void embed_bf16_matvec_fused_neon(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
-float embed_dot_f32_neon(const float *a, const float *b, int n);
-void embed_vec_scale_inplace_neon(float *dst, float scale, int n);
-void embed_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n);
-void embed_vec_scale_add_neon(float *dst, const float *src, float correction, int n);
-
-#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_neon
-#define embed_dot_f32_impl embed_dot_f32_neon
-#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_neon
-#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_neon
-#define embed_vec_scale_add_impl embed_vec_scale_add_neon
-
-#elif defined(__AVX2__) && defined(__FMA__)
-void embed_bf16_matvec_fused_avx(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim);
-float embed_dot_f32_avx(const float *a, const float *b, int n);
-void embed_vec_scale_inplace_avx(float *dst, float scale, int n);
-void embed_vec_axpy_inplace_avx(float *dst, const float *src, float alpha, int n);
-void embed_vec_scale_add_avx(float *dst, const float *src, float correction, int n);
-
-#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_avx
-#define embed_dot_f32_impl embed_dot_f32_avx
-#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_avx
-#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_avx
-#define embed_vec_scale_add_impl embed_vec_scale_add_avx
-
-#else
-#define embed_bf16_matvec_fused_impl embed_bf16_matvec_fused_generic
-#define embed_dot_f32_impl embed_dot_f32_generic
-#define embed_vec_scale_inplace_impl embed_vec_scale_inplace_generic
-#define embed_vec_axpy_inplace_impl embed_vec_axpy_inplace_generic
-#define embed_vec_scale_add_impl embed_vec_scale_add_generic
-#endif
-
-#endif /* EMBED_KERNELS_IMPL_H */
diff --git a/src/kernels_neon.c b/src/kernels_neon.c
deleted file mode 100644
index 3e2a423..0000000
--- a/src/kernels_neon.c
+++ /dev/null
@@ -1,179 +0,0 @@
-/*
- * embed_kernels_neon.c - ARM NEON hot kernels
- */
-
-#include "kernels_impl.h"
-
-#ifdef __ARM_NEON
-
-#include <arm_neon.h>
-#include <string.h>
-
-void embed_bf16_matvec_fused_neon(
- float *y, const float *x, const uint16_t *W_bf16, const float *bias, int in_dim, int out_dim) {
- int o = 0;
-
- /* Process 2 output rows at a time, 32 elements/iter, 8 accumulators */
- for (; o + 1 < out_dim; o += 2) {
- const uint16_t *w0 = W_bf16 + (size_t)o * in_dim;
- const uint16_t *w1 = W_bf16 + (size_t)(o + 1) * in_dim;
- float s0 = bias ? bias[o] : 0.0f;
- float s1 = bias ? bias[o + 1] : 0.0f;
-
- float32x4_t a0 = vdupq_n_f32(0.0f), a1 = vdupq_n_f32(0.0f);
- float32x4_t a2 = vdupq_n_f32(0.0f), a3 = vdupq_n_f32(0.0f);
- float32x4_t b0 = vdupq_n_f32(0.0f), b1 = vdupq_n_f32(0.0f);
- float32x4_t b2 = vdupq_n_f32(0.0f), b3 = vdupq_n_f32(0.0f);
- int k = 0;
-
- for (; k + 32 <= in_dim; k += 32) {
- float32x4_t x0 = vld1q_f32(x + k);
- float32x4_t x1 = vld1q_f32(x + k + 4);
- float32x4_t x2 = vld1q_f32(x + k + 8);
- float32x4_t x3 = vld1q_f32(x + k + 12);
- float32x4_t x4 = vld1q_f32(x + k + 16);
- float32x4_t x5 = vld1q_f32(x + k + 20);
- float32x4_t x6 = vld1q_f32(x + k + 24);
- float32x4_t x7 = vld1q_f32(x + k + 28);
-
- uint16x8_t r0a = vld1q_u16(w0 + k);
- uint16x8_t r0b = vld1q_u16(w0 + k + 8);
- uint16x8_t r0c = vld1q_u16(w0 + k + 16);
- uint16x8_t r0d = vld1q_u16(w0 + k + 24);
- a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0a), 16)), x0);
- a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0a), 16)), x1);
- a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0b), 16)), x2);
- a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0b), 16)), x3);
- a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0c), 16)), x4);
- a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0c), 16)), x5);
- a2 = vfmaq_f32(a2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0d), 16)), x6);
- a3 = vfmaq_f32(a3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0d), 16)), x7);
-
- uint16x8_t r1a = vld1q_u16(w1 + k);
- uint16x8_t r1b = vld1q_u16(w1 + k + 8);
- uint16x8_t r1c = vld1q_u16(w1 + k + 16);
- uint16x8_t r1d = vld1q_u16(w1 + k + 24);
- b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1a), 16)), x0);
- b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1a), 16)), x1);
- b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1b), 16)), x2);
- b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1b), 16)), x3);
- b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1c), 16)), x4);
- b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1c), 16)), x5);
- b2 = vfmaq_f32(b2, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1d), 16)), x6);
- b3 = vfmaq_f32(b3, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1d), 16)), x7);
- }
- for (; k + 8 <= in_dim; k += 8) {
- float32x4_t x0 = vld1q_f32(x + k);
- float32x4_t x1 = vld1q_f32(x + k + 4);
- uint16x8_t r0 = vld1q_u16(w0 + k);
- uint16x8_t r1 = vld1q_u16(w1 + k);
- a0 = vfmaq_f32(a0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r0), 16)), x0);
- a1 = vfmaq_f32(a1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r0), 16)), x1);
- b0 = vfmaq_f32(b0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(r1), 16)), x0);
- b1 = vfmaq_f32(b1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(r1), 16)), x1);
- }
- s0 += vaddvq_f32(vaddq_f32(vaddq_f32(a0, a2), vaddq_f32(a1, a3)));
- s1 += vaddvq_f32(vaddq_f32(vaddq_f32(b0, b2), vaddq_f32(b1, b3)));
-
- for (; k < in_dim; k++) {
- uint32_t bits0 = ((uint32_t)w0[k]) << 16;
- uint32_t bits1 = ((uint32_t)w1[k]) << 16;
- float wv0, wv1;
- memcpy(&wv0, &bits0, sizeof(float));
- memcpy(&wv1, &bits1, sizeof(float));
- s0 += wv0 * x[k];
- s1 += wv1 * x[k];
- }
- y[o] = s0;
- y[o + 1] = s1;
- }
-
- /* Handle remaining odd row */
- for (; o < out_dim; o++) {
- const uint16_t *w_row = W_bf16 + (size_t)o * in_dim;
- float sum = bias ? bias[o] : 0.0f;
- int k = 0;
-
- float32x4_t acc0 = vdupq_n_f32(0.0f);
- float32x4_t acc1 = vdupq_n_f32(0.0f);
- for (; k + 8 <= in_dim; k += 8) {
- uint16x8_t bf = vld1q_u16(w_row + k);
- acc0 = vfmaq_f32(acc0, vreinterpretq_f32_u32(vshll_n_u16(vget_low_u16(bf), 16)),
- vld1q_f32(x + k));
- acc1 = vfmaq_f32(acc1, vreinterpretq_f32_u32(vshll_n_u16(vget_high_u16(bf), 16)),
- vld1q_f32(x + k + 4));
- }
- sum += vaddvq_f32(vaddq_f32(acc0, acc1));
-
- for (; k < in_dim; k++) {
- uint32_t f32_bits = ((uint32_t)w_row[k]) << 16;
- float w_val;
- memcpy(&w_val, &f32_bits, sizeof(float));
- sum += w_val * x[k];
- }
- y[o] = sum;
- }
-}
-
-float embed_dot_f32_neon(const float *a, const float *b, int n) {
- int i = 0;
- float32x4_t acc0 = vdupq_n_f32(0.0f);
- float32x4_t acc1 = vdupq_n_f32(0.0f);
- for (; i + 8 <= n; i += 8) {
- float32x4_t a0 = vld1q_f32(a + i);
- float32x4_t b0 = vld1q_f32(b + i);
- float32x4_t a1 = vld1q_f32(a + i + 4);
- float32x4_t b1 = vld1q_f32(b + i + 4);
- acc0 = vfmaq_f32(acc0, a0, b0);
- acc1 = vfmaq_f32(acc1, a1, b1);
- }
- float sum = vaddvq_f32(vaddq_f32(acc0, acc1));
- for (; i < n; i++)
- sum += a[i] * b[i];
- return sum;
-}
-
-void embed_vec_scale_inplace_neon(float *dst, float scale, int n) {
- int i = 0;
- float32x4_t s = vdupq_n_f32(scale);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(vdupq_n_f32(0.0f), d0, s));
- vst1q_f32(dst + i + 4, vfmaq_f32(vdupq_n_f32(0.0f), d1, s));
- }
- for (; i < n; i++)
- dst[i] *= scale;
-}
-
-void embed_vec_axpy_inplace_neon(float *dst, const float *src, float alpha, int n) {
- int i = 0;
- float32x4_t a = vdupq_n_f32(alpha);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t s0 = vld1q_f32(src + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- float32x4_t s1 = vld1q_f32(src + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(d0, s0, a));
- vst1q_f32(dst + i + 4, vfmaq_f32(d1, s1, a));
- }
- for (; i < n; i++)
- dst[i] += alpha * src[i];
-}
-
-void embed_vec_scale_add_neon(float *dst, const float *src, float correction, int n) {
- int i = 0;
- float32x4_t c = vdupq_n_f32(correction);
- for (; i + 8 <= n; i += 8) {
- float32x4_t d0 = vld1q_f32(dst + i);
- float32x4_t s0 = vld1q_f32(src + i);
- float32x4_t d1 = vld1q_f32(dst + i + 4);
- float32x4_t s1 = vld1q_f32(src + i + 4);
- vst1q_f32(dst + i, vfmaq_f32(s0, d0, c));
- vst1q_f32(dst + i + 4, vfmaq_f32(s1, d1, c));
- }
- for (; i < n; i++)
- dst[i] = dst[i] * correction + src[i];
-}
-
-#endif /* __ARM_NEON */
diff --git a/src/kernels_norm.c b/src/kernels_norm.c
deleted file mode 100644
index 6bae71d..0000000
--- a/src/kernels_norm.c
+++ /dev/null
@@ -1,245 +0,0 @@
-#include "kernels.h"
-#include "kernels_impl.h"
-
-#include <math.h>
-#include <stdlib.h>
-#include <string.h>
-
-#if (defined(__AVX512F__) || defined(__AVX2__)) && \
- (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
-#include <immintrin.h>
-#endif
-#if defined(__ARM_NEON) || defined(__ARM_NEON__)
-#include <arm_neon.h>
-#endif
-
-#ifdef USE_BLAS
-#ifdef __APPLE__
-#include <Accelerate/Accelerate.h>
-#else
-#include <cblas.h>
-#endif
-#endif
-
-#ifndef M_PI
-#define M_PI 3.14159265358979323846
-#endif
-
-/* Normalization kernels: RMSNorm (Qwen) and LayerNorm (BERT), plus the
- * per-head RMSNorm used for QK-norm. */
-
-/* ========================================================================
- * Normalization
- * ======================================================================== */
-
-/* Rows split across the pool only when the tensor is big enough to pay
- * for the dispatch. */
-#define EMBED_RMS_NORM_PARALLEL_ELEMS 262144
-
-static void embed_rms_norm_range(
- float *out, const float *x, const float *weight, int start, int end, int hidden, float eps) {
- for (int s = start; s < end; s++) {
- const float *x_row = x + (size_t)s * hidden;
- float *out_row = out + (size_t)s * hidden;
-
-#if defined(__AVX512F__) && defined(__FMA__)
- __m512 accv = _mm512_setzero_ps();
- int i = 0;
- for (; i + 16 <= hidden; i += 16) {
- __m512 v = _mm512_loadu_ps(x_row + i);
- accv = _mm512_fmadd_ps(v, v, accv);
- }
- float sum_sq = _mm512_reduce_add_ps(accv);
- for (; i < hidden; i++)
- sum_sq += x_row[i] * x_row[i];
-#elif defined(__AVX2__) && defined(__FMA__)
- __m256 accv = _mm256_setzero_ps();
- int i = 0;
- for (; i + 8 <= hidden; i += 8) {
- __m256 v = _mm256_loadu_ps(x_row + i);
- accv = _mm256_fmadd_ps(v, v, accv);
- }
- __m128 acc128 = _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1));
- acc128 = _mm_hadd_ps(acc128, acc128);
- acc128 = _mm_hadd_ps(acc128, acc128);
- float sum_sq = _mm_cvtss_f32(acc128);
- for (; i < hidden; i++)
- sum_sq += x_row[i] * x_row[i];
-#else
- float sum_sq = 0.0f;
- for (int i = 0; i < hidden; i++) {
- sum_sq += x_row[i] * x_row[i];
- }
-#endif
- float rms_inv = 1.0f / sqrtf(sum_sq / hidden + eps);
-
-#if defined(__AVX512F__)
- __m512 scale = _mm512_set1_ps(rms_inv);
- int j = 0;
- for (; j + 16 <= hidden; j += 16) {
- __m512 vx = _mm512_loadu_ps(x_row + j);
- __m512 vw = _mm512_loadu_ps(weight + j);
- _mm512_storeu_ps(out_row + j, _mm512_mul_ps(_mm512_mul_ps(vx, vw), scale));
- }
- for (; j < hidden; j++)
- out_row[j] = x_row[j] * rms_inv * weight[j];
-#elif defined(__AVX2__)
- __m256 scale = _mm256_set1_ps(rms_inv);
- int j = 0;
- for (; j + 8 <= hidden; j += 8) {
- __m256 vx = _mm256_loadu_ps(x_row + j);
- __m256 vw = _mm256_loadu_ps(weight + j);
- _mm256_storeu_ps(out_row + j, _mm256_mul_ps(_mm256_mul_ps(vx, vw), scale));
- }
- for (; j < hidden; j++)
- out_row[j] = x_row[j] * rms_inv * weight[j];
-#else
- for (int i = 0; i < hidden; i++) {
- out_row[i] = x_row[i] * rms_inv * weight[i];
- }
-#endif
- }
-}
-
-typedef struct {
- float *out;
- const float *x;
- const float *weight;
- int seq_len;
- int hidden;
- float eps;
-} rms_norm_task_t;
-
-static void rms_norm_worker(int tid, int n_threads, void *arg) {
- rms_norm_task_t *t = (rms_norm_task_t *)arg;
- int chunk = (t->seq_len + n_threads - 1) / n_threads;
- int start = tid * chunk;
- int end = start + chunk;
- if (end > t->seq_len)
- end = t->seq_len;
- if (start >= end)
- return;
- embed_rms_norm_range(t->out, t->x, t->weight, start, end, t->hidden, t->eps);
-}
-
-void embed_rms_norm(
- float *out, const float *x, const float *weight, int seq_len, int hidden, float eps) {
- if (seq_len <= 0 || hidden <= 0)
- return;
-
- long long elems = (long long)seq_len * hidden;
- if (embed_num_threads() > 1 && elems >= EMBED_RMS_NORM_PARALLEL_ELEMS) {
- rms_norm_task_t task = {
- .out = out,
- .x = x,
- .weight = weight,
- .seq_len = seq_len,
- .hidden = hidden,
- .eps = eps,
- };
- parallel_for(rms_norm_worker, &task);
- } else {
- embed_rms_norm_range(out, x, weight, 0, seq_len, hidden, eps);
- }
-}
-
-void embed_rms_norm_per_head(
- float *x, const float *weight, int seq_len, int n_heads, int head_dim, float eps) {
- /* x is [seq, n_heads * head_dim] - normalize each [head_dim] segment */
- int hidden = n_heads * head_dim;
- for (int s = 0; s < seq_len; s++) {
- for (int h = 0; h < n_heads; h++) {
- float *vec = x + s * hidden + h * head_dim;
-
-#if defined(__AVX512F__) && defined(__FMA__)
- __m512 accv = _mm512_setzero_ps();
- int d = 0;
- for (; d + 16 <= head_dim; d += 16) {
- __m512 v = _mm512_loadu_ps(vec + d);
- accv = _mm512_fmadd_ps(v, v, accv);
- }
- float sum_sq = _mm512_reduce_add_ps(accv);
- for (; d < head_dim; d++)
- sum_sq += vec[d] * vec[d];
-#elif defined(__AVX2__) && defined(__FMA__)
- __m256 accv = _mm256_setzero_ps();
- int d = 0;
- for (; d + 8 <= head_dim; d += 8) {
- __m256 v = _mm256_loadu_ps(vec + d);
- accv = _mm256_fmadd_ps(v, v, accv);
- }
- __m128 acc128 =
- _mm_add_ps(_mm256_castps256_ps128(accv), _mm256_extractf128_ps(accv, 1));
- acc128 = _mm_hadd_ps(acc128, acc128);
- acc128 = _mm_hadd_ps(acc128, acc128);
- float sum_sq = _mm_cvtss_f32(acc128);
- for (; d < head_dim; d++)
- sum_sq += vec[d] * vec[d];
-#else
- float sum_sq = 0.0f;
- for (int d = 0; d < head_dim; d++) {
- sum_sq += vec[d] * vec[d];
- }
-#endif
- float rms_inv = 1.0f / sqrtf(sum_sq / head_dim + eps);
-
-#if defined(__AVX512F__)
- __m512 scale = _mm512_set1_ps(rms_inv);
- int j = 0;
- for (; j + 16 <= head_dim; j += 16) {
- __m512 v = _mm512_loadu_ps(vec + j);
- __m512 w = _mm512_loadu_ps(weight + j);
- _mm512_storeu_ps(vec + j, _mm512_mul_ps(_mm512_mul_ps(v, w), scale));
- }
- for (; j < head_dim; j++)
- vec[j] = vec[j] * rms_inv * weight[j];
-#elif defined(__AVX2__)
- __m256 scale = _mm256_set1_ps(rms_inv);
- int j = 0;
- for (; j + 8 <= head_dim; j += 8) {
- __m256 v = _mm256_loadu_ps(vec + j);
- __m256 w = _mm256_loadu_ps(weight + j);
- _mm256_storeu_ps(vec + j, _mm256_mul_ps(_mm256_mul_ps(v, w), scale));
- }
- for (; j < head_dim; j++)
- vec[j] = vec[j] * rms_inv * weight[j];
-#else
- for (int d = 0; d < head_dim; d++) {
- vec[d] = vec[d] * rms_inv * weight[d];
- }
-#endif
- }
- }
-}
-
-/* Mean-subtracting LayerNorm with bias. Safe in place (out may alias x): each
- * row's mean and variance are reduced before the row is rewritten. Scalar for
- * now; LayerNorm is not GEMM-bound, so SIMD/threading is a later perf step. */
-void embed_layer_norm(float *out,
- const float *x,
- const float *gamma,
- const float *beta,
- int seq_len,
- int hidden,
- float eps) {
- if (seq_len <= 0 || hidden <= 0)
- return;
- float inv_h = 1.0f / (float)hidden;
- for (int s = 0; s < seq_len; s++) {
- const float *row = x + (size_t)s * hidden;
- float *dst = out + (size_t)s * hidden;
- float mean = 0.0f;
- for (int d = 0; d < hidden; d++)
- mean += row[d];
- mean *= inv_h;
- float var = 0.0f;
- for (int d = 0; d < hidden; d++) {
- float diff = row[d] - mean;
- var += diff * diff;
- }
- var *= inv_h;
- float inv_std = 1.0f / sqrtf(var + eps);
- for (int d = 0; d < hidden; d++)
- dst[d] = gamma[d] * (row[d] - mean) * inv_std + beta[d];
- }
-}
diff --git a/src/kernels_rope.c b/src/kernels_rope.c
deleted file mode 100644
index ef1709a..0000000
--- a/src/kernels_rope.c
+++ /dev/null
@@ -1,137 +0,0 @@
-#include "kernels.h"
-#include "kernels_impl.h"
-
-#include <math.h>
-#include <stdlib.h>
-#include <string.h>
-
-#if (defined(__AVX512F__) || defined(__AVX2__)) && \
- (defined(__x86_64__) || defined(__i386__) || defined(_M_X64) || defined(_M_IX86))
-#include <immintrin.h>
-#endif
-#if defined(__ARM_NEON) || defined(__ARM_NEON__)
-#include <arm_neon.h>
-#endif
-
-#ifdef USE_BLAS
-#ifdef __APPLE__
-#include <Accelerate/Accelerate.h>
-#else
-#include <cblas.h>
-#endif
-#endif
-
-#ifndef M_PI
-#define M_PI 3.14159265358979323846
-#endif
-
-/* Rotary position embeddings: cache-free NeoX-style RoPE compute and apply. */
-
-/* ========================================================================
- * Position Embeddings
- * ======================================================================== */
-
-void embed_compute_rope_neox(
- float *cos_out, float *sin_out, const int *positions, int seq, int head_dim, float theta) {
- int half = head_dim / 2;
-
- for (int s = 0; s < seq; s++) {
- float pos = (float)positions[s];
- for (int d = 0; d < half; d++) {
- float freq = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
- float angle = pos * freq;
- float c = cosf(angle);
- float sn = sinf(angle);
- /* Duplicate for full head_dim */
- cos_out[s * head_dim + d] = c;
- cos_out[s * head_dim + half + d] = c;
- sin_out[s * head_dim + d] = sn;
- sin_out[s * head_dim + half + d] = sn;
- }
- }
-}
-
-void embed_apply_rope_neox(
- float *x, const float *cos_vals, const float *sin_vals, int seq, int n_heads, int head_dim) {
- /*
- * NeoX split-half style:
- * x1 = x[..., :half], x2 = x[..., half:]
- * rotated = cat(-x2, x1)
- * result = x * cos + rotated * sin
- */
- int half = head_dim / 2;
- int hidden = n_heads * head_dim;
-
- for (int s = 0; s < seq; s++) {
- const float *c = cos_vals + s * head_dim;
- const float *sn = sin_vals + s * head_dim;
-
- for (int h = 0; h < n_heads; h++) {
- float *vec = x + s * hidden + h * head_dim;
-
-#if defined(__AVX512F__) && defined(__FMA__)
- int d = 0;
- for (; d + 16 <= half; d += 16) {
- __m512 x1 = _mm512_loadu_ps(vec + d);
- __m512 x2 = _mm512_loadu_ps(vec + half + d);
- /* RoPE cache duplicates cos/sin across halves. */
- __m512 cc = _mm512_loadu_ps(c + d);
- __m512 ss = _mm512_loadu_ps(sn + d);
- __m512 new1 = _mm512_fmsub_ps(x1, cc, _mm512_mul_ps(x2, ss));
- __m512 new2 = _mm512_fmadd_ps(x2, cc, _mm512_mul_ps(x1, ss));
- _mm512_storeu_ps(vec + d, new1);
- _mm512_storeu_ps(vec + half + d, new2);
- }
- for (; d < half; d++) {
- float x1 = vec[d];
- float x2 = vec[half + d];
- vec[d] = x1 * c[d] + (-x2) * sn[d];
- vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
- }
-#elif defined(__AVX2__) && defined(__FMA__)
- int d = 0;
- for (; d + 8 <= half; d += 8) {
- __m256 x1 = _mm256_loadu_ps(vec + d);
- __m256 x2 = _mm256_loadu_ps(vec + half + d);
- __m256 cc = _mm256_loadu_ps(c + d);
- __m256 ss = _mm256_loadu_ps(sn + d);
- __m256 new1 = _mm256_fmsub_ps(x1, cc, _mm256_mul_ps(x2, ss));
- __m256 new2 = _mm256_fmadd_ps(x2, cc, _mm256_mul_ps(x1, ss));
- _mm256_storeu_ps(vec + d, new1);
- _mm256_storeu_ps(vec + half + d, new2);
- }
- for (; d < half; d++) {
- float x1 = vec[d];
- float x2 = vec[half + d];
- vec[d] = x1 * c[d] + (-x2) * sn[d];
- vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
- }
-#elif defined(__ARM_NEON) || defined(__ARM_NEON__)
- int d = 0;
- for (; d + 4 <= half; d += 4) {
- float32x4_t x1 = vld1q_f32(vec + d);
- float32x4_t x2 = vld1q_f32(vec + half + d);
- float32x4_t cc = vld1q_f32(c + d);
- float32x4_t ss = vld1q_f32(sn + d);
- float32x4_t new1 = vmlsq_f32(vmulq_f32(x1, cc), x2, ss);
- float32x4_t new2 = vfmaq_f32(vmulq_f32(x2, cc), x1, ss);
- vst1q_f32(vec + d, new1);
- vst1q_f32(vec + half + d, new2);
- }
- for (; d < half; d++) {
- float x1 = vec[d];
- float x2 = vec[half + d];
- vec[d] = x1 * c[d] + (-x2) * sn[d];
- vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
- }
-#else
- for (int d = 0; d < half; d++) {
- float x1 = vec[d]; /* first half */
- float x2 = vec[half + d]; /* second half */
- vec[d] = x1 * c[d] + (-x2) * sn[d];
- vec[half + d] = x2 * c[half + d] + x1 * sn[half + d];
- }
-#endif
- }
- }
-}
diff --git a/src/late.c b/src/late.c
deleted file mode 100644
index 6ba52cc..0000000
--- a/src/late.c
+++ /dev/null
@@ -1,383 +0,0 @@
-#include "embed.h"
-#include "model_internal.h"
-#include "kernels.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <limits.h>
-#include <float.h>
-
-/* ========================================================================
- * Late-interaction model
- *
- * A late-interaction model is a pooled base model plus a 1_Dense linear
- * projection from hidden states to token vectors. Instead of pooling to one
- * vector, it keeps one projected vector per kept token and scores query/doc
- * pairs with MaxSim. It reuses the base model's packed forward pass and
- * workspace; only the projection, the token-vector buffer, and the scoring
- * live here.
- * ======================================================================== */
-
-struct embed_late_model {
- embed_model_t *base;
- safetensors_file_t *dense_sf;
- embed_weight_ref_t projection; /* [token_dim, hidden] */
- int token_dim;
-};
-
-struct embed_late_workspace {
- const embed_late_model_t *model;
- embed_workspace_t *base_ws;
- float *states;
- int states_seq_cap;
-};
-
-static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) {
- if (!ws || !ws->model || n_tokens <= 0)
- return -1;
- if (n_tokens <= ws->states_seq_cap)
- return 0;
- int hidden = ws->model->base->config.hidden_size;
- if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0)
- return -1;
- ws->states_seq_cap = n_tokens;
- return 0;
-}
-
-static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) {
- if (!sf || !name)
- return NULL;
- for (int i = 0; i < sf->num_tensors; i++) {
- if (strcmp(sf->tensors[i].name, name) == 0)
- return &sf->tensors[i];
- }
- return NULL;
-}
-
-embed_late_model_t *embed_late_model_load(const char *model_dir) {
- if (!model_dir_has_late_projection(model_dir)) {
- fprintf(stderr, "embed_late_model_load: missing 1_Dense projection\n");
- return NULL;
- }
-
- embed_late_model_t *late = (embed_late_model_t *)calloc(1, sizeof(*late));
- if (!late)
- return NULL;
-
- late->base = model_load_range_ex(model_dir, 0, -1, 1);
- if (!late->base)
- goto fail;
-
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir);
- if (n < 0 || (size_t)n >= sizeof(path)) {
- fprintf(stderr, "embed_late_model_load: model path too long\n");
- goto fail;
- }
-
- late->dense_sf = safetensors_open(path);
- if (!late->dense_sf) {
- fprintf(stderr, "embed_late_model_load: cannot open %s\n", path);
- goto fail;
- }
-
- const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight");
- if (!t) {
- fprintf(stderr, "embed_late_model_load: missing linear.weight\n");
- goto fail;
- }
- if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX) {
- fprintf(stderr, "embed_late_model_load: bad projection shape\n");
- goto fail;
- }
-
- int64_t shape[2] = {t->shape[0], late->base->config.hidden_size};
- if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1))
- goto fail;
-
- /* Same F32/BF16-borrow, F16-widen contract as the backbone weights. */
- late->projection = weight_ref_from_tensor(late->dense_sf, t);
- if (!late->projection.data)
- goto fail;
- late->token_dim = (int)t->shape[0];
- return late;
-
-fail:
- embed_late_model_free(late);
- return NULL;
-}
-
-void embed_late_model_free(embed_late_model_t *model) {
- if (!model)
- return;
- embed_model_free(model->base);
- /* Free the projection before the mmap: only an F16-widened ref owns its
- * buffer; a borrowed F32/BF16 ref points into dense_sf and must not be freed. */
- if (model->projection.owned)
- free((void *)model->projection.data);
- safetensors_close(model->dense_sf);
- free(model);
-}
-
-embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model) {
- if (!model || !model->base)
- return NULL;
- embed_late_workspace_t *ws = (embed_late_workspace_t *)calloc(1, sizeof(*ws));
- if (!ws)
- return NULL;
- ws->model = model;
- ws->base_ws = embed_workspace_new(model->base);
- if (!ws->base_ws) {
- free(ws);
- return NULL;
- }
- return ws;
-}
-
-void embed_late_workspace_free(embed_late_workspace_t *ws) {
- if (!ws)
- return;
- embed_workspace_free(ws->base_ws);
- free(ws->states);
- free(ws);
-}
-
-const embed_config_t *embed_late_model_config(const embed_late_model_t *model) {
- return model && model->base ? &model->base->config : NULL;
-}
-
-int embed_late_model_token_dim(const embed_late_model_t *model) {
- return model ? model->token_dim : 0;
-}
-
-/* Internal (embed_internal.h): expose the base model and projection so GPU
- * backends can upload them to device without seeing struct embed_late_model. */
-embed_model_t *embed_late_model_base(const embed_late_model_t *model) {
- return model ? model->base : NULL;
-}
-
-const embed_weight_ref_t *embed_late_model_projection(const embed_late_model_t *model) {
- return model ? &model->projection : NULL;
-}
-
-int embed_late_model_encode_tokens(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors) {
- if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_vectors ||
- model->token_dim <= 0)
- return -1;
-
- int hidden = model->base->config.hidden_size;
- int dim = model->token_dim;
- if (ensure_late_state_buffer(ws, n_tokens) != 0)
- return -1;
- if (embed_model_forward_into(model->base, ws->base_ws, token_ids, n_tokens, ws->states) != 0)
- return -1;
-
- linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, n_tokens, hidden,
- dim);
- if (normalize) {
- for (int i = 0; i < n_tokens; i++) {
- if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
- return -1;
- }
- }
- return 0;
-}
-
-float embed_late_maxsim(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- int doc_tokens,
- int dim) {
- if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0)
- return 0.0f;
-
- float score = 0.0f;
- for (int qi = 0; qi < query_tokens; qi++) {
- const float *q = query_vectors + (size_t)qi * dim;
- float best = -FLT_MAX;
- for (int di = 0; di < doc_tokens; di++) {
- const float *d = doc_vectors + (size_t)di * dim;
- float dot = embed_dot_f32(q, d, dim);
- if (dot > best)
- best = dot;
- }
- score += best;
- }
- return score;
-}
-
-int embed_late_maxsim_batch(const float *query_vectors,
- int query_tokens,
- const float *doc_vectors,
- const int *doc_offsets,
- int docs,
- int dim,
- float *scores) {
- if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 ||
- docs <= 0 || dim <= 0 || doc_offsets[0] != 0)
- return -1;
-
- for (int i = 0; i < docs; i++) {
- if (doc_offsets[i] < 0 || doc_offsets[i + 1] <= doc_offsets[i])
- return -1;
- }
-
-#ifdef USE_BLAS
- /* One similarity GEMM per group of whole candidates:
- * S[query_tokens, group_tokens] = Q @ D_group^T, then per-document
- * row-max plus sum. The group token budget bounds the scratch matrix;
- * one oversized document still runs alone in a group of its own. */
- enum { LATE_GROUP_TOKEN_BUDGET = 4096 };
- int total = doc_offsets[docs];
- int budget = LATE_GROUP_TOKEN_BUDGET;
- size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget);
- int largest = 0;
- for (int i = 0; i < docs; i++) {
- int len = doc_offsets[i + 1] - doc_offsets[i];
- if (len > largest)
- largest = len;
- }
- if ((size_t)query_tokens * (size_t)largest > cap)
- cap = (size_t)query_tokens * (size_t)largest;
- float *sim = (float *)malloc(cap * sizeof(float));
- if (sim) {
- int doc = 0;
- while (doc < docs) {
- int first = doc;
- int start = doc_offsets[first];
- doc++;
- while (doc < docs && doc_offsets[doc + 1] - start <= budget &&
- doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens))
- doc++;
- int group_tokens = doc_offsets[doc] - start;
-
- embed_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim,
- group_tokens);
-
- for (int i = first; i < doc; i++) {
- int s0 = doc_offsets[i] - start;
- int s1 = doc_offsets[i + 1] - start;
- float score = 0.0f;
- for (int qi = 0; qi < query_tokens; qi++) {
- const float *row = sim + (size_t)qi * group_tokens;
- float best = row[s0];
- for (int t = s0 + 1; t < s1; t++)
- if (row[t] > best)
- best = row[t];
- score += best;
- }
- scores[i] = score;
- }
- }
- free(sim);
- return 0;
- }
- /* Allocation failure: fall through to the scalar path. */
-#endif
-
- for (int i = 0; i < docs; i++) {
- int start = doc_offsets[i];
- int end = doc_offsets[i + 1];
- scores[i] = embed_late_maxsim(query_vectors, query_tokens,
- doc_vectors + (size_t)start * dim, end - start, dim);
- }
- return 0;
-}
-
-/* Late-interaction batch encoder. Lives here, beside the per-document encoder,
- * and reuses the base model's packed/ragged batch forward pass. */
-int embed_late_model_encode_docs(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- float *out_vectors,
- int *out_offsets) {
- if (!model || !ws || ws->model != model || !doc_ids || !n_tokens || !keep || !n_keep ||
- n_docs <= 0 || !out_vectors || !out_offsets || model->token_dim <= 0)
- return -1;
-
- int hidden = model->base->config.hidden_size;
- int dim = model->token_dim;
-
- /* Pack all candidates for one block-diagonal forward. inputs/offsets index
- * the full token stream; out_offsets is the prefix sum of kept tokens. */
- embed_input_t *inputs = (embed_input_t *)malloc((size_t)n_docs * sizeof(*inputs));
- int *offsets = (int *)malloc((size_t)(n_docs + 1) * sizeof(*offsets));
- if (!inputs || !offsets) {
- free(inputs);
- free(offsets);
- return -1;
- }
-
- int total_keep = 0;
- out_offsets[0] = 0;
- for (int d = 0; d < n_docs; d++) {
- if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
- n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d]) {
- free(inputs);
- free(offsets);
- return -1;
- }
- for (int k = 0; k < n_keep[d]; k++) {
- if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d]) {
- free(inputs);
- free(offsets);
- return -1;
- }
- }
- inputs[d].ids = doc_ids[d];
- inputs[d].n_tokens = n_tokens[d];
- total_keep += n_keep[d];
- out_offsets[d + 1] = total_keep;
- }
-
- int total_seq = 0, max_seq = 0;
- int rc = build_offsets(inputs, n_docs, 1, offsets, &total_seq, &max_seq);
- if (rc == 0)
- rc = forward_packed_inplace(model->base, ws->base_ws, inputs, n_docs, offsets, total_seq,
- max_seq, 1);
- free(inputs);
- if (rc != 0) {
- free(offsets);
- return -1;
- }
-
- /* Gather each document's kept hidden rows into ws->states (its natural
- * [seq, hidden] shape), then project the packed [total_keep, hidden] block
- * to [total_keep, dim] in one GEMM, matching the per-document path. */
- if (ensure_late_state_buffer(ws, total_keep) != 0) {
- free(offsets);
- return -1;
- }
- const float *states = ws->base_ws->x;
- int pos = 0;
- for (int d = 0; d < n_docs; d++) {
- int start = offsets[d];
- for (int k = 0; k < n_keep[d]; k++) {
- memcpy(ws->states + (size_t)pos * hidden,
- states + (size_t)(start + keep[d][k]) * hidden, (size_t)hidden * sizeof(float));
- pos++;
- }
- }
- free(offsets);
-
- linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, total_keep,
- hidden, dim);
- if (normalize) {
- for (int i = 0; i < total_keep; i++) {
- if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
- return -1;
- }
- }
- return 0;
-}
diff --git a/src/mlx.c b/src/mlx.c
deleted file mode 100644
index 7d05e91..0000000
--- a/src/mlx.c
+++ /dev/null
@@ -1,2783 +0,0 @@
-#include "mlx.h"
-#include "config.h"
-#include "dtype.h"
-#include "embed.h"
-#include "safetensors.h"
-
-#include <mlx/c/mlx.h>
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <math.h>
-#include <limits.h>
-#include <float.h>
-
-/* ========================================================================
- * Per-layer MLX weight arrays
- * ======================================================================== */
-
-typedef struct {
- mlx_array w;
- mlx_array wt;
- mlx_array scales;
- mlx_array biases;
- int quantized;
- int bits;
- int group_size;
-} mlx_linear_t;
-
-typedef struct {
- mlx_linear_t wq, wk, wv, wo;
- mlx_array q_bias, k_bias, v_bias;
- mlx_array q_norm, k_norm;
- mlx_array input_norm, post_attn_norm;
- mlx_linear_t gate_proj, up_proj, down_proj;
- /* BERT family (unset for Qwen3; see embed_internal.h). wo/up_proj/down_proj
- * and input_norm/post_attn_norm are reused as the BERT attention output,
- * FFN matrices, and the two block LayerNorm weights. */
- mlx_array o_bias, attn_ln_bias, ffn_inter_bias, ffn_out_bias, ffn_ln_bias;
-} mlx_layer_t;
-
-struct embed_mlx_ctx {
- embed_config_t config;
- mlx_array embed_tokens;
- mlx_layer_t *layers; /* heap-allocated [n_layers] */
- mlx_array norm;
- /* BERT family (unset for Qwen3): learned positions, token-type, embed LN. */
- mlx_array position_embeddings, token_type_embeddings, embed_ln_w, embed_ln_b;
- mlx_stream stream;
- multi_safetensors_t *ms;
- int layer_start; /* loaded transformer range, inclusive */
- int layer_end; /* loaded transformer range, exclusive */
- int quantize_bits;
- int quantize_group_size;
- mlx_dtype weight_dtype;
-};
-
-struct embed_mlx_late_ctx {
- embed_mlx_ctx_t *base;
- multi_safetensors_t *dense_ms;
- mlx_linear_t projection;
- int token_dim;
-};
-
-struct embed_mlx_late_vectors {
- embed_mlx_late_ctx_t *owner;
- mlx_array vectors; /* [tokens, dim] */
- int tokens;
- int dim;
-};
-
-/* ========================================================================
- * Helpers
- * ======================================================================== */
-
-static int arr_ok(mlx_array a) { return a.ctx != NULL; }
-
-static int mlx_mul_size(size_t a, size_t b, size_t *out) {
- if (a != 0 && b > SIZE_MAX / a)
- return -1;
- *out = a * b;
- return 0;
-}
-
-static mlx_dtype mlx_weight_dtype(const embed_mlx_ctx_t *ctx) { return ctx->weight_dtype; }
-
-static size_t safetensor_dtype_size(safetensor_dtype_t dtype) {
- switch (dtype) {
- case DTYPE_F32:
- return sizeof(float);
- case DTYPE_BF16:
- case DTYPE_F16:
- return sizeof(uint16_t);
- default:
- return 0;
- }
-}
-
-static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf,
- const safetensor_t *t,
- const char *name,
- const int *shape,
- int ndim) {
- size_t elem_size = safetensor_dtype_size(t->dtype);
- if (elem_size == 0) {
- fprintf(stderr, "mlx: expected F32, BF16, or F16 for %s\n", name);
- return 0;
- }
- if (t->ndim != ndim) {
- fprintf(stderr, "mlx: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
- return 0;
- }
-
- size_t numel = 1;
- for (int i = 0; i < ndim; i++) {
- if (shape[i] <= 0 || t->shape[i] != shape[i]) {
- fprintf(stderr,
- "mlx: bad shape for %s at dim %d: "
- "got %lld, expected %d\n",
- name, i, (long long)t->shape[i], shape[i]);
- return 0;
- }
- if (mlx_mul_size(numel, (size_t)shape[i], &numel) != 0) {
- fprintf(stderr, "mlx: shape too large for %s\n", name);
- return 0;
- }
- }
-
- size_t bytes;
- if (mlx_mul_size(numel, elem_size, &bytes) != 0) {
- fprintf(stderr, "mlx: tensor data too large for %s\n", name);
- return 0;
- }
- if (t->data_size != bytes) {
- fprintf(stderr, "mlx: bad data size for %s: got %zu, expected %zu\n", name, t->data_size,
- bytes);
- return 0;
- }
-
- if (sf->header_size > SIZE_MAX - 8)
- return 0;
- size_t data_start = 8 + sf->header_size;
- if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
- t->data_size > sf->file_size - data_start - t->data_offset) {
- fprintf(stderr, "mlx: tensor data out of bounds for %s\n", name);
- return 0;
- }
- return 1;
-}
-
-static mlx_array
-load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) {
- safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
- if (!t) {
- fprintf(stderr, "mlx: tensor not found: %s\n", name);
- return (mlx_array){0};
- }
- if (!mlx_tensor_has_supported_shape(sf, t, name, shape, ndim))
- return (mlx_array){0};
- if (t->dtype == DTYPE_F16) {
- /* No F16 forward path on MLX (the matmul fast path keys on BF16). Widen
- * to F32 on the host at load so the whole forward runs in F32, mirroring
- * the CPU loader. mlx_array_new_data copies its buffer, so the temporary
- * is freed immediately. Public F16 snapshots load through here. */
- int64_t n = safetensor_numel(t);
- float *tmp = (float *)malloc((size_t)n * sizeof(float));
- if (!tmp)
- return (mlx_array){0};
- const uint16_t *src = (const uint16_t *)safetensors_data(sf, t);
- for (int64_t i = 0; i < n; i++)
- tmp[i] = embed_f16_to_f32(src[i]);
- mlx_array a = mlx_array_new_data(tmp, shape, ndim, MLX_FLOAT32);
- free(tmp);
- return a;
- }
- mlx_dtype dtype;
- if (t->dtype == DTYPE_F32) {
- dtype = MLX_FLOAT32;
- } else if (t->dtype == DTYPE_BF16) {
- dtype = MLX_BFLOAT16;
- } else {
- fprintf(stderr, "mlx: expected F32, BF16, or F16 for %s\n", name);
- return (mlx_array){0};
- }
- return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype);
-}
-
-static mlx_linear_t
-load_linear_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndim) {
- mlx_linear_t l = {0};
- l.w = load_tensor(ms, name, shape, ndim);
- return l;
-}
-
-static int linear_ok(mlx_linear_t l) { return arr_ok(l.w) || arr_ok(l.wt); }
-
-static void free_linear(mlx_linear_t *l) {
- mlx_array_free(l->w);
- mlx_array_free(l->wt);
- mlx_array_free(l->scales);
- mlx_array_free(l->biases);
- memset(l, 0, sizeof(*l));
-}
-
-static int prepare_linear_transpose(mlx_linear_t *l, mlx_stream s) {
- if (!l || l->quantized || !arr_ok(l->w) || arr_ok(l->wt))
- return 0;
-
- mlx_array transposed = mlx_array_new();
- mlx_transpose(&transposed, l->w, s);
- if (!arr_ok(transposed)) {
- mlx_array_free(transposed);
- fprintf(stderr, "mlx: failed to prepare transposed linear weight\n");
- return -1;
- }
-
- l->wt = mlx_array_new();
- mlx_contiguous(&l->wt, transposed, false, s);
- mlx_array_free(transposed);
- if (!arr_ok(l->wt)) {
- mlx_array_free(l->wt);
- l->wt = (mlx_array){0};
- fprintf(stderr, "mlx: failed to materialize transposed linear weight\n");
- return -1;
- }
- mlx_array_eval(l->wt);
- mlx_synchronize(s);
- mlx_array_free(l->w);
- l->w = (mlx_array){0};
- return 0;
-}
-
-static int quantize_linear(mlx_linear_t *l, int bits, int group_size, mlx_stream s) {
- if (!l || !arr_ok(l->w) || bits == 0)
- return 0;
- if (bits != 8) {
- fprintf(stderr, "mlx: quantized weights support only 8 bits\n");
- return -1;
- }
- if (group_size <= 0)
- group_size = 64;
-
- mlx_optional_int gs = {.value = group_size, .has_value = true};
- mlx_optional_int qb = {.value = bits, .has_value = true};
- mlx_array null_arr = (mlx_array){0};
- mlx_vector_array q = mlx_vector_array_new();
- mlx_quantize(&q, l->w, gs, qb, "affine", null_arr, s);
- if (!q.ctx || mlx_vector_array_size(q) < 2) {
- mlx_vector_array_free(q);
- fprintf(stderr, "mlx: quantize failed\n");
- return -1;
- }
-
- mlx_array qw = (mlx_array){0};
- mlx_array scales = (mlx_array){0};
- mlx_array biases = (mlx_array){0};
- mlx_vector_array_get(&qw, q, 0);
- mlx_vector_array_get(&scales, q, 1);
- if (mlx_vector_array_size(q) >= 3)
- mlx_vector_array_get(&biases, q, 2);
- mlx_vector_array_free(q);
-
- if (!arr_ok(qw) || !arr_ok(scales)) {
- mlx_array_free(qw);
- mlx_array_free(scales);
- mlx_array_free(biases);
- fprintf(stderr, "mlx: quantize returned invalid arrays\n");
- return -1;
- }
- mlx_array_eval(qw);
- mlx_array_eval(scales);
- if (arr_ok(biases))
- mlx_array_eval(biases);
- mlx_synchronize(s);
-
- mlx_array_free(l->w);
- mlx_array_free(l->wt);
- l->w = qw;
- l->wt = (mlx_array){0};
- l->scales = scales;
- l->biases = biases;
- l->quantized = 1;
- l->bits = bits;
- l->group_size = group_size;
- return 0;
-}
-
-/* y = x @ W^T */
-static mlx_array linear(mlx_array x, const mlx_linear_t *W, mlx_stream s) {
- if (W->quantized) {
- if (!arr_ok(W->w))
- return (mlx_array){0};
- mlx_array res = mlx_array_new();
- mlx_optional_int gs = {.value = W->group_size, .has_value = true};
- mlx_optional_int bits = {.value = W->bits, .has_value = true};
- mlx_array null_arr = (mlx_array){0};
- mlx_quantized_matmul(&res, x, W->w, W->scales, arr_ok(W->biases) ? W->biases : null_arr,
- true, gs, bits, "affine", s);
- return res;
- }
- mlx_array Wt;
- int free_wt = 0;
- if (arr_ok(W->wt)) {
- Wt = W->wt;
- } else if (arr_ok(W->w)) {
- Wt = mlx_array_new();
- mlx_transpose(&Wt, W->w, s);
- free_wt = 1;
- } else {
- return (mlx_array){0};
- }
- mlx_array res = mlx_array_new();
- if (mlx_array_dtype(Wt) == MLX_BFLOAT16 && mlx_array_dtype(x) == MLX_FLOAT32) {
- /* F32 activations with BF16 weights: upcast the weight operand and run
- * the matmul in F32 (so accumulation is F32). Only the weights are
- * BF16-rounded; the residual stream stays F32 (see mlx_forward_layers),
- * matching the CPU/CUDA accuracy while keeping BF16 weight storage. */
- mlx_array Wf = mlx_array_new();
- mlx_astype(&Wf, Wt, MLX_FLOAT32, s);
- mlx_matmul(&res, x, Wf, s);
- mlx_array_free(Wf);
- } else {
- mlx_matmul(&res, x, Wt, s);
- }
- if (free_wt)
- mlx_array_free(Wt);
- return res;
-}
-
-const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx) {
- return ctx ? &ctx->config : NULL;
-}
-
-/* Upcast a tensor to F32 in place (no-op if already F32 or empty). Used for the
- * tiny RMSNorm vectors so non-quantized BF16 models can run norms and the
- * residual stream in F32 while keeping the large matmul weights in BF16. */
-static void upcast_f32_inplace(mlx_array *a, mlx_stream s) {
- if (!a || !arr_ok(*a) || mlx_array_dtype(*a) == MLX_FLOAT32)
- return;
- mlx_array f = mlx_array_new();
- mlx_astype(&f, *a, MLX_FLOAT32, s);
- mlx_array_free(*a);
- *a = f;
-}
-
-/* ========================================================================
- * Load
- * ======================================================================== */
-
-static int mlx_model_dir_has_file(const char *model_dir, const char *rel) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel);
- if (n < 0 || (size_t)n >= sizeof(path))
- return 0;
- FILE *f = fopen(path, "rb");
- if (!f)
- return 0;
- fclose(f);
- return 1;
-}
-
-static int mlx_model_dir_has_late_projection(const char *model_dir) {
- return mlx_model_dir_has_file(model_dir, "1_Dense/config.json") &&
- mlx_model_dir_has_file(model_dir, "1_Dense/model.safetensors");
-}
-
-static embed_mlx_options_t normalize_mlx_options(const embed_mlx_options_t *opts) {
- embed_mlx_options_t out = {0};
- if (opts)
- out = *opts;
- if (out.quantize_bits != 0 && out.quantize_bits != 8) {
- fprintf(stderr, "mlx: --mlx-quantize-bits supports only 8\n");
- out.quantize_bits = -1;
- }
- if (out.quantize_bits && out.quantize_group_size <= 0)
- out.quantize_group_size = 64;
- return out;
-}
-
-static int quantize_layer(mlx_layer_t *l, int bits, int group_size, mlx_stream s) {
- if (bits == 0)
- return 0;
- return quantize_linear(&l->wq, bits, group_size, s) ||
- quantize_linear(&l->wk, bits, group_size, s) ||
- quantize_linear(&l->wv, bits, group_size, s) ||
- quantize_linear(&l->wo, bits, group_size, s) ||
- quantize_linear(&l->gate_proj, bits, group_size, s) ||
- quantize_linear(&l->up_proj, bits, group_size, s) ||
- quantize_linear(&l->down_proj, bits, group_size, s);
-}
-
-static int prepare_layer_transposes(mlx_layer_t *l, mlx_stream s) {
- return prepare_linear_transpose(&l->wq, s) || prepare_linear_transpose(&l->wk, s) ||
- prepare_linear_transpose(&l->wv, s) || prepare_linear_transpose(&l->wo, s) ||
- prepare_linear_transpose(&l->gate_proj, s) || prepare_linear_transpose(&l->up_proj, s) ||
- prepare_linear_transpose(&l->down_proj, s);
-}
-
-/* BERT-family MLX loader, mirroring load_bert_weights on the CPU side. The
- * shared post-load below (stream, transposes, optional BF16 upcast) runs after
- * either family fills ctx->layers. */
-static int load_mlx_bert_weights(embed_mlx_ctx_t *ctx, int layer_start, int layer_end) {
- const embed_config_t *c = &ctx->config;
- int h = c->hidden_size, inter = c->intermediate_size;
- char name[256];
-
- const char *p;
- if (multi_safetensors_find(ctx->ms, "embeddings.word_embeddings.weight", NULL))
- p = "";
- else if (multi_safetensors_find(ctx->ms, "bert.embeddings.word_embeddings.weight", NULL))
- p = "bert.";
- else if (multi_safetensors_find(ctx->ms, "roberta.embeddings.word_embeddings.weight", NULL))
- p = "roberta.";
- else {
- fprintf(stderr, "mlx: BERT-family word embeddings not found\n");
- return -1;
- }
-
- if (layer_start == 0) {
- int emb_shape[] = {c->vocab_size, h};
- snprintf(name, sizeof(name), "%sembeddings.word_embeddings.weight", p);
- ctx->embed_tokens = load_tensor(ctx->ms, name, emb_shape, 2);
- int pos_shape[] = {c->max_position_embeddings, h};
- snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p);
- ctx->position_embeddings = load_tensor(ctx->ms, name, pos_shape, 2);
- int type_shape[] = {c->type_vocab_size, h};
- snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p);
- ctx->token_type_embeddings = load_tensor(ctx->ms, name, type_shape, 2);
- int ln_shape[] = {h};
- snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p);
- ctx->embed_ln_w = load_tensor(ctx->ms, name, ln_shape, 1);
- snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p);
- ctx->embed_ln_b = load_tensor(ctx->ms, name, ln_shape, 1);
- if (!arr_ok(ctx->embed_tokens) || !arr_ok(ctx->position_embeddings) ||
- !arr_ok(ctx->token_type_embeddings) || !arr_ok(ctx->embed_ln_w) ||
- !arr_ok(ctx->embed_ln_b))
- return -1;
- }
-
- ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t));
- if (!ctx->layers)
- return -1;
-
- for (int i = layer_start; i < layer_end; i++) {
- mlx_layer_t *l = &ctx->layers[i];
-#define LDL(fld, suffix, ...) \
- do { \
- snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
- if (!linear_ok(l->fld)) \
- return -1; \
- } while (0)
-#define LDV(fld, suffix, ...) \
- do { \
- snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
- if (!arr_ok(l->fld)) \
- return -1; \
- } while (0)
-
- LDL(wq, "attention.self.query.weight", h, h);
- LDV(q_bias, "attention.self.query.bias", h);
- LDL(wk, "attention.self.key.weight", h, h);
- LDV(k_bias, "attention.self.key.bias", h);
- LDL(wv, "attention.self.value.weight", h, h);
- LDV(v_bias, "attention.self.value.bias", h);
- LDL(wo, "attention.output.dense.weight", h, h);
- LDV(o_bias, "attention.output.dense.bias", h);
- LDV(input_norm, "attention.output.LayerNorm.weight", h);
- LDV(attn_ln_bias, "attention.output.LayerNorm.bias", h);
- LDL(up_proj, "intermediate.dense.weight", inter, h);
- LDV(ffn_inter_bias, "intermediate.dense.bias", inter);
- LDL(down_proj, "output.dense.weight", h, inter);
- LDV(ffn_out_bias, "output.dense.bias", h);
- LDV(post_attn_norm, "output.LayerNorm.weight", h);
- LDV(ffn_ln_bias, "output.LayerNorm.bias", h);
-#undef LDL
-#undef LDV
- }
- return 0;
-}
-
-static embed_mlx_ctx_t *mlx_load_range_ex(const char *model_dir,
- int layer_start,
- int layer_end,
- const embed_mlx_options_t *opts,
- int allow_late) {
- embed_mlx_options_t options = normalize_mlx_options(opts);
- if (options.quantize_bits < 0)
- return NULL;
- if (!allow_late && mlx_model_dir_has_late_projection(model_dir)) {
- fprintf(stderr, "mlx: late-interaction models require token-level "
- "MaxSim support and are not valid pooled embedding models "
- "yet\n");
- return NULL;
- }
-
- embed_mlx_ctx_t *ctx = calloc(1, sizeof(embed_mlx_ctx_t));
- if (!ctx)
- return NULL;
- ctx->quantize_bits = options.quantize_bits;
- ctx->quantize_group_size = options.quantize_group_size;
-
- if (embed_config_parse(&ctx->config, model_dir) != 0) {
- free(ctx);
- return NULL;
- }
-
- const embed_config_t *c = &ctx->config;
- if (layer_end == -1)
- layer_end = c->n_layers;
- if (layer_start < 0 || layer_start >= layer_end || layer_end > c->n_layers) {
- fprintf(stderr, "mlx: invalid layer range [%d, %d) for %d-layer model\n", layer_start,
- layer_end, c->n_layers);
- free(ctx);
- return NULL;
- }
- ctx->layer_start = layer_start;
- ctx->layer_end = layer_end;
- int h = c->hidden_size;
-
- ctx->ms = multi_safetensors_open(model_dir);
- if (!ctx->ms) {
- fprintf(stderr, "mlx: failed to open safetensors in %s\n", model_dir);
- free(ctx);
- return NULL;
- }
-
- if (c->family == EMBED_FAMILY_BERT) {
- if (load_mlx_bert_weights(ctx, layer_start, layer_end) != 0)
- goto fail;
- goto loaded;
- }
-
- const char *weight_prefix = multi_safetensors_weight_prefix(ctx->ms, "embed_tokens.weight");
- if (!weight_prefix) {
- fprintf(stderr, "mlx: neither embed_tokens.weight nor "
- "model.embed_tokens.weight was found\n");
- goto fail;
- }
-
- int qd = c->q_dim, kvd = c->kv_dim;
- int inter = c->intermediate_size, hd = c->head_dim;
- char name[256];
-
- /* Only the first stage needs token embeddings. */
- if (layer_start == 0) {
- int emb_shape[] = {c->vocab_size, h};
- snprintf(name, sizeof(name), "%sembed_tokens.weight", weight_prefix);
- ctx->embed_tokens = load_tensor(ctx->ms, name, emb_shape, 2);
- if (!arr_ok(ctx->embed_tokens))
- goto fail;
- }
-
- /* Layers */
- ctx->layers = calloc(c->n_layers, sizeof(mlx_layer_t));
- if (!ctx->layers)
- goto fail;
-
- for (int i = layer_start; i < layer_end; i++) {
- mlx_layer_t *l = &ctx->layers[i];
-
-#define LD(fld, fmt, ...) \
- do { \
- snprintf(name, sizeof(name), fmt, weight_prefix, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
- if (!linear_ok(l->fld)) \
- goto fail; \
- } while (0)
-
- LD(wq, "%slayers.%d.self_attn.q_proj.weight", qd, h);
- LD(wk, "%slayers.%d.self_attn.k_proj.weight", kvd, h);
- LD(wv, "%slayers.%d.self_attn.v_proj.weight", kvd, h);
- LD(wo, "%slayers.%d.self_attn.o_proj.weight", h, qd);
-#undef LD
-
-#define LD(fld, fmt, ...) \
- do { \
- snprintf(name, sizeof(name), fmt, weight_prefix, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
- if (!arr_ok(l->fld)) \
- goto fail; \
- } while (0)
-
- if (c->qkv_bias) {
- LD(q_bias, "%slayers.%d.self_attn.q_proj.bias", qd);
- LD(k_bias, "%slayers.%d.self_attn.k_proj.bias", kvd);
- LD(v_bias, "%slayers.%d.self_attn.v_proj.bias", kvd);
- }
- if (c->qk_norm) {
- LD(q_norm, "%slayers.%d.self_attn.q_norm.weight", hd);
- LD(k_norm, "%slayers.%d.self_attn.k_norm.weight", hd);
- }
- LD(input_norm, "%slayers.%d.input_layernorm.weight", h);
- LD(post_attn_norm, "%slayers.%d.post_attention_layernorm.weight", h);
-#undef LD
-
-#define LD(fld, fmt, ...) \
- do { \
- snprintf(name, sizeof(name), fmt, weight_prefix, i); \
- int sh[] = {__VA_ARGS__}; \
- l->fld = load_linear_tensor(ctx->ms, name, sh, sizeof(sh) / sizeof(sh[0])); \
- if (!linear_ok(l->fld)) \
- goto fail; \
- } while (0)
-
- LD(gate_proj, "%slayers.%d.mlp.gate_proj.weight", inter, h);
- LD(up_proj, "%slayers.%d.mlp.up_proj.weight", inter, h);
- LD(down_proj, "%slayers.%d.mlp.down_proj.weight", h, inter);
-#undef LD
-
- if (embed_verbose >= 2)
- fprintf(stderr, " mlx layer %d loaded\n", i);
- }
-
- if (layer_end == c->n_layers) {
- int ns[] = {h};
- snprintf(name, sizeof(name), "%snorm.weight", weight_prefix);
- ctx->norm = load_tensor(ctx->ms, name, ns, 1);
- if (!arr_ok(ctx->norm))
- goto fail;
- }
-
-loaded:
- ctx->stream = mlx_default_gpu_stream_new();
- ctx->weight_dtype = mlx_array_dtype(ctx->layers[layer_start].wq.w);
- /* Non-quantized BF16: run RMSNorm and the residual stream in F32. Upcast the
- * tiny norm vectors so they match the F32 activations in mlx_forward_layers;
- * the large matmul weights stay BF16. Q8 keeps its existing BF16 norms. */
- if (ctx->weight_dtype == MLX_BFLOAT16 && !ctx->quantize_bits) {
- for (int i = layer_start; i < layer_end; i++) {
- mlx_layer_t *l = &ctx->layers[i];
- upcast_f32_inplace(&l->q_bias, ctx->stream);
- upcast_f32_inplace(&l->k_bias, ctx->stream);
- upcast_f32_inplace(&l->v_bias, ctx->stream);
- upcast_f32_inplace(&l->q_norm, ctx->stream);
- upcast_f32_inplace(&l->k_norm, ctx->stream);
- upcast_f32_inplace(&l->input_norm, ctx->stream);
- upcast_f32_inplace(&l->post_attn_norm, ctx->stream);
- /* BERT block biases (no-ops for Qwen, whose arrays are unset); the
- * two BERT LayerNorm weights reuse input_norm/post_attn_norm above. */
- upcast_f32_inplace(&l->o_bias, ctx->stream);
- upcast_f32_inplace(&l->attn_ln_bias, ctx->stream);
- upcast_f32_inplace(&l->ffn_inter_bias, ctx->stream);
- upcast_f32_inplace(&l->ffn_out_bias, ctx->stream);
- upcast_f32_inplace(&l->ffn_ln_bias, ctx->stream);
- }
- upcast_f32_inplace(&ctx->norm, ctx->stream);
- /* BERT embedding stage (no-ops for Qwen): learned position and token-type
- * embeddings and the embedding LayerNorm all feed the F32 residual. */
- upcast_f32_inplace(&ctx->position_embeddings, ctx->stream);
- upcast_f32_inplace(&ctx->token_type_embeddings, ctx->stream);
- upcast_f32_inplace(&ctx->embed_ln_w, ctx->stream);
- upcast_f32_inplace(&ctx->embed_ln_b, ctx->stream);
- }
- if (ctx->quantize_bits) {
- for (int i = layer_start; i < layer_end; i++) {
- if (quantize_layer(&ctx->layers[i], ctx->quantize_bits, ctx->quantize_group_size,
- ctx->stream) != 0)
- goto fail;
- }
- mlx_clear_cache();
- if (embed_verbose >= 1)
- fprintf(stderr,
- "mlx: quantized linear weights to %d-bit "
- "(group_size=%d)\n",
- ctx->quantize_bits, ctx->quantize_group_size);
- } else {
- for (int i = layer_start; i < layer_end; i++) {
- if (prepare_layer_transposes(&ctx->layers[i], ctx->stream) != 0)
- goto fail;
- }
- }
-
- if (embed_verbose >= 2) {
- size_t active = 0, cache = 0, peak = 0;
- mlx_get_active_memory(&active);
- mlx_get_cache_memory(&cache);
- mlx_get_peak_memory(&peak);
- fprintf(stderr,
- "mlx: memory active=%.2f GiB cache=%.2f GiB "
- "peak=%.2f GiB\n",
- (double)active / (1024.0 * 1024.0 * 1024.0),
- (double)cache / (1024.0 * 1024.0 * 1024.0),
- (double)peak / (1024.0 * 1024.0 * 1024.0));
- }
-
- if (embed_verbose >= 1)
- fprintf(stderr, "mlx: layers [%d, %d) loaded (%d-dim)\n", layer_start, layer_end, h);
-
- return ctx;
-
-fail:
- embed_mlx_free(ctx);
- return NULL;
-}
-
-embed_mlx_ctx_t *embed_mlx_load(const char *model_dir) {
- return mlx_load_range_ex(model_dir, 0, -1, NULL, 0);
-}
-
-embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts) {
- return mlx_load_range_ex(model_dir, 0, -1, opts, 0);
-}
-
-/* ========================================================================
- * Free
- * ======================================================================== */
-
-static void free_mlx_layer(mlx_layer_t *l) {
- free_linear(&l->wq);
- free_linear(&l->wk);
- free_linear(&l->wv);
- free_linear(&l->wo);
- mlx_array_free(l->q_bias);
- mlx_array_free(l->k_bias);
- mlx_array_free(l->v_bias);
- mlx_array_free(l->q_norm);
- mlx_array_free(l->k_norm);
- mlx_array_free(l->input_norm);
- mlx_array_free(l->post_attn_norm);
- free_linear(&l->gate_proj);
- free_linear(&l->up_proj);
- free_linear(&l->down_proj);
- mlx_array_free(l->o_bias);
- mlx_array_free(l->attn_ln_bias);
- mlx_array_free(l->ffn_inter_bias);
- mlx_array_free(l->ffn_out_bias);
- mlx_array_free(l->ffn_ln_bias);
-}
-
-void embed_mlx_free(embed_mlx_ctx_t *ctx) {
- if (!ctx)
- return;
- mlx_array_free(ctx->embed_tokens);
- if (ctx->layers) {
- for (int i = 0; i < ctx->config.n_layers; i++)
- free_mlx_layer(&ctx->layers[i]);
- free(ctx->layers);
- }
- mlx_array_free(ctx->norm);
- mlx_array_free(ctx->position_embeddings);
- mlx_array_free(ctx->token_type_embeddings);
- mlx_array_free(ctx->embed_ln_w);
- mlx_array_free(ctx->embed_ln_b);
- if (ctx->stream.ctx)
- mlx_stream_free(ctx->stream);
- if (ctx->ms)
- multi_safetensors_close(ctx->ms);
- free(ctx);
-}
-
-embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts) {
- if (!mlx_model_dir_has_late_projection(model_dir)) {
- fprintf(stderr, "mlx late: missing 1_Dense projection\n");
- return NULL;
- }
-
- embed_mlx_late_ctx_t *late = calloc(1, sizeof(*late));
- if (!late)
- return NULL;
-
- late->base = mlx_load_range_ex(model_dir, 0, -1, opts, 1);
- if (!late->base)
- goto fail;
-
- char dense_dir[1024];
- int n = snprintf(dense_dir, sizeof(dense_dir), "%s/1_Dense", model_dir);
- if (n < 0 || (size_t)n >= sizeof(dense_dir)) {
- fprintf(stderr, "mlx late: model path too long\n");
- goto fail;
- }
-
- late->dense_ms = multi_safetensors_open(dense_dir);
- if (!late->dense_ms) {
- fprintf(stderr, "mlx late: failed to open safetensors in %s\n", dense_dir);
- goto fail;
- }
-
- const safetensor_t *t = multi_safetensors_find(late->dense_ms, "linear.weight", NULL);
- if (!t) {
- fprintf(stderr, "mlx late: missing linear.weight\n");
- goto fail;
- }
- if (t->ndim != 2 || t->shape[0] <= 0 || t->shape[0] > INT_MAX ||
- t->shape[1] != late->base->config.hidden_size) {
- fprintf(stderr, "mlx late: bad projection shape\n");
- goto fail;
- }
-
- int shape[] = {(int)t->shape[0], late->base->config.hidden_size};
- late->projection = load_linear_tensor(late->dense_ms, "linear.weight", shape, 2);
- if (!linear_ok(late->projection))
- goto fail;
- if (prepare_linear_transpose(&late->projection, late->base->stream) != 0)
- goto fail;
- late->token_dim = shape[0];
-
- if (embed_verbose >= 1)
- fprintf(stderr, "mlx late: token projection loaded (%d-dim)\n", late->token_dim);
- return late;
-
-fail:
- embed_mlx_late_free(late);
- return NULL;
-}
-
-embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir) {
- return embed_mlx_late_load_with_options(model_dir, NULL);
-}
-
-void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx) {
- if (!ctx)
- return;
- free_linear(&ctx->projection);
- if (ctx->dense_ms)
- multi_safetensors_close(ctx->dense_ms);
- embed_mlx_free(ctx->base);
- free(ctx);
-}
-
-const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx) {
- return ctx && ctx->base ? &ctx->base->config : NULL;
-}
-
-int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx) { return ctx ? ctx->token_dim : 0; }
-
-/* ========================================================================
- * Forward pass
- * ======================================================================== */
-
-/* Exact erf GeLU: 0.5 * x * (1 + erf(x / sqrt(2))). Frees nothing it is given. */
-static mlx_array mlx_gelu_erf(mlx_array x, mlx_stream S) {
- mlx_array c = mlx_array_new_float32(0.70710678118654752f);
- mlx_array scaled = mlx_array_new();
- mlx_multiply(&scaled, x, c, S);
- mlx_array_free(c);
- mlx_array e = mlx_array_new();
- mlx_erf(&e, scaled, S);
- mlx_array_free(scaled);
- mlx_array one = mlx_array_new_float32(1.0f);
- mlx_array e1 = mlx_array_new();
- mlx_add(&e1, e, one, S);
- mlx_array_free(e);
- mlx_array_free(one);
- mlx_array half = mlx_array_new_float32(0.5f);
- mlx_array xh = mlx_array_new();
- mlx_multiply(&xh, x, half, S);
- mlx_array_free(half);
- mlx_array g = mlx_array_new();
- mlx_multiply(&g, xh, e1, S);
- mlx_array_free(xh);
- mlx_array_free(e1);
- return g;
-}
-
-/* Tanh-approximation GeLU (gelu_new / gelu_pytorch_tanh):
- * 0.5 * x * (1 + tanh(sqrt(2/pi) * (x + 0.044715 * x^3))). Frees nothing given. */
-static mlx_array mlx_gelu_tanh(mlx_array x, mlx_stream S) {
- mlx_array x2 = mlx_array_new();
- mlx_multiply(&x2, x, x, S);
- mlx_array x3 = mlx_array_new();
- mlx_multiply(&x3, x2, x, S);
- mlx_array_free(x2);
- mlx_array c = mlx_array_new_float32(0.044715f);
- mlx_array cx3 = mlx_array_new();
- mlx_multiply(&cx3, x3, c, S);
- mlx_array_free(x3);
- mlx_array_free(c);
- mlx_array sum = mlx_array_new();
- mlx_add(&sum, x, cx3, S);
- mlx_array_free(cx3);
- mlx_array s2pi = mlx_array_new_float32(0.79788456080286536f);
- mlx_array inner = mlx_array_new();
- mlx_multiply(&inner, sum, s2pi, S);
- mlx_array_free(sum);
- mlx_array_free(s2pi);
- mlx_array t = mlx_array_new();
- mlx_tanh(&t, inner, S);
- mlx_array_free(inner);
- mlx_array one = mlx_array_new_float32(1.0f);
- mlx_array t1 = mlx_array_new();
- mlx_add(&t1, t, one, S);
- mlx_array_free(t);
- mlx_array_free(one);
- mlx_array half = mlx_array_new_float32(0.5f);
- mlx_array xh = mlx_array_new();
- mlx_multiply(&xh, x, half, S);
- mlx_array_free(half);
- mlx_array g = mlx_array_new();
- mlx_multiply(&g, xh, t1, S);
- mlx_array_free(xh);
- mlx_array_free(t1);
- return g;
-}
-
-/* BERT-family forward: post-norm encoder layers (bias on every projection, GeLU
- * feed-forward, no RoPE, no per-head norm) over an already-embedded x. Frees the
- * x it is given and returns the new hidden states. */
-static mlx_array mlx_forward_bert(embed_mlx_ctx_t *ctx,
- mlx_array x,
- int batch,
- int max_seq,
- int has_padding,
- mlx_array attn_mask,
- int layer_start,
- int layer_end) {
- const embed_config_t *c = &ctx->config;
- mlx_stream S = ctx->stream;
- int n_heads = c->n_heads, head_dim = c->head_dim, q_dim = c->q_dim;
- float scale = 1.0f / sqrtf((float)head_dim);
- float eps = c->layer_norm_eps;
- mlx_array null_arr = (mlx_array){0};
-
- for (int layer = layer_start; layer < layer_end; layer++) {
- mlx_layer_t *l = &ctx->layers[layer];
-
- /* Self-attention reads x directly (post-norm). */
- mlx_array q_flat = linear(x, &l->wq, S);
- mlx_array k_flat = linear(x, &l->wk, S);
- mlx_array v_flat = linear(x, &l->wv, S);
- mlx_array t = mlx_array_new();
- mlx_add(&t, q_flat, l->q_bias, S);
- mlx_array_free(q_flat);
- q_flat = t;
- t = mlx_array_new();
- mlx_add(&t, k_flat, l->k_bias, S);
- mlx_array_free(k_flat);
- k_flat = t;
- t = mlx_array_new();
- mlx_add(&t, v_flat, l->v_bias, S);
- mlx_array_free(v_flat);
- v_flat = t;
-
- int s4[] = {batch, max_seq, n_heads, head_dim};
- mlx_array q = mlx_array_new(), k = mlx_array_new(), v = mlx_array_new();
- mlx_reshape(&q, q_flat, s4, 4, S);
- mlx_reshape(&k, k_flat, s4, 4, S);
- mlx_reshape(&v, v_flat, s4, 4, S);
- mlx_array_free(q_flat);
- mlx_array_free(k_flat);
- mlx_array_free(v_flat);
-
- int perm[] = {0, 2, 1, 3};
- mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new();
- mlx_transpose_axes(&qt, q, perm, 4, S);
- mlx_transpose_axes(&kt, k, perm, 4, S);
- mlx_transpose_axes(&vt, v, perm, 4, S);
- mlx_array_free(q);
- mlx_array_free(k);
- mlx_array_free(v);
-
- mlx_array attn = mlx_array_new();
- mlx_fast_scaled_dot_product_attention(&attn, qt, kt, vt, scale, "",
- has_padding ? attn_mask : null_arr, null_arr, S);
- mlx_array_free(qt);
- mlx_array_free(kt);
- mlx_array_free(vt);
-
- mlx_array attn_t = mlx_array_new();
- mlx_transpose_axes(&attn_t, attn, perm, 4, S);
- mlx_array_free(attn);
- int s3[] = {batch, max_seq, q_dim};
- mlx_array attn_flat = mlx_array_new();
- mlx_reshape(&attn_flat, attn_t, s3, 3, S);
- mlx_array_free(attn_t);
-
- mlx_array proj = linear(attn_flat, &l->wo, S);
- mlx_array_free(attn_flat);
- mlx_array proj_b = mlx_array_new();
- mlx_add(&proj_b, proj, l->o_bias, S);
- mlx_array_free(proj);
- mlx_array res = mlx_array_new();
- mlx_add(&res, x, proj_b, S);
- mlx_array_free(x);
- mlx_array_free(proj_b);
- x = mlx_array_new();
- mlx_fast_layer_norm(&x, res, l->input_norm, l->attn_ln_bias, eps, S);
- mlx_array_free(res);
-
- /* Feed-forward: dense -> GeLU -> dense. */
- mlx_array mid = linear(x, &l->up_proj, S);
- mlx_array mid_b = mlx_array_new();
- mlx_add(&mid_b, mid, l->ffn_inter_bias, S);
- mlx_array_free(mid);
- mlx_array g = ctx->config.ffn_act == EMBED_ACT_GELU_TANH ? mlx_gelu_tanh(mid_b, S)
- : mlx_gelu_erf(mid_b, S);
- mlx_array_free(mid_b);
- mlx_array out = linear(g, &l->down_proj, S);
- mlx_array_free(g);
- mlx_array out_b = mlx_array_new();
- mlx_add(&out_b, out, l->ffn_out_bias, S);
- mlx_array_free(out);
- mlx_array res2 = mlx_array_new();
- mlx_add(&res2, x, out_b, S);
- mlx_array_free(x);
- mlx_array_free(out_b);
- x = mlx_array_new();
- mlx_fast_layer_norm(&x, res2, l->post_attn_norm, l->ffn_ln_bias, eps, S);
- mlx_array_free(res2);
- }
- return x;
-}
-
-static mlx_array mlx_forward_layers(embed_mlx_ctx_t *ctx,
- mlx_array x,
- int batch,
- int max_seq,
- int has_padding,
- mlx_array attn_mask,
- int layer_start,
- int layer_end) {
- const embed_config_t *c = &ctx->config;
- mlx_stream S = ctx->stream;
- int n_heads = c->n_heads;
- int n_kv_heads = c->n_kv_heads;
- int head_dim = c->head_dim;
- int q_dim = c->q_dim;
- float scale = 1.0f / sqrtf((float)head_dim);
- mlx_array null_arr = (mlx_array){0};
- const char *mask_mode = c->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "";
-
- /* For non-quantized BF16 weights, run the residual stream, norms, and
- * attention in F32; matmuls downcast their operands to BF16 internally
- * (see linear). This keeps weight storage at BF16 while avoiding the
- * cross-layer accumulation error of a fully BF16 forward. F32 and quantized
- * models are unaffected. */
- mlx_array local_mask = attn_mask;
- int free_mask = 0;
- int mixed_bf16 =
- mlx_weight_dtype(ctx) == MLX_BFLOAT16 && !ctx->layers[layer_start].wq.quantized;
- if (mixed_bf16) {
- if (mlx_array_dtype(x) != MLX_FLOAT32) {
- mlx_array xf = mlx_array_new();
- mlx_astype(&xf, x, MLX_FLOAT32, S);
- mlx_array_free(x);
- x = xf;
- }
- if (has_padding && arr_ok(attn_mask) && mlx_array_dtype(attn_mask) != MLX_FLOAT32) {
- local_mask = mlx_array_new();
- mlx_astype(&local_mask, attn_mask, MLX_FLOAT32, S);
- free_mask = 1;
- }
- }
-
- for (int layer = layer_start; layer < layer_end; layer++) {
- mlx_layer_t *l = &ctx->layers[layer];
-
- mlx_array xn = mlx_array_new();
- mlx_fast_rms_norm(&xn, x, l->input_norm, c->rms_norm_eps, S);
-
- mlx_array q_flat = linear(xn, &l->wq, S);
- mlx_array k_flat = linear(xn, &l->wk, S);
- mlx_array v_flat = linear(xn, &l->wv, S);
- mlx_array_free(xn);
- if (c->qkv_bias) {
- mlx_array biased = mlx_array_new();
- mlx_add(&biased, q_flat, l->q_bias, S);
- mlx_array_free(q_flat);
- q_flat = biased;
- biased = mlx_array_new();
- mlx_add(&biased, k_flat, l->k_bias, S);
- mlx_array_free(k_flat);
- k_flat = biased;
- biased = mlx_array_new();
- mlx_add(&biased, v_flat, l->v_bias, S);
- mlx_array_free(v_flat);
- v_flat = biased;
- }
-
- int q_shape[] = {batch, max_seq, n_heads, head_dim};
- int k_shape[] = {batch, max_seq, n_kv_heads, head_dim};
- mlx_array q = mlx_array_new();
- mlx_array k = mlx_array_new();
- mlx_array v = mlx_array_new();
- mlx_reshape(&q, q_flat, q_shape, 4, S);
- mlx_reshape(&k, k_flat, k_shape, 4, S);
- mlx_reshape(&v, v_flat, k_shape, 4, S);
- mlx_array_free(q_flat);
- mlx_array_free(k_flat);
- mlx_array_free(v_flat);
-
- mlx_array qn = q;
- mlx_array kn = k;
- if (c->qk_norm) {
- qn = mlx_array_new();
- kn = mlx_array_new();
- mlx_fast_rms_norm(&qn, q, l->q_norm, c->rms_norm_eps, S);
- mlx_fast_rms_norm(&kn, k, l->k_norm, c->rms_norm_eps, S);
- mlx_array_free(q);
- mlx_array_free(k);
- }
-
- /* [B, T, heads, D] -> [B, heads, T, D] for RoPE + SDPA. */
- int perm[] = {0, 2, 1, 3};
- mlx_array qt = mlx_array_new(), kt = mlx_array_new(), vt = mlx_array_new();
- mlx_transpose_axes(&qt, qn, perm, 4, S);
- mlx_transpose_axes(&kt, kn, perm, 4, S);
- mlx_transpose_axes(&vt, v, perm, 4, S);
- mlx_array_free(qn);
- mlx_array_free(kn);
- mlx_array_free(v);
-
- mlx_optional_float base = {.value = c->rope_theta, .has_value = true};
- mlx_array qr = mlx_array_new(), kr = mlx_array_new();
- mlx_fast_rope(&qr, qt, head_dim, false, base, 1.0f, 0, null_arr, S);
- mlx_fast_rope(&kr, kt, head_dim, false, base, 1.0f, 0, null_arr, S);
- mlx_array_free(qt);
- mlx_array_free(kt);
-
- mlx_array attn = mlx_array_new();
- mlx_fast_scaled_dot_product_attention(&attn, qr, kr, vt, scale, mask_mode,
- has_padding ? local_mask : null_arr, null_arr, S);
- mlx_array_free(qr);
- mlx_array_free(kr);
- mlx_array_free(vt);
-
- mlx_array attn_t = mlx_array_new();
- mlx_transpose_axes(&attn_t, attn, perm, 4, S);
- mlx_array_free(attn);
-
- int flat_shape[] = {batch, max_seq, q_dim};
- mlx_array attn_flat = mlx_array_new();
- mlx_reshape(&attn_flat, attn_t, flat_shape, 3, S);
- mlx_array_free(attn_t);
-
- mlx_array proj = linear(attn_flat, &l->wo, S);
- mlx_array_free(attn_flat);
- mlx_array x2 = mlx_array_new();
- mlx_add(&x2, x, proj, S);
- mlx_array_free(x);
- mlx_array_free(proj);
- x = x2;
-
- mlx_array xn2 = mlx_array_new();
- mlx_fast_rms_norm(&xn2, x, l->post_attn_norm, c->rms_norm_eps, S);
-
- mlx_array gate = linear(xn2, &l->gate_proj, S);
- mlx_array up = linear(xn2, &l->up_proj, S);
- mlx_array_free(xn2);
-
- mlx_array gate_sig = mlx_array_new();
- mlx_sigmoid(&gate_sig, gate, S);
- mlx_array silu_gate = mlx_array_new();
- mlx_multiply(&silu_gate, gate, gate_sig, S);
- mlx_array_free(gate);
- mlx_array_free(gate_sig);
-
- mlx_array mid = mlx_array_new();
- mlx_multiply(&mid, silu_gate, up, S);
- mlx_array_free(silu_gate);
- mlx_array_free(up);
-
- mlx_array ffn = linear(mid, &l->down_proj, S);
- mlx_array_free(mid);
-
- mlx_array x3 = mlx_array_new();
- mlx_add(&x3, x, ffn, S);
- mlx_array_free(x);
- mlx_array_free(ffn);
- x = x3;
- }
- if (free_mask)
- mlx_array_free(local_mask);
- return x;
-}
-
-embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize) {
- if (!ctx || !ctx->base || !token_ids || n_tokens <= 0 || ctx->token_dim <= 0 ||
- !linear_ok(ctx->projection))
- return NULL;
-
- embed_mlx_ctx_t *base = ctx->base;
- if (base->layer_start != 0 || base->layer_end != base->config.n_layers ||
- !arr_ok(base->embed_tokens) || !arr_ok(base->norm))
- return NULL;
-
- const embed_config_t *c = &base->config;
- mlx_stream S = base->stream;
- for (int i = 0; i < n_tokens; i++) {
- if (token_ids[i] < 0 || token_ids[i] >= c->vocab_size) {
- fprintf(stderr, "mlx late: invalid token id %d at position %d\n", token_ids[i], i);
- return NULL;
- }
- }
-
- size_t id_bytes;
- if (mlx_mul_size((size_t)n_tokens, sizeof(int), &id_bytes) != 0)
- return NULL;
-
- int *ids_data = (int *)malloc(id_bytes);
- if (!ids_data)
- return NULL;
- memcpy(ids_data, token_ids, id_bytes);
-
- int ids_shape[] = {1, n_tokens};
- mlx_array ids = mlx_array_new_data(ids_data, ids_shape, 2, MLX_INT32);
- free(ids_data);
- if (!arr_ok(ids))
- return NULL;
-
- embed_mlx_late_vectors_t *result = NULL;
- mlx_array x = mlx_array_new();
- mlx_array x_normed = (mlx_array){0};
- mlx_array projected = (mlx_array){0};
- mlx_array reshaped = (mlx_array){0};
-
- mlx_take_axis(&x, base->embed_tokens, ids, 0, S);
- mlx_array_free(ids);
- if (!arr_ok(x))
- goto cleanup;
-
- x = mlx_forward_layers(base, x, 1, n_tokens, 0, (mlx_array){0}, 0, c->n_layers);
- if (!arr_ok(x))
- goto cleanup;
-
- x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S);
- mlx_array_free(x);
- x = (mlx_array){0};
- if (!arr_ok(x_normed))
- goto cleanup;
-
- projected = linear(x_normed, &ctx->projection, S);
- mlx_array_free(x_normed);
- x_normed = (mlx_array){0};
- if (!arr_ok(projected))
- goto cleanup;
-
- if (normalize) {
- mlx_array sq = mlx_array_new();
- mlx_square(&sq, projected, S);
- if (!arr_ok(sq)) {
- mlx_array_free(sq);
- goto cleanup;
- }
-
- mlx_array norm2 = mlx_array_new();
- mlx_sum_axis(&norm2, sq, 2, true, S);
- mlx_array_free(sq);
- if (!arr_ok(norm2)) {
- mlx_array_free(norm2);
- goto cleanup;
- }
-
- mlx_array inv_norm = mlx_array_new();
- mlx_rsqrt(&inv_norm, norm2, S);
- mlx_array_free(norm2);
- if (!arr_ok(inv_norm)) {
- mlx_array_free(inv_norm);
- goto cleanup;
- }
-
- mlx_array normalized = mlx_array_new();
- mlx_multiply(&normalized, projected, inv_norm, S);
- mlx_array_free(inv_norm);
- mlx_array_free(projected);
- projected = normalized;
- if (!arr_ok(projected))
- goto cleanup;
- }
-
- int vec_shape[] = {n_tokens, ctx->token_dim};
- reshaped = mlx_array_new();
- mlx_reshape(&reshaped, projected, vec_shape, 2, S);
- if (!arr_ok(reshaped))
- goto cleanup;
-
- result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result));
- if (!result)
- goto cleanup;
- result->owner = ctx;
- result->vectors = reshaped;
- result->tokens = n_tokens;
- result->dim = ctx->token_dim;
- reshaped = (mlx_array){0};
-
-cleanup:
- mlx_array_free(reshaped);
- mlx_array_free(projected);
- mlx_array_free(x_normed);
- mlx_array_free(x);
- return result;
-}
-
-void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs) {
- if (!vecs)
- return;
- mlx_array_free(vecs->vectors);
- free(vecs);
-}
-
-int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs) {
- return vecs ? vecs->tokens : 0;
-}
-
-int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs) {
- return vecs ? vecs->dim : 0;
-}
-
-int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors) {
- if (!vecs || !out_vectors || !arr_ok(vecs->vectors) || vecs->tokens <= 0 || vecs->dim <= 0)
- return -1;
-
- size_t values, bytes;
- if (mlx_mul_size((size_t)vecs->tokens, (size_t)vecs->dim, &values) != 0 ||
- mlx_mul_size(values, sizeof(float), &bytes) != 0)
- return -1;
-
- mlx_stream S = vecs->owner->base->stream;
- mlx_array out_f32 = mlx_array_new();
- mlx_astype(&out_f32, vecs->vectors, MLX_FLOAT32, S);
- if (!arr_ok(out_f32)) {
- mlx_array_free(out_f32);
- return -1;
- }
-
- int rc = -1;
- mlx_array_eval(out_f32);
- const float *data = mlx_array_data_float32(out_f32);
- if (data) {
- memcpy(out_vectors, data, bytes);
- rc = 0;
- }
- mlx_array_free(out_f32);
- return rc;
-}
-
-/*
- * Batched late-interaction document encoder.
- *
- * Encodes every candidate in one padded transformer pass, then gathers each
- * document's kept token vectors into a single packed [total_keep, token_dim]
- * array in document order. out_offsets receives the prefix sum of kept counts
- * (out_offsets[i + 1] - out_offsets[i] == n_keep[i]), the exact layout
- * embed_mlx_late_maxsim_batch_device consumes. This collapses N per-candidate
- * transformer calls into one forward.
- *
- * Right-padding keeps real tokens at positions 0..n_tokens[d]-1 so RoPE
- * positions are unchanged; a bidirectional model gets a padding attention mask
- * so real tokens never attend to pad slots (the same masking the dense pooled
- * batch path uses). Kept-token vectors therefore match per-document encoding to
- * within MLX's batched-GEMM rounding.
- */
-embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t *ctx,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- int *out_offsets) {
- if (!ctx || !ctx->base || !doc_ids || !n_tokens || !keep || !n_keep || !out_offsets ||
- n_docs <= 0 || ctx->token_dim <= 0 || !linear_ok(ctx->projection))
- return NULL;
-
- embed_mlx_ctx_t *base = ctx->base;
- if (base->layer_start != 0 || base->layer_end != base->config.n_layers ||
- !arr_ok(base->embed_tokens) || !arr_ok(base->norm))
- return NULL;
-
- const embed_config_t *c = &base->config;
- mlx_stream S = base->stream;
- int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
-
- /* Validate inputs and size the batch: longest sequence and total kept
- * tokens. out_offsets is the running prefix sum of kept counts. */
- int max_seq = 0;
- int total_keep = 0;
- out_offsets[0] = 0;
- for (int d = 0; d < n_docs; d++) {
- if (!doc_ids[d] || n_tokens[d] <= 0 || !keep[d] || n_keep[d] <= 0 ||
- n_keep[d] > n_tokens[d] || total_keep > INT_MAX - n_keep[d])
- return NULL;
- if (n_tokens[d] > max_seq)
- max_seq = n_tokens[d];
- for (int k = 0; k < n_keep[d]; k++) {
- if (keep[d][k] < 0 || keep[d][k] >= n_tokens[d])
- return NULL;
- }
- total_keep += n_keep[d];
- out_offsets[d + 1] = total_keep;
- }
-
- /* The gather index lives in an MLX int32 array, so the flattened
- * [n_docs * max_seq, dim] row count must stay within int range. */
- size_t elems, ids_bytes, mask_bytes, gather_bytes;
- if (mlx_mul_size((size_t)n_docs, (size_t)max_seq, &elems) != 0 || elems > INT_MAX ||
- mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 ||
- mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 ||
- mlx_mul_size((size_t)total_keep, sizeof(int), &gather_bytes) != 0)
- return NULL;
-
- int has_padding = 0;
- for (int d = 0; d < n_docs; d++)
- if (n_tokens[d] != max_seq)
- has_padding = 1;
- int needs_attn_mask = has_padding && !causal;
- if (ids_bytes == 0 || gather_bytes == 0 || (needs_attn_mask && mask_bytes == 0))
- return NULL;
-
- int *padded_ids = (int *)malloc(ids_bytes);
- float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL;
- int *gather_idx = (int *)malloc(gather_bytes);
- if (!padded_ids || (needs_attn_mask && !attn_mask_data) || !gather_idx) {
- free(padded_ids);
- free(attn_mask_data);
- free(gather_idx);
- return NULL;
- }
-
- int gpos = 0;
- for (int d = 0; d < n_docs; d++) {
- for (int t = 0; t < max_seq; t++) {
- size_t idx = (size_t)d * max_seq + t;
- if (t < n_tokens[d]) {
- int id = doc_ids[d][t];
- if (id < 0 || id >= c->vocab_size) {
- fprintf(stderr, "mlx late: invalid token id %d at doc %d position %d\n", id, d,
- t);
- free(padded_ids);
- free(attn_mask_data);
- free(gather_idx);
- return NULL;
- }
- padded_ids[idx] = id;
- if (needs_attn_mask)
- attn_mask_data[idx] = 0.0f;
- } else {
- padded_ids[idx] = 0;
- if (needs_attn_mask)
- attn_mask_data[idx] = -1.0e9f;
- }
- }
- /* Map each kept token to its row in the flattened projection. */
- for (int k = 0; k < n_keep[d]; k++)
- gather_idx[gpos++] = d * max_seq + keep[d][k];
- }
-
- int ids_shape[] = {n_docs, max_seq};
- int attn_mask_shape[] = {n_docs, 1, 1, max_seq};
- int gather_shape[] = {total_keep};
- mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
- mlx_array attn_mask = needs_attn_mask
- ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
- mlx_array gather = mlx_array_new_data(gather_idx, gather_shape, 1, MLX_INT32);
- free(padded_ids);
- free(attn_mask_data);
- free(gather_idx);
- if (!arr_ok(ids) || (needs_attn_mask && !arr_ok(attn_mask)) || !arr_ok(gather)) {
- mlx_array_free(ids);
- mlx_array_free(attn_mask);
- mlx_array_free(gather);
- return NULL;
- }
- if (needs_attn_mask && mlx_weight_dtype(base) == MLX_BFLOAT16) {
- mlx_array attn_mask_bf16 = mlx_array_new();
- mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
- mlx_array_free(attn_mask);
- attn_mask = attn_mask_bf16;
- if (!arr_ok(attn_mask)) {
- mlx_array_free(ids);
- mlx_array_free(gather);
- return NULL;
- }
- }
-
- embed_mlx_late_vectors_t *result = NULL;
- mlx_array x = mlx_array_new();
- mlx_array x_normed = (mlx_array){0};
- mlx_array projected = (mlx_array){0};
- mlx_array flat = (mlx_array){0};
- mlx_array gathered = (mlx_array){0};
-
- mlx_take_axis(&x, base->embed_tokens, ids, 0, S);
- mlx_array_free(ids);
- if (!arr_ok(x))
- goto cleanup;
-
- x = mlx_forward_layers(base, x, n_docs, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers);
- if (!arr_ok(x))
- goto cleanup;
-
- x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, base->norm, c->rms_norm_eps, S);
- mlx_array_free(x);
- x = (mlx_array){0};
- if (!arr_ok(x_normed))
- goto cleanup;
-
- projected = linear(x_normed, &ctx->projection, S);
- mlx_array_free(x_normed);
- x_normed = (mlx_array){0};
- if (!arr_ok(projected))
- goto cleanup;
-
- /* Flatten [n_docs, max_seq, dim] -> [n_docs * max_seq, dim] and gather the
- * kept tokens in document order into one packed [total_keep, dim] array. */
- int flat_shape[] = {(int)elems, ctx->token_dim};
- flat = mlx_array_new();
- mlx_reshape(&flat, projected, flat_shape, 2, S);
- mlx_array_free(projected);
- projected = (mlx_array){0};
- if (!arr_ok(flat))
- goto cleanup;
-
- gathered = mlx_array_new();
- mlx_take_axis(&gathered, flat, gather, 0, S);
- mlx_array_free(flat);
- flat = (mlx_array){0};
- if (!arr_ok(gathered))
- goto cleanup;
-
- /* Normalize after gathering: per-token L2 is row-independent, so this is
- * identical to normalizing the full batch but skips the dropped and padded
- * rows entirely. */
- if (normalize) {
- mlx_array sq = mlx_array_new();
- mlx_square(&sq, gathered, S);
- if (!arr_ok(sq)) {
- mlx_array_free(sq);
- goto cleanup;
- }
- mlx_array norm2 = mlx_array_new();
- mlx_sum_axis(&norm2, sq, 1, true, S);
- mlx_array_free(sq);
- if (!arr_ok(norm2)) {
- mlx_array_free(norm2);
- goto cleanup;
- }
- mlx_array inv_norm = mlx_array_new();
- mlx_rsqrt(&inv_norm, norm2, S);
- mlx_array_free(norm2);
- if (!arr_ok(inv_norm)) {
- mlx_array_free(inv_norm);
- goto cleanup;
- }
- mlx_array normalized = mlx_array_new();
- mlx_multiply(&normalized, gathered, inv_norm, S);
- mlx_array_free(inv_norm);
- mlx_array_free(gathered);
- gathered = normalized;
- if (!arr_ok(gathered))
- goto cleanup;
- }
-
- result = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*result));
- if (!result)
- goto cleanup;
- result->owner = ctx;
- result->vectors = gathered;
- result->tokens = total_keep;
- result->dim = ctx->token_dim;
- gathered = (mlx_array){0};
-
-cleanup:
- mlx_array_free(gathered);
- mlx_array_free(flat);
- mlx_array_free(projected);
- mlx_array_free(x_normed);
- mlx_array_free(x);
- mlx_array_free(gather);
- mlx_array_free(attn_mask);
- return result;
-}
-
-/* Device-resident select/concat primitives. The server rerank path batches
- * candidates through embed_mlx_late_encode_docs_device, but the late-interaction
- * verification harness (scripts/check_late_interaction.py) drives these directly
- * to validate per-document gather and concat against the CPU MaxSim reference. */
-embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
- embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count) {
- if (!ctx || !ctx->base || !items || count <= 0)
- return NULL;
- int dim = 0;
- int total = 0;
- for (int i = 0; i < count; i++) {
- const embed_mlx_late_vectors_t *v = items[i];
- if (!v || v->owner != ctx || !arr_ok(v->vectors) || v->tokens <= 0 || v->dim <= 0)
- return NULL;
- if (i == 0)
- dim = v->dim;
- if (v->dim != dim || total > INT_MAX - v->tokens)
- return NULL;
- total += v->tokens;
- }
-
- mlx_array *parts = (mlx_array *)malloc((size_t)count * sizeof(*parts));
- if (!parts)
- return NULL;
- for (int i = 0; i < count; i++)
- parts[i] = items[i]->vectors;
-
- mlx_stream S = ctx->base->stream;
- mlx_vector_array vec = mlx_vector_array_new_data(parts, (size_t)count);
- free(parts);
- if (!vec.ctx)
- return NULL;
-
- mlx_array joined = mlx_array_new();
- mlx_concatenate_axis(&joined, vec, 0, S);
- mlx_vector_array_free(vec);
- if (!arr_ok(joined)) {
- mlx_array_free(joined);
- return NULL;
- }
-
- embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
- if (!out) {
- mlx_array_free(joined);
- return NULL;
- }
- out->owner = ctx;
- out->vectors = joined;
- out->tokens = total;
- out->dim = dim;
- return out;
-}
-
-embed_mlx_late_vectors_t *embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *vecs,
- const int *token_indices,
- int count) {
- if (!ctx || !ctx->base || !vecs || vecs->owner != ctx || !arr_ok(vecs->vectors) ||
- !token_indices || count <= 0 || vecs->tokens <= 0 || vecs->dim <= 0)
- return NULL;
-
- for (int i = 0; i < count; i++) {
- if (token_indices[i] < 0 || token_indices[i] >= vecs->tokens)
- return NULL;
- }
-
- size_t idx_bytes;
- if (mlx_mul_size((size_t)count, sizeof(int), &idx_bytes) != 0)
- return NULL;
- int *idx_data = (int *)malloc(idx_bytes);
- if (!idx_data)
- return NULL;
- memcpy(idx_data, token_indices, idx_bytes);
-
- int idx_shape[] = {count};
- mlx_array idx = mlx_array_new_data(idx_data, idx_shape, 1, MLX_INT32);
- free(idx_data);
- if (!arr_ok(idx))
- return NULL;
-
- mlx_stream S = ctx->base->stream;
- mlx_array selected = mlx_array_new();
- mlx_take_axis(&selected, vecs->vectors, idx, 0, S);
- mlx_array_free(idx);
- if (!arr_ok(selected)) {
- mlx_array_free(selected);
- return NULL;
- }
-
- embed_mlx_late_vectors_t *out = (embed_mlx_late_vectors_t *)calloc(1, sizeof(*out));
- if (!out) {
- mlx_array_free(selected);
- return NULL;
- }
- out->owner = ctx;
- out->vectors = selected;
- out->tokens = count;
- out->dim = vecs->dim;
- return out;
-}
-
-int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *query,
- const embed_mlx_late_vectors_t *docs,
- const int *doc_offsets,
- int docs_count,
- float *scores) {
- if (!ctx || !query || !docs || !doc_offsets || docs_count <= 0 || !scores ||
- query->owner != ctx || docs->owner != ctx || query->dim <= 0 || query->dim != docs->dim ||
- query->tokens <= 0 || docs->tokens <= 0 || !arr_ok(query->vectors) ||
- !arr_ok(docs->vectors) || doc_offsets[0] != 0)
- return -1;
-
- int *lengths = (int *)malloc((size_t)docs_count * sizeof(*lengths));
- if (!lengths)
- return -1;
- int max_doc_tokens = 0;
- for (int i = 0; i < docs_count; i++) {
- if (doc_offsets[i] < 0 || doc_offsets[i] >= doc_offsets[i + 1] ||
- doc_offsets[i + 1] > docs->tokens) {
- free(lengths);
- return -1;
- }
- int n = doc_offsets[i + 1] - doc_offsets[i];
- lengths[i] = n;
- if (n > max_doc_tokens)
- max_doc_tokens = n;
- }
- if (max_doc_tokens <= 0) {
- free(lengths);
- return -1;
- }
-
- mlx_stream S = ctx->base->stream;
- int dim = query->dim;
- int rc = -1;
- mlx_array *doc_rows = (mlx_array *)calloc((size_t)docs_count, sizeof(*doc_rows));
- if (!doc_rows) {
- free(lengths);
- return -1;
- }
-
- mlx_vector_array doc_vec = (mlx_vector_array){0};
- mlx_array pad_value = mlx_array_new_float32(0.0f);
- mlx_array doc_batch = (mlx_array){0};
- mlx_array query_3d = (mlx_array){0};
- mlx_array query_batched = (mlx_array){0};
- mlx_array docs_t = (mlx_array){0};
- mlx_array sims = (mlx_array){0};
- mlx_array lengths_arr = (mlx_array){0};
- mlx_array positions = (mlx_array){0};
- mlx_array lengths_2d = (mlx_array){0};
- mlx_array positions_2d = (mlx_array){0};
- mlx_array valid = (mlx_array){0};
- mlx_array valid_3d = (mlx_array){0};
- mlx_array invalid_value = mlx_array_new_float32(-FLT_MAX);
- mlx_array masked_sims = (mlx_array){0};
- mlx_array maxes = (mlx_array){0};
- mlx_array sums = (mlx_array){0};
- mlx_array sums_f32 = (mlx_array){0};
-
- for (int i = 0; i < docs_count; i++) {
- int n = lengths[i];
- int start[] = {doc_offsets[i], 0};
- int stop[] = {doc_offsets[i + 1], dim};
- int strides[] = {1, 1};
-
- mlx_array doc = mlx_array_new();
- mlx_array padded = (mlx_array){0};
-
- mlx_slice(&doc, docs->vectors, start, 2, stop, 2, strides, 2, S);
- if (!arr_ok(doc))
- goto cleanup;
-
- if (n < max_doc_tokens) {
- int axes[] = {0};
- int low[] = {0};
- int high[] = {max_doc_tokens - n};
- padded = mlx_array_new();
- mlx_pad(&padded, doc, axes, 1, low, 1, high, 1, pad_value, "constant", S);
- mlx_array_free(doc);
- doc = padded;
- padded = (mlx_array){0};
- if (!arr_ok(doc))
- goto cleanup;
- }
-
- doc_rows[i] = mlx_array_new();
- mlx_expand_dims(&doc_rows[i], doc, 0, S);
- mlx_array_free(doc);
- if (!arr_ok(doc_rows[i]))
- goto cleanup;
- }
-
- doc_vec = mlx_vector_array_new_data(doc_rows, (size_t)docs_count);
- if (!doc_vec.ctx)
- goto cleanup;
-
- doc_batch = mlx_array_new();
- mlx_concatenate_axis(&doc_batch, doc_vec, 0, S);
- if (!arr_ok(doc_batch))
- goto cleanup;
-
- query_3d = mlx_array_new();
- mlx_expand_dims(&query_3d, query->vectors, 0, S);
- if (!arr_ok(query_3d))
- goto cleanup;
-
- int q_shape[] = {docs_count, query->tokens, dim};
- query_batched = mlx_array_new();
- mlx_broadcast_to(&query_batched, query_3d, q_shape, 3, S);
- if (!arr_ok(query_batched))
- goto cleanup;
-
- int perm[] = {0, 2, 1};
- docs_t = mlx_array_new();
- mlx_transpose_axes(&docs_t, doc_batch, perm, 3, S);
- if (!arr_ok(docs_t))
- goto cleanup;
-
- sims = mlx_array_new();
- mlx_matmul(&sims, query_batched, docs_t, S);
- if (!arr_ok(sims))
- goto cleanup;
-
- int len_shape[] = {docs_count};
- lengths_arr = mlx_array_new_data(lengths, len_shape, 1, MLX_INT32);
- if (!arr_ok(lengths_arr))
- goto cleanup;
-
- positions = mlx_array_new();
- mlx_arange(&positions, 0.0, (double)max_doc_tokens, 1.0, MLX_INT32, S);
- if (!arr_ok(positions))
- goto cleanup;
-
- lengths_2d = mlx_array_new();
- mlx_expand_dims(&lengths_2d, lengths_arr, 1, S);
- if (!arr_ok(lengths_2d))
- goto cleanup;
-
- positions_2d = mlx_array_new();
- mlx_expand_dims(&positions_2d, positions, 0, S);
- if (!arr_ok(positions_2d))
- goto cleanup;
-
- valid = mlx_array_new();
- mlx_less(&valid, positions_2d, lengths_2d, S);
- if (!arr_ok(valid))
- goto cleanup;
-
- valid_3d = mlx_array_new();
- mlx_expand_dims(&valid_3d, valid, 1, S);
- if (!arr_ok(valid_3d))
- goto cleanup;
-
- masked_sims = mlx_array_new();
- mlx_where(&masked_sims, valid_3d, sims, invalid_value, S);
- if (!arr_ok(masked_sims))
- goto cleanup;
-
- maxes = mlx_array_new();
- mlx_max_axis(&maxes, masked_sims, 2, false, S);
- if (!arr_ok(maxes))
- goto cleanup;
-
- sums = mlx_array_new();
- mlx_sum_axis(&sums, maxes, 1, false, S);
- if (!arr_ok(sums))
- goto cleanup;
-
- sums_f32 = mlx_array_new();
- mlx_astype(&sums_f32, sums, MLX_FLOAT32, S);
- if (!arr_ok(sums_f32))
- goto cleanup;
-
- mlx_array_eval(sums_f32);
- const float *data = mlx_array_data_float32(sums_f32);
- if (!data)
- goto cleanup;
- memcpy(scores, data, (size_t)docs_count * sizeof(float));
- rc = 0;
-
-cleanup:
- mlx_array_free(sums_f32);
- mlx_array_free(sums);
- mlx_array_free(maxes);
- mlx_array_free(masked_sims);
- mlx_array_free(invalid_value);
- mlx_array_free(valid_3d);
- mlx_array_free(valid);
- mlx_array_free(positions_2d);
- mlx_array_free(lengths_2d);
- mlx_array_free(positions);
- mlx_array_free(lengths_arr);
- mlx_array_free(sims);
- mlx_array_free(docs_t);
- mlx_array_free(query_batched);
- mlx_array_free(query_3d);
- mlx_array_free(doc_batch);
- if (doc_vec.ctx)
- mlx_vector_array_free(doc_vec);
- mlx_array_free(pad_value);
- for (int i = 0; i < docs_count; i++)
- mlx_array_free(doc_rows[i]);
- free(doc_rows);
- free(lengths);
- return rc;
-}
-
-int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors) {
- embed_mlx_late_vectors_t *vecs =
- embed_mlx_late_encode_tokens_device(ctx, token_ids, n_tokens, normalize);
- if (!vecs)
- return -1;
- int rc = embed_mlx_late_vectors_copy(vecs, out_vectors);
- embed_mlx_late_vectors_free(vecs);
- return rc;
-}
-
-static int embed_mlx_encode_batch_dense(embed_mlx_ctx_t *ctx,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
- ctx->layer_end != ctx->config.n_layers)
- return -1;
-
- const embed_config_t *c = &ctx->config;
- mlx_stream S = ctx->stream;
-
- int hidden = c->hidden_size;
- int max_seq = 0;
- int has_padding = 0;
- int causal = c->attention_mode == EMBED_ATTENTION_CAUSAL;
- int cls_pool = c->pooling_mode == EMBED_POOL_CLS;
- /* CLS and last-token are both single-token pooling: gather one row per
- * sequence (the first for CLS, the last otherwise) rather than mean-pool. */
- int single_token_pool = c->pooling_mode == EMBED_POOL_LAST_TOKEN || cls_pool;
- if (hidden <= 0 || c->vocab_size <= 0)
- return -1;
-
- for (int b = 0; b < batch; b++) {
- if (!inputs[b].ids || inputs[b].n_tokens <= 0)
- return -1;
- if (inputs[b].n_tokens > max_seq)
- max_seq = inputs[b].n_tokens;
- }
- if (max_seq <= 0)
- return -1;
-
- for (int b = 0; b < batch; b++) {
- if (inputs[b].n_tokens != max_seq)
- has_padding = 1;
- }
- int needs_pool_mask = has_padding && !single_token_pool;
- int needs_attn_mask = has_padding && !causal;
-
- size_t elems, ids_bytes, mask_bytes, len_bytes, out_values, out_bytes;
- size_t last_index_bytes = 0;
- if (mlx_mul_size((size_t)batch, (size_t)max_seq, &elems) != 0 ||
- mlx_mul_size(elems, sizeof(int), &ids_bytes) != 0 ||
- mlx_mul_size(elems, sizeof(float), &mask_bytes) != 0 ||
- mlx_mul_size((size_t)batch, sizeof(float), &len_bytes) != 0 ||
- mlx_mul_size((size_t)batch, (size_t)hidden, &out_values) != 0 ||
- mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0 ||
- (single_token_pool && mlx_mul_size(out_values, sizeof(int), &last_index_bytes) != 0))
- return -1;
- if (ids_bytes == 0 || out_bytes == 0 ||
- (needs_pool_mask && (mask_bytes == 0 || len_bytes == 0)) ||
- (needs_attn_mask && mask_bytes == 0) || (single_token_pool && last_index_bytes == 0))
- return -1;
- int *padded_ids = (int *)malloc(ids_bytes);
- float *pool_mask_data = needs_pool_mask ? (float *)malloc(mask_bytes) : NULL;
- float *attn_mask_data = needs_attn_mask ? (float *)malloc(mask_bytes) : NULL;
- float *len_data = needs_pool_mask ? (float *)malloc(len_bytes) : NULL;
- int *last_index_data = single_token_pool ? (int *)malloc(last_index_bytes) : NULL;
- if (!padded_ids || (needs_pool_mask && (!pool_mask_data || !len_data)) ||
- (needs_attn_mask && !attn_mask_data)) {
- free(padded_ids);
- free(pool_mask_data);
- free(attn_mask_data);
- free(len_data);
- free(last_index_data);
- return -1;
- }
- if (single_token_pool && !last_index_data) {
- free(padded_ids);
- free(pool_mask_data);
- free(attn_mask_data);
- free(len_data);
- return -1;
- }
-
- for (int b = 0; b < batch; b++) {
- if (needs_pool_mask)
- len_data[b] = (float)inputs[b].n_tokens;
- if (single_token_pool) {
- int index = cls_pool ? 0 : inputs[b].n_tokens - 1;
- for (int d = 0; d < hidden; d++)
- last_index_data[(size_t)b * hidden + d] = index;
- }
- for (int t = 0; t < max_seq; t++) {
- size_t idx = (size_t)b * max_seq + t;
- if (t < inputs[b].n_tokens) {
- int id = inputs[b].ids[t];
- if (id < 0 || id >= c->vocab_size) {
- fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n", id, b, t);
- free(padded_ids);
- free(pool_mask_data);
- free(attn_mask_data);
- free(len_data);
- free(last_index_data);
- return -1;
- }
- padded_ids[idx] = id;
- if (needs_pool_mask)
- pool_mask_data[idx] = 1.0f;
- if (needs_attn_mask)
- attn_mask_data[idx] = 0.0f;
- } else {
- padded_ids[idx] = 0;
- if (needs_pool_mask)
- pool_mask_data[idx] = 0.0f;
- if (needs_attn_mask)
- attn_mask_data[idx] = -1.0e9f;
- }
- }
- }
-
- int ids_shape[] = {batch, max_seq};
- int pool_mask_shape[] = {batch, max_seq, 1};
- int attn_mask_shape[] = {batch, 1, 1, max_seq};
- int len_shape[] = {batch, 1};
- int last_index_shape[] = {batch, 1, hidden};
-
- mlx_array ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
- mlx_array pool_mask = needs_pool_mask
- ? mlx_array_new_data(pool_mask_data, pool_mask_shape, 3, MLX_FLOAT32)
- : (mlx_array){0};
- mlx_array attn_mask = needs_attn_mask
- ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
- mlx_array lengths =
- needs_pool_mask ? mlx_array_new_data(len_data, len_shape, 2, MLX_FLOAT32) : (mlx_array){0};
- mlx_array last_indices =
- single_token_pool ? mlx_array_new_data(last_index_data, last_index_shape, 3, MLX_INT32)
- : (mlx_array){0};
- free(padded_ids);
- free(pool_mask_data);
- free(attn_mask_data);
- free(len_data);
- free(last_index_data);
-
- if (!arr_ok(ids) || (needs_pool_mask && (!arr_ok(pool_mask) || !arr_ok(lengths))) ||
- (needs_attn_mask && !arr_ok(attn_mask)) || (single_token_pool && !arr_ok(last_indices))) {
- mlx_array_free(ids);
- mlx_array_free(pool_mask);
- mlx_array_free(attn_mask);
- mlx_array_free(lengths);
- mlx_array_free(last_indices);
- return -1;
- }
- if (needs_attn_mask && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
- mlx_array attn_mask_bf16 = mlx_array_new();
- mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
- mlx_array_free(attn_mask);
- attn_mask = attn_mask_bf16;
- if (!arr_ok(attn_mask)) {
- mlx_array_free(ids);
- mlx_array_free(pool_mask);
- mlx_array_free(lengths);
- mlx_array_free(last_indices);
- return -1;
- }
- }
-
- /* 1. Embedding lookup: [B, T] -> [B, T, hidden]. */
- mlx_array x = mlx_array_new();
- mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
- mlx_array_free(ids);
-
- /* 2. Family forward. BERT adds learned positions + token-type[0] and an
- * embedding LayerNorm, then runs post-norm encoder layers; its states are
- * already final, so pooling uses them directly. Qwen runs the decoder block
- * and applies the final RMSNorm here. */
- mlx_array x_normed;
- if (c->family == EMBED_FAMILY_BERT) {
- int start2[] = {0, 0}, stride2[] = {1, 1};
- int pos_start[] = {c->position_id_offset, 0};
- int pos_stop[] = {c->position_id_offset + max_seq, hidden}, type_stop[] = {1, hidden};
- if (pos_stop[0] > c->max_position_embeddings) {
- mlx_array_free(x);
- mlx_array_free(pool_mask);
- mlx_array_free(lengths);
- mlx_array_free(last_indices);
- if (needs_attn_mask)
- mlx_array_free(attn_mask);
- return -1;
- }
- mlx_array pos = mlx_array_new();
- mlx_slice(&pos, ctx->position_embeddings, pos_start, 2, pos_stop, 2, stride2, 2, S);
- mlx_array xp = mlx_array_new();
- mlx_add(&xp, x, pos, S);
- mlx_array_free(x);
- mlx_array_free(pos);
- mlx_array tt = mlx_array_new();
- mlx_slice(&tt, ctx->token_type_embeddings, start2, 2, type_stop, 2, stride2, 2, S);
- mlx_array xt = mlx_array_new();
- mlx_add(&xt, xp, tt, S);
- mlx_array_free(xp);
- mlx_array_free(tt);
- mlx_array xe = mlx_array_new();
- mlx_fast_layer_norm(&xe, xt, ctx->embed_ln_w, ctx->embed_ln_b, c->layer_norm_eps, S);
- mlx_array_free(xt);
- x_normed =
- mlx_forward_bert(ctx, xe, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers);
- } else {
- x = mlx_forward_layers(ctx, x, batch, max_seq, needs_attn_mask, attn_mask, 0, c->n_layers);
- x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
- mlx_array_free(x);
- }
-
- mlx_array emb = mlx_array_new();
- if (single_token_pool) {
- mlx_array selected = mlx_array_new();
- mlx_take_along_axis(&selected, x_normed, last_indices, 1, S);
- mlx_array_free(x_normed);
- mlx_array_free(last_indices);
- int emb_shape[] = {batch, hidden};
- mlx_reshape(&emb, selected, emb_shape, 2, S);
- mlx_array_free(selected);
- mlx_array_free(pool_mask);
- mlx_array_free(attn_mask);
- mlx_array_free(lengths);
- } else if (has_padding) {
- /* 4. Masked mean pool over T. */
- mlx_array masked = mlx_array_new();
- mlx_multiply(&masked, x_normed, pool_mask, S);
- mlx_array_free(x_normed);
- mlx_array_free(pool_mask);
-
- mlx_array emb_sum = mlx_array_new();
- mlx_sum_axis(&emb_sum, masked, 1, false, S);
- mlx_array_free(masked);
-
- mlx_divide(&emb, emb_sum, lengths, S);
- mlx_array_free(emb_sum);
- mlx_array_free(lengths);
- mlx_array_free(attn_mask);
- } else {
- /* Equal-length batches do not need masks or a separate divide. */
- mlx_mean_axis(&emb, x_normed, 1, false, S);
- mlx_array_free(x_normed);
- }
-
- /* 5. Eval and copy [B, hidden] to CPU. Perplexity embeddings are
- * intentionally unnormalized; callers should use cosine similarity. */
- mlx_array emb_f32 = mlx_array_new();
- mlx_astype(&emb_f32, emb, MLX_FLOAT32, S);
- mlx_array_free(emb);
-
- mlx_array_eval(emb_f32);
- const float *data = mlx_array_data_float32(emb_f32);
- int rc = -1;
- if (data) {
- memcpy(out_embeddings, data, out_bytes);
- if (c->normalize_embeddings) {
- rc = 0;
- for (int b = 0; b < batch; b++) {
- if (embed_l2_normalize(out_embeddings + (size_t)b * hidden, hidden) != 0) {
- rc = -1;
- break;
- }
- }
- } else {
- rc = 0;
- }
- }
- mlx_array_free(emb_f32);
- return rc;
-}
-
-typedef struct {
- int index;
- const embed_input_t *input;
-} mlx_batch_item_t;
-
-static int mlx_batch_item_cmp(const void *a, const void *b) {
- const mlx_batch_item_t *ia = (const mlx_batch_item_t *)a;
- const mlx_batch_item_t *ib = (const mlx_batch_item_t *)b;
- int da = ia->input->n_tokens;
- int db = ib->input->n_tokens;
- return (da > db) - (da < db);
-}
-
-static int mlx_padding_exceeds_limit(size_t padded, size_t tokens) {
- if (tokens > SIZE_MAX / 11 || padded > SIZE_MAX / 5)
- return 1;
- return padded * 5 > tokens * 11;
-}
-
-static int mlx_batch_should_split(const embed_input_t *inputs,
- int batch,
- int *out_total_tokens,
- int *out_max_seq) {
- int max_seq = 0;
- int total_tokens = 0;
- for (int b = 0; b < batch; b++) {
- if (!inputs[b].ids || inputs[b].n_tokens <= 0)
- return 0;
- if (inputs[b].n_tokens > max_seq)
- max_seq = inputs[b].n_tokens;
- if (total_tokens > INT_MAX - inputs[b].n_tokens)
- return 0;
- total_tokens += inputs[b].n_tokens;
- }
- if (out_total_tokens)
- *out_total_tokens = total_tokens;
- if (out_max_seq)
- *out_max_seq = max_seq;
- if (batch < 4 || max_seq < 64 || total_tokens <= 0)
- return 0;
-
- size_t padded;
- if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0)
- return 0;
-
- return mlx_padding_exceeds_limit(padded, (size_t)total_tokens);
-}
-
-static int mlx_should_add_to_group(const embed_mlx_ctx_t *ctx,
- int group_count,
- int group_max,
- int next_tokens) {
- if (!ctx || ctx->config.hidden_size < 2048 || group_count <= 0)
- return 1;
-
- int next_count = group_count + 1;
- int next_max = next_tokens > group_max ? next_tokens : group_max;
- size_t merged, split_left, split_total;
- if (mlx_mul_size((size_t)next_count, (size_t)next_max, &merged) != 0 ||
- mlx_mul_size((size_t)group_count, (size_t)group_max, &split_left) != 0)
- return 0;
- /* Treat one extra MLX launch as roughly 256 padded token rows for 4B.
- * This preserves small mixed groups while splitting larger padded tails. */
- if (split_left > SIZE_MAX - (size_t)next_tokens ||
- split_left + (size_t)next_tokens > SIZE_MAX - 256)
- return 0;
- split_total = split_left + (size_t)next_tokens + 256;
- return merged <= split_total;
-}
-
-int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!ctx || !inputs || batch <= 0 || !out_embeddings)
- return -1;
-
- if (!mlx_batch_should_split(inputs, batch, NULL, NULL))
- return embed_mlx_encode_batch_dense(ctx, inputs, batch, out_embeddings);
-
- const int hidden = ctx->config.hidden_size;
- mlx_batch_item_t *items = (mlx_batch_item_t *)malloc((size_t)batch * sizeof(*items));
- embed_input_t *group_inputs = (embed_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
- size_t group_values, group_bytes;
- if (mlx_mul_size((size_t)batch, (size_t)hidden, &group_values) != 0 ||
- mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
- free(items);
- free(group_inputs);
- return -1;
- }
- float *group_out = (float *)malloc(group_bytes);
- if (!items || !group_inputs || !group_out) {
- free(items);
- free(group_inputs);
- free(group_out);
- return -1;
- }
- for (int i = 0; i < batch; i++) {
- items[i].index = i;
- items[i].input = &inputs[i];
- }
- qsort(items, (size_t)batch, sizeof(*items), mlx_batch_item_cmp);
-
- int rc = 0;
- for (int start = 0; start < batch && rc == 0;) {
- int group_max = 0;
- int group_tokens = 0;
- int end = start;
- while (end < batch) {
- int next_tokens = items[end].input->n_tokens;
- int next_count = end - start + 1;
- int next_max = next_tokens > group_max ? next_tokens : group_max;
- int next_total = group_tokens + next_tokens;
- if (end > start) {
- size_t next_padded;
- if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 ||
- mlx_padding_exceeds_limit(next_padded, (size_t)next_total) ||
- !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens))
- break;
- }
- group_max = next_max;
- group_tokens = next_total;
- end++;
- }
-
- int group_count = end - start;
- for (int i = 0; i < group_count; i++)
- group_inputs[i] = *items[start + i].input;
- rc = embed_mlx_encode_batch_dense(ctx, group_inputs, group_count, group_out);
- if (rc != 0)
- break;
- for (int i = 0; i < group_count; i++) {
- int dst = items[start + i].index;
- memcpy(out_embeddings + (size_t)dst * hidden, group_out + (size_t)i * hidden,
- (size_t)hidden * sizeof(float));
- }
- start = end;
- }
-
- free(items);
- free(group_inputs);
- free(group_out);
- return rc;
-}
-
-int embed_mlx_encode_into(embed_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- float *out_embedding) {
- embed_input_t input = {token_ids, n_tokens};
- return embed_mlx_encode_batch(ctx, &input, 1, out_embedding);
-}
-
-typedef struct {
- int index;
- int span_offset;
- const embed_context_input_t *input;
-} mlx_context_batch_item_t;
-
-static int mlx_context_batch_item_cmp(const void *a, const void *b) {
- const mlx_context_batch_item_t *ia = (const mlx_context_batch_item_t *)a;
- const mlx_context_batch_item_t *ib = (const mlx_context_batch_item_t *)b;
- int da = ia->input->input.n_tokens;
- int db = ib->input->input.n_tokens;
- return (da > db) - (da < db);
-}
-
-static int mlx_context_padding_exceeds_limit(size_t padded, size_t tokens) {
- if (tokens > SIZE_MAX / 5 || padded > SIZE_MAX / 4)
- return 1;
- return padded * 4 > tokens * 5;
-}
-
-static int mlx_context_batch_should_split(const embed_context_input_t *inputs,
- int batch,
- int *out_total_spans) {
- int max_seq = 0;
- int total_tokens = 0;
- int total_spans = 0;
- for (int b = 0; b < batch; b++) {
- int n_tokens = inputs[b].input.n_tokens;
- if (!inputs[b].input.ids || n_tokens <= 0 || !inputs[b].spans || inputs[b].n_spans <= 0)
- return 0;
- if (n_tokens > max_seq)
- max_seq = n_tokens;
- if (total_tokens > INT_MAX - n_tokens || total_spans > INT_MAX - inputs[b].n_spans)
- return 0;
- total_tokens += n_tokens;
- total_spans += inputs[b].n_spans;
- }
- if (out_total_spans)
- *out_total_spans = total_spans;
- if (batch < 2 || max_seq < 64 || total_tokens <= 0)
- return 0;
-
- size_t padded;
- if (mlx_mul_size((size_t)batch, (size_t)max_seq, &padded) != 0)
- return 0;
- return mlx_context_padding_exceeds_limit(padded, (size_t)total_tokens);
-}
-
-static int embed_mlx_encode_spans_batch_dense(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!ctx || !inputs || batch <= 0 || !out_embeddings || ctx->layer_start != 0 ||
- ctx->layer_end != ctx->config.n_layers || !arr_ok(ctx->embed_tokens) || !arr_ok(ctx->norm))
- return -1;
-
- const embed_config_t *c = &ctx->config;
- mlx_stream S = ctx->stream;
- int hidden = c->hidden_size;
- int max_seq = 0;
- int total_spans = 0;
- int has_padding = 0;
- for (int b = 0; b < batch; b++) {
- const embed_context_input_t *input = &inputs[b];
- int n_tokens = input->input.n_tokens;
- if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
- total_spans > INT_MAX - input->n_spans)
- return -1;
- total_spans += input->n_spans;
- if (n_tokens > max_seq)
- max_seq = n_tokens;
- for (int t = 0; t < n_tokens; t++) {
- if (input->input.ids[t] < 0 || input->input.ids[t] >= c->vocab_size) {
- fprintf(stderr, "mlx: invalid token id %d at input %d position %d\n",
- input->input.ids[t], b, t);
- return -1;
- }
- }
- for (int s = 0; s < input->n_spans; s++) {
- if (input->spans[s].start < 0 || input->spans[s].start > n_tokens ||
- input->spans[s].n_tokens <= 0 ||
- input->spans[s].n_tokens > n_tokens - input->spans[s].start)
- return -1;
- }
- }
- for (int b = 0; b < batch; b++)
- if (inputs[b].input.n_tokens != max_seq)
- has_padding = 1;
-
- size_t rows, ids_bytes, mask_bytes, index_bytes;
- size_t span_float_bytes, out_values, out_bytes;
- if (mlx_mul_size((size_t)batch, (size_t)max_seq, &rows) != 0 ||
- mlx_mul_size(rows, sizeof(int), &ids_bytes) != 0 ||
- mlx_mul_size(rows, sizeof(float), &mask_bytes) != 0 ||
- mlx_mul_size((size_t)total_spans, sizeof(int), &index_bytes) != 0 ||
- mlx_mul_size((size_t)total_spans, sizeof(float), &span_float_bytes) != 0 ||
- mlx_mul_size((size_t)total_spans, (size_t)hidden, &out_values) != 0 ||
- mlx_mul_size(out_values, sizeof(float), &out_bytes) != 0)
- return -1;
- if (rows > INT_MAX)
- return -1;
- int *padded_ids = (int *)malloc(ids_bytes);
- float *attn_mask_data = has_padding ? (float *)malloc(mask_bytes) : NULL;
- if (!padded_ids || (has_padding && !attn_mask_data)) {
- free(padded_ids);
- free(attn_mask_data);
- return -1;
- }
- for (int b = 0; b < batch; b++) {
- const embed_input_t *input = &inputs[b].input;
- for (int t = 0; t < max_seq; t++) {
- size_t row = (size_t)b * max_seq + t;
- if (t < input->n_tokens) {
- padded_ids[row] = input->ids[t];
- if (has_padding)
- attn_mask_data[row] = 0.0f;
- } else {
- padded_ids[row] = 0;
- if (has_padding)
- attn_mask_data[row] = -1.0e9f;
- }
- }
- }
-
- mlx_array ids = (mlx_array){0};
- mlx_array attn_mask = (mlx_array){0};
- mlx_array x = (mlx_array){0};
- mlx_array x_normed = (mlx_array){0};
- mlx_array prefix = (mlx_array){0};
- mlx_array flat_prefix = (mlx_array){0};
- mlx_array end_idx = (mlx_array){0};
- mlx_array start_idx = (mlx_array){0};
- mlx_array lengths = (mlx_array){0};
- mlx_array start_mask = (mlx_array){0};
- mlx_array end_values = (mlx_array){0};
- mlx_array start_values = (mlx_array){0};
- mlx_array masked_start = (mlx_array){0};
- mlx_array span_sums = (mlx_array){0};
- mlx_array embeddings = (mlx_array){0};
- mlx_array embeddings_f32 = (mlx_array){0};
- mlx_array *pooled = NULL;
- mlx_vector_array pooled_vec = (mlx_vector_array){0};
- int *end_idx_data = NULL;
- int *start_idx_data = NULL;
- float *lengths_data = NULL;
- float *start_mask_data = NULL;
- int rc = -1;
-
- int ids_shape[] = {batch, max_seq};
- int attn_mask_shape[] = {batch, 1, 1, max_seq};
- ids = mlx_array_new_data(padded_ids, ids_shape, 2, MLX_INT32);
- attn_mask = has_padding ? mlx_array_new_data(attn_mask_data, attn_mask_shape, 4, MLX_FLOAT32)
- : (mlx_array){0};
- free(padded_ids);
- free(attn_mask_data);
- if (!arr_ok(ids))
- goto cleanup;
- if (has_padding && !arr_ok(attn_mask))
- goto cleanup;
- if (has_padding && mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
- mlx_array attn_mask_bf16 = mlx_array_new();
- mlx_astype(&attn_mask_bf16, attn_mask, MLX_BFLOAT16, S);
- mlx_array_free(attn_mask);
- attn_mask = attn_mask_bf16;
- if (!arr_ok(attn_mask))
- goto cleanup;
- }
-
- x = mlx_array_new();
- mlx_take_axis(&x, ctx->embed_tokens, ids, 0, S);
- mlx_array_free(ids);
- ids = (mlx_array){0};
- if (!arr_ok(x))
- goto cleanup;
-
- x = mlx_forward_layers(ctx, x, batch, max_seq, has_padding, attn_mask, 0, c->n_layers);
- if (has_padding) {
- mlx_array_free(attn_mask);
- attn_mask = (mlx_array){0};
- }
- if (!arr_ok(x))
- goto cleanup;
-
- x_normed = mlx_array_new();
- mlx_fast_rms_norm(&x_normed, x, ctx->norm, c->rms_norm_eps, S);
- mlx_array_free(x);
- x = (mlx_array){0};
- if (!arr_ok(x_normed))
- goto cleanup;
-
- if (mlx_weight_dtype(ctx) == MLX_BFLOAT16) {
- pooled = (mlx_array *)calloc((size_t)total_spans, sizeof(*pooled));
- if (!pooled)
- goto cleanup;
-
- int pooled_count = 0;
- for (int b = 0; b < batch; b++) {
- for (int s = 0; s < inputs[b].n_spans; s++) {
- const embed_span_t *span = &inputs[b].spans[s];
- int start[] = {b, span->start, 0};
- int stop[] = {b + 1, span->start + span->n_tokens, hidden};
- int strides[] = {1, 1, 1};
- mlx_array slice = mlx_array_new();
- mlx_slice(&slice, x_normed, start, 3, stop, 3, strides, 3, S);
- if (!arr_ok(slice)) {
- mlx_array_free(slice);
- goto cleanup;
- }
-
- pooled[pooled_count] = mlx_array_new();
- mlx_mean_axis(&pooled[pooled_count], slice, 1, false, S);
- mlx_array_free(slice);
- if (!arr_ok(pooled[pooled_count]))
- goto cleanup;
- pooled_count++;
- }
- }
-
- pooled_vec = mlx_vector_array_new_data(pooled, (size_t)total_spans);
- if (!pooled_vec.ctx)
- goto cleanup;
- embeddings = mlx_array_new();
- mlx_concatenate_axis(&embeddings, pooled_vec, 0, S);
- if (!arr_ok(embeddings))
- goto cleanup;
- } else {
- end_idx_data = (int *)malloc(index_bytes);
- start_idx_data = (int *)malloc(index_bytes);
- lengths_data = (float *)malloc(span_float_bytes);
- start_mask_data = (float *)malloc(span_float_bytes);
- if (!end_idx_data || !start_idx_data || !lengths_data || !start_mask_data)
- goto cleanup;
-
- int span_count = 0;
- for (int b = 0; b < batch; b++) {
- for (int s = 0; s < inputs[b].n_spans; s++) {
- const embed_span_t *span = &inputs[b].spans[s];
- int base = b * max_seq;
- int start = span->start;
- int stop = span->start + span->n_tokens;
- end_idx_data[span_count] = base + stop - 1;
- start_idx_data[span_count] = start > 0 ? base + start - 1 : base;
- lengths_data[span_count] = (float)span->n_tokens;
- start_mask_data[span_count] = start > 0 ? 1.0f : 0.0f;
- span_count++;
- }
- }
-
- int index_shape[] = {total_spans};
- int scale_shape[] = {total_spans, 1};
- end_idx = mlx_array_new_data(end_idx_data, index_shape, 1, MLX_INT32);
- start_idx = mlx_array_new_data(start_idx_data, index_shape, 1, MLX_INT32);
- lengths = mlx_array_new_data(lengths_data, scale_shape, 2, MLX_FLOAT32);
- start_mask = mlx_array_new_data(start_mask_data, scale_shape, 2, MLX_FLOAT32);
- free(end_idx_data);
- end_idx_data = NULL;
- free(start_idx_data);
- start_idx_data = NULL;
- free(lengths_data);
- lengths_data = NULL;
- free(start_mask_data);
- start_mask_data = NULL;
- if (!arr_ok(end_idx) || !arr_ok(start_idx) || !arr_ok(lengths) || !arr_ok(start_mask))
- goto cleanup;
-
- prefix = mlx_array_new();
- mlx_cumsum(&prefix, x_normed, 1, false, true, S);
- mlx_array_free(x_normed);
- x_normed = (mlx_array){0};
- if (!arr_ok(prefix))
- goto cleanup;
-
- int flat_shape[] = {(int)rows, hidden};
- flat_prefix = mlx_array_new();
- mlx_reshape(&flat_prefix, prefix, flat_shape, 2, S);
- mlx_array_free(prefix);
- prefix = (mlx_array){0};
- if (!arr_ok(flat_prefix))
- goto cleanup;
-
- end_values = mlx_array_new();
- mlx_take_axis(&end_values, flat_prefix, end_idx, 0, S);
- if (!arr_ok(end_values))
- goto cleanup;
-
- start_values = mlx_array_new();
- mlx_take_axis(&start_values, flat_prefix, start_idx, 0, S);
- if (!arr_ok(start_values))
- goto cleanup;
-
- masked_start = mlx_array_new();
- mlx_multiply(&masked_start, start_values, start_mask, S);
- if (!arr_ok(masked_start))
- goto cleanup;
-
- span_sums = mlx_array_new();
- mlx_subtract(&span_sums, end_values, masked_start, S);
- if (!arr_ok(span_sums))
- goto cleanup;
-
- embeddings = mlx_array_new();
- mlx_divide(&embeddings, span_sums, lengths, S);
- if (!arr_ok(embeddings))
- goto cleanup;
- }
-
- embeddings_f32 = mlx_array_new();
- mlx_astype(&embeddings_f32, embeddings, MLX_FLOAT32, S);
- if (!arr_ok(embeddings_f32))
- goto cleanup;
-
- mlx_array_eval(embeddings_f32);
- const float *data = mlx_array_data_float32(embeddings_f32);
- if (data) {
- memcpy(out_embeddings, data, out_bytes);
- rc = 0;
- }
-
-cleanup:
- free(start_mask_data);
- free(lengths_data);
- free(start_idx_data);
- free(end_idx_data);
- mlx_array_free(embeddings_f32);
- mlx_array_free(embeddings);
- if (pooled_vec.ctx)
- mlx_vector_array_free(pooled_vec);
- if (pooled) {
- for (int s = 0; s < total_spans; s++)
- mlx_array_free(pooled[s]);
- free(pooled);
- }
- mlx_array_free(span_sums);
- mlx_array_free(masked_start);
- mlx_array_free(start_values);
- mlx_array_free(end_values);
- mlx_array_free(start_mask);
- mlx_array_free(lengths);
- mlx_array_free(start_idx);
- mlx_array_free(end_idx);
- mlx_array_free(flat_prefix);
- mlx_array_free(prefix);
- mlx_array_free(x_normed);
- mlx_array_free(x);
- mlx_array_free(attn_mask);
- mlx_array_free(ids);
- return rc;
-}
-
-int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings) {
- if (!ctx || !inputs || batch <= 0 || !out_embeddings)
- return -1;
-
- int total_spans = 0;
- if (!mlx_context_batch_should_split(inputs, batch, &total_spans))
- return embed_mlx_encode_spans_batch_dense(ctx, inputs, batch, out_embeddings);
-
- const int hidden = ctx->config.hidden_size;
- mlx_context_batch_item_t *items =
- (mlx_context_batch_item_t *)malloc((size_t)batch * sizeof(*items));
- embed_context_input_t *group_inputs =
- (embed_context_input_t *)malloc((size_t)batch * sizeof(*group_inputs));
- size_t group_values, group_bytes;
- if (mlx_mul_size((size_t)total_spans, (size_t)hidden, &group_values) != 0 ||
- mlx_mul_size(group_values, sizeof(float), &group_bytes) != 0) {
- free(items);
- free(group_inputs);
- return -1;
- }
- float *group_out = (float *)malloc(group_bytes);
- if (!items || !group_inputs || !group_out) {
- free(items);
- free(group_inputs);
- free(group_out);
- return -1;
- }
-
- int span_offset = 0;
- for (int i = 0; i < batch; i++) {
- items[i].index = i;
- items[i].span_offset = span_offset;
- items[i].input = &inputs[i];
- span_offset += inputs[i].n_spans;
- }
- qsort(items, (size_t)batch, sizeof(*items), mlx_context_batch_item_cmp);
-
- int rc = 0;
- for (int start = 0; start < batch && rc == 0;) {
- int group_max = 0;
- int group_tokens = 0;
- int end = start;
- while (end < batch) {
- int next_tokens = items[end].input->input.n_tokens;
- int next_count = end - start + 1;
- int next_max = next_tokens > group_max ? next_tokens : group_max;
- int next_total = group_tokens + next_tokens;
- if (end > start) {
- size_t next_padded;
- if (mlx_mul_size((size_t)next_count, (size_t)next_max, &next_padded) != 0 ||
- mlx_context_padding_exceeds_limit(next_padded, (size_t)next_total) ||
- !mlx_should_add_to_group(ctx, end - start, group_max, next_tokens))
- break;
- }
- group_max = next_max;
- group_tokens = next_total;
- end++;
- }
-
- int group_count = end - start;
- for (int i = 0; i < group_count; i++)
- group_inputs[i] = *items[start + i].input;
- rc = embed_mlx_encode_spans_batch_dense(ctx, group_inputs, group_count, group_out);
- if (rc != 0)
- break;
- int src_span = 0;
- for (int i = 0; i < group_count; i++) {
- int n_spans = group_inputs[i].n_spans;
- int dst_span = items[start + i].span_offset;
- memcpy(out_embeddings + (size_t)dst_span * hidden,
- group_out + (size_t)src_span * hidden, (size_t)n_spans * hidden * sizeof(float));
- src_span += n_spans;
- }
- start = end;
- }
-
- free(items);
- free(group_inputs);
- free(group_out);
- return rc;
-}
-
-int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings) {
- embed_context_input_t input = {
- .input = {token_ids, n_tokens},
- .spans = spans,
- .n_spans = n_spans,
- };
- return embed_mlx_encode_spans_batch(ctx, &input, 1, out_embeddings);
-}
-
-float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens) {
- if (!ctx || !token_ids || n_tokens <= 0)
- return NULL;
-
- int hidden = ctx->config.hidden_size;
- size_t out_bytes;
- if (mlx_mul_size((size_t)hidden, sizeof(float), &out_bytes) != 0)
- return NULL;
- float *out = (float *)malloc(out_bytes);
- if (!out)
- return NULL;
-
- if (embed_mlx_encode_into(ctx, token_ids, n_tokens, out) != 0) {
- free(out);
- return NULL;
- }
- return out;
-}
diff --git a/src/mlx.h b/src/mlx.h
deleted file mode 100644
index 55f5666..0000000
--- a/src/mlx.h
+++ /dev/null
@@ -1,167 +0,0 @@
-/*
- * embed_mlx.h - MLX GPU backend for Apple Silicon
- *
- * Uses Apple's mlx-c to run the transformer on Metal.
- */
-
-#ifndef EMBED_MLX_H
-#define EMBED_MLX_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-#include <stddef.h>
-#include <stdint.h>
-#include "embed.h"
-
-/* Opaque MLX model context */
-typedef struct embed_mlx_ctx embed_mlx_ctx_t;
-typedef struct embed_mlx_late_ctx embed_mlx_late_ctx_t;
-typedef struct embed_mlx_late_vectors embed_mlx_late_vectors_t;
-
-typedef struct {
- int quantize_bits; /* 0 disables; currently 8 */
- int quantize_group_size; /* default 64 when quantize_bits is set */
-} embed_mlx_options_t;
-
-/*
- * Load model into MLX arrays from safetensors + config.json.
- * Returns NULL on error.
- */
-EMBED_API embed_mlx_ctx_t *embed_mlx_load(const char *model_dir);
-EMBED_API embed_mlx_ctx_t *embed_mlx_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts);
-
-EMBED_API void embed_mlx_free(embed_mlx_ctx_t *ctx);
-
-/*
- * Late-interaction MLX path. This is separate from pooled embeddings because
- * late snapshots produce one projected token vector per input token for MaxSim
- * scoring, not one pooled document vector.
- */
-EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load(const char *model_dir);
-EMBED_API embed_mlx_late_ctx_t *embed_mlx_late_load_with_options(const char *model_dir,
- const embed_mlx_options_t *opts);
-EMBED_API void embed_mlx_late_free(embed_mlx_late_ctx_t *ctx);
-
-EMBED_API const embed_config_t *embed_mlx_late_config(const embed_mlx_late_ctx_t *ctx);
-EMBED_API int embed_mlx_late_token_dim(const embed_mlx_late_ctx_t *ctx);
-
-/*
- * Encode token_ids into out_vectors[n_tokens, token_dim].
- *
- * If normalize is non-zero, every token vector is L2-normalized on device
- * before copying back to CPU memory.
- */
-EMBED_API int embed_mlx_late_encode_tokens(embed_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize,
- float *out_vectors);
-
-/*
- * Device-resident late-interaction token vectors.
- *
- * The model context owns the MLX stream; the returned vector handle must not
- * outlive ctx. Copying is optional: MaxSim can consume these handles directly
- * and copy back only final candidate scores.
- */
-EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_tokens_device(embed_mlx_late_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- int normalize);
-EMBED_API void embed_mlx_late_vectors_free(embed_mlx_late_vectors_t *vecs);
-EMBED_API int embed_mlx_late_vectors_token_count(const embed_mlx_late_vectors_t *vecs);
-EMBED_API int embed_mlx_late_vectors_dim(const embed_mlx_late_vectors_t *vecs);
-EMBED_API int embed_mlx_late_vectors_copy(const embed_mlx_late_vectors_t *vecs, float *out_vectors);
-/*
- * Encode n_docs documents in one padded transformer pass and pack each
- * document's kept token vectors back-to-back into [total_keep, token_dim] in
- * document order. out_offsets[n_docs + 1] receives the prefix sum of kept
- * counts, so the result feeds embed_mlx_late_maxsim_batch_device directly.
- * Returns NULL on error; the handle must not outlive ctx.
- */
-EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_encode_docs_device(embed_mlx_late_ctx_t *ctx,
- const int *const *doc_ids,
- const int *n_tokens,
- const int *const *keep,
- const int *n_keep,
- int n_docs,
- int normalize,
- int *out_offsets);
-/* Device-resident select/concat primitives, retained for the late-interaction
- * verification harness (scripts/check_late_interaction.py); the server rerank
- * path uses embed_mlx_late_encode_docs_device. */
-EMBED_API embed_mlx_late_vectors_t *embed_mlx_late_vectors_concat(
- embed_mlx_late_ctx_t *ctx, const embed_mlx_late_vectors_t *const *items, int count);
-EMBED_API embed_mlx_late_vectors_t *
-embed_mlx_late_vectors_select(embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *vecs,
- const int *token_indices,
- int count);
-EMBED_API int embed_mlx_late_maxsim_batch_device(embed_mlx_late_ctx_t *ctx,
- const embed_mlx_late_vectors_t *query,
- const embed_mlx_late_vectors_t *docs,
- const int *doc_offsets,
- int docs_count,
- float *scores);
-
-/*
- * Compute embedding for token_ids[0..n_tokens-1].
- * Returns malloc'd float[hidden_size] (caller frees). NULL on error.
- */
-EMBED_API float *embed_mlx_encode(embed_mlx_ctx_t *ctx, const int *token_ids, int n_tokens);
-
-/*
- * Compute one embedding into caller-provided out[hidden_size].
- * Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_mlx_encode_into(embed_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- float *out_embedding);
-
-/*
- * Compute a true padded dense batch on MLX.
- *
- * out_embeddings is caller-provided [batch, hidden_size].
- * Pooling and attention masking follow the loaded model configuration.
- * Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_mlx_encode_batch(embed_mlx_ctx_t *ctx,
- const embed_input_t *inputs,
- int batch,
- float *out_embeddings);
-
-/*
- * Run one contextual sequence and pool selected token spans.
- * Returns 0 on success, -1 on error.
- */
-EMBED_API int embed_mlx_encode_spans(embed_mlx_ctx_t *ctx,
- const int *token_ids,
- int n_tokens,
- const embed_span_t *spans,
- int n_spans,
- float *out_embeddings);
-
-/*
- * Run a padded dense contextual document batch and pool every selected span.
- * out_embeddings contains spans in document order.
- */
-EMBED_API int embed_mlx_encode_spans_batch(embed_mlx_ctx_t *ctx,
- const embed_context_input_t *inputs,
- int batch,
- float *out_embeddings);
-
-/* Get the config (so embed_cli.c can read hidden_size etc.) */
-EMBED_API const embed_config_t *embed_mlx_config(const embed_mlx_ctx_t *ctx);
-
-#endif /* EMBED_MLX_H */
diff --git a/src/model.c b/src/model.c
deleted file mode 100644
index c39c526..0000000
--- a/src/model.c
+++ /dev/null
@@ -1,688 +0,0 @@
-#include "embed.h"
-#include "model_internal.h"
-#include "config.h"
-#include "kernels.h"
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <math.h>
-#include <limits.h>
-
-/* ========================================================================
- * Globals
- * ======================================================================== */
-
-int embed_verbose = 0;
-
-/* ========================================================================
- * Model-directory probes
- * ======================================================================== */
-
-static int model_dir_has_file(const char *model_dir, const char *rel) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel);
- if (n < 0 || (size_t)n >= sizeof(path))
- return 0;
- FILE *f = fopen(path, "rb");
- if (!f)
- return 0;
- fclose(f);
- return 1;
-}
-
-int model_dir_has_late_projection(const char *model_dir) {
- return model_dir_has_file(model_dir, "1_Dense/config.json") &&
- model_dir_has_file(model_dir, "1_Dense/model.safetensors");
-}
-
-/* ========================================================================
- * Weight tensor loading and validation (direct mmap pointers into safetensors)
- * ======================================================================== */
-
-static size_t dtype_size(safetensor_dtype_t dtype) {
- switch (dtype) {
- case DTYPE_F32:
- return sizeof(float);
- case DTYPE_BF16:
- case DTYPE_F16:
- return sizeof(uint16_t);
- default:
- return 0;
- }
-}
-
-static const char *dtype_name(safetensor_dtype_t dtype) {
- switch (dtype) {
- case DTYPE_F32:
- return "F32";
- case DTYPE_F16:
- return "F16";
- case DTYPE_BF16:
- return "BF16";
- case DTYPE_I32:
- return "I32";
- case DTYPE_I64:
- return "I64";
- case DTYPE_BOOL:
- return "BOOL";
- default:
- return "UNKNOWN";
- }
-}
-
-int tensor_has_supported_shape(const safetensors_file_t *sf,
- const safetensor_t *t,
- const char *name,
- const int64_t *shape,
- int ndim,
- int bf16_ok) {
- /* bf16_ok admits either 16-bit float format (BF16 or F16); both are widened
- * to F32 by the caller (BF16 lazily per matmul, F16 once at load). */
- int sixteen_ok = bf16_ok && (t->dtype == DTYPE_BF16 || t->dtype == DTYPE_F16);
- if (t->dtype != DTYPE_F32 && !sixteen_ok) {
- fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name,
- dtype_name(t->dtype), bf16_ok ? "F32, BF16, or F16" : "F32");
- return 0;
- }
- if (t->ndim != ndim) {
- fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
- return 0;
- }
-
- size_t numel = 1;
- for (int i = 0; i < ndim; i++) {
- if (t->shape[i] != shape[i]) {
- fprintf(stderr,
- "embed: bad shape for %s at dim %d: "
- "got %lld, expected %lld\n",
- name, i, (long long)t->shape[i], (long long)shape[i]);
- return 0;
- }
- if (shape[i] < 0 || (uint64_t)shape[i] > SIZE_MAX / numel) {
- fprintf(stderr, "embed: shape too large for %s\n", name);
- return 0;
- }
- numel *= (size_t)shape[i];
- }
-
- size_t elem_size = dtype_size(t->dtype);
- if (elem_size == 0) {
- fprintf(stderr, "embed: unsupported dtype for %s: got %s\n", name, dtype_name(t->dtype));
- return 0;
- }
-
- size_t bytes;
- if (mul_size(numel, elem_size, &bytes) != 0) {
- fprintf(stderr, "embed: tensor data too large for %s\n", name);
- return 0;
- }
- if (t->data_size != bytes) {
- fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n", name, t->data_size,
- bytes);
- return 0;
- }
-
- size_t data_start = 8 + sf->header_size;
- if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
- t->data_size > sf->file_size - data_start - t->data_offset) {
- fprintf(stderr, "embed: tensor data out of bounds for %s\n", name);
- return 0;
- }
- return 1;
-}
-
-embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t) {
- embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0};
- /* F32/BF16 are consumed directly by the forward pass, so the ref borrows the
- * mmap (zero copy; BF16 is widened lazily per matmul). F16 has neither a
- * fused kernel nor a zero-copy F32 view, so widen it once here into an owned
- * buffer the model frees on unload. DTYPE_F16 never reaches the forward. */
- if (t->dtype == DTYPE_F16) {
- float *buf = safetensors_get_f32(sf, t);
- if (!buf)
- return ref;
- ref.dtype = DTYPE_F32;
- ref.data = buf;
- ref.owned = 1;
- } else {
- ref.dtype = t->dtype;
- ref.data = safetensors_data(sf, t);
- }
- return ref;
-}
-
-static embed_weight_ref_t
-load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) {
- embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0};
- safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
- if (!t) {
- fprintf(stderr, "embed: tensor not found: %s\n", name);
- return ref;
- }
- if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1))
- return ref;
-
- return weight_ref_from_tensor(sf, t);
-}
-
-static float *
-load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) {
- safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
- if (!t) {
- fprintf(stderr, "embed: tensor not found: %s\n", name);
- return NULL;
- }
- if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1))
- return NULL;
- return safetensors_get_f32(sf, t);
-}
-
-/* ========================================================================
- * Model / workspace load and free
- * ======================================================================== */
-
-/* BERT-family weight loader. The tensor names differ entirely from the Qwen
- * block (encoder.layer.N.*, embeddings.*), so the family gets its own entry
- * point. Reused embed_layer_t slots carry their BERT meaning: wo is the
- * attention output dense, input_norm/post_attn_norm are the two block LayerNorm
- * weights, up_proj/down_proj are the two FFN matrices (see embed_internal.h). */
-static int
-load_bert_weights(embed_model_t *model, multi_safetensors_t *ms, int layer_start, int layer_end) {
- const embed_config_t *cfg = &model->config;
- embed_weights_t *w = &model->weights;
- char name[256];
-
- /* sentence-transformers checkpoints store encoder tensors unprefixed; raw
- * HF BERT and RoBERTa/XLM-R checkpoints use "bert." or "roberta.". */
- const char *p;
- if (multi_safetensors_find(ms, "embeddings.word_embeddings.weight", NULL))
- p = "";
- else if (multi_safetensors_find(ms, "bert.embeddings.word_embeddings.weight", NULL))
- p = "bert.";
- else if (multi_safetensors_find(ms, "roberta.embeddings.word_embeddings.weight", NULL))
- p = "roberta.";
- else {
- fprintf(stderr, "embed_model_load: BERT-family word embeddings not found\n");
- return -1;
- }
-
- int hidden = cfg->hidden_size, inter = cfg->intermediate_size;
-
- if (layer_start == 0) {
- const int64_t emb_shape[2] = {cfg->vocab_size, hidden};
- snprintf(name, sizeof(name), "%sembeddings.word_embeddings.weight", p);
- w->embed_tokens = load_weight_direct(ms, name, emb_shape, 2);
- const int64_t pos_shape[2] = {cfg->max_position_embeddings, hidden};
- snprintf(name, sizeof(name), "%sembeddings.position_embeddings.weight", p);
- w->position_embeddings = load_norm_f32(ms, name, pos_shape, 2);
- const int64_t type_shape[2] = {cfg->type_vocab_size, hidden};
- snprintf(name, sizeof(name), "%sembeddings.token_type_embeddings.weight", p);
- w->token_type_embeddings = load_norm_f32(ms, name, type_shape, 2);
- const int64_t ln_shape[1] = {hidden};
- snprintf(name, sizeof(name), "%sembeddings.LayerNorm.weight", p);
- w->embed_ln_w = load_norm_f32(ms, name, ln_shape, 1);
- snprintf(name, sizeof(name), "%sembeddings.LayerNorm.bias", p);
- w->embed_ln_b = load_norm_f32(ms, name, ln_shape, 1);
- if (!w->embed_tokens.data || !w->position_embeddings || !w->token_type_embeddings ||
- !w->embed_ln_w || !w->embed_ln_b)
- return -1;
- }
-
- w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t));
- if (!w->layers)
- return -1;
-
- for (int i = layer_start; i < layer_end; i++) {
- embed_layer_t *l = &w->layers[i];
-
-#define BERT_W(field, suffix, d0, d1) \
- do { \
- const int64_t e[2] = {(d0), (d1)}; \
- snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \
- l->field = load_weight_direct(ms, name, e, 2); \
- if (!l->field.data) \
- return -1; \
- } while (0)
-#define BERT_V(field, suffix, d0) \
- do { \
- const int64_t e[1] = {(d0)}; \
- snprintf(name, sizeof(name), "%sencoder.layer.%d." suffix, p, i); \
- l->field = load_norm_f32(ms, name, e, 1); \
- if (!l->field) \
- return -1; \
- } while (0)
-
- BERT_W(wq, "attention.self.query.weight", hidden, hidden);
- BERT_V(q_bias, "attention.self.query.bias", hidden);
- BERT_W(wk, "attention.self.key.weight", hidden, hidden);
- BERT_V(k_bias, "attention.self.key.bias", hidden);
- BERT_W(wv, "attention.self.value.weight", hidden, hidden);
- BERT_V(v_bias, "attention.self.value.bias", hidden);
- BERT_W(wo, "attention.output.dense.weight", hidden, hidden);
- BERT_V(o_bias, "attention.output.dense.bias", hidden);
- BERT_V(input_norm, "attention.output.LayerNorm.weight", hidden);
- BERT_V(attn_ln_bias, "attention.output.LayerNorm.bias", hidden);
- BERT_W(up_proj, "intermediate.dense.weight", inter, hidden);
- BERT_V(ffn_inter_bias, "intermediate.dense.bias", inter);
- BERT_W(down_proj, "output.dense.weight", hidden, inter);
- BERT_V(ffn_out_bias, "output.dense.bias", hidden);
- BERT_V(post_attn_norm, "output.LayerNorm.weight", hidden);
- BERT_V(ffn_ln_bias, "output.LayerNorm.bias", hidden);
-
-#undef BERT_W
-#undef BERT_V
- }
- /* BERT has no final norm: w->norm stays NULL, and pooling skips it. */
- return 0;
-}
-
-embed_model_t *
-model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) {
- if (!allow_late && model_dir_has_late_projection(model_dir)) {
- fprintf(stderr, "embed_model_load: late-interaction models require "
- "token-level MaxSim support and are not valid pooled "
- "embedding models yet\n");
- return NULL;
- }
-
- embed_model_t *model = (embed_model_t *)calloc(1, sizeof(embed_model_t));
- if (!model)
- return NULL;
-
- /* Parse config.json */
- if (embed_config_parse(&model->config, model_dir) != 0) {
- free(model);
- return NULL;
- }
-
- const embed_config_t *cfg = &model->config;
- if (layer_end == -1)
- layer_end = cfg->n_layers;
- if (layer_start < 0 || layer_start >= layer_end || layer_end > cfg->n_layers) {
- fprintf(stderr,
- "embed_model_load: invalid layer range [%d, %d) "
- "for %d-layer model\n",
- layer_start, layer_end, cfg->n_layers);
- free(model);
- return NULL;
- }
- model->layer_start = layer_start;
- model->layer_end = layer_end;
- model->attention = cfg->attention_mode == EMBED_ATTENTION_CAUSAL
- ? embed_causal_gqa_attention_packed_with_scratch
- : embed_bidirectional_gqa_attention_packed_with_scratch;
-
- /* Open safetensors (handles single file or multi-shard) */
- multi_safetensors_t *ms = multi_safetensors_open(model_dir);
- if (!ms) {
- fprintf(stderr, "embed_model_load: failed to open safetensors in %s\n", model_dir);
- free(model);
- return NULL;
- }
- model->safetensors = ms;
-
- if (cfg->family == EMBED_FAMILY_BERT) {
- if (load_bert_weights(model, ms, layer_start, layer_end) != 0)
- goto fail;
- if (embed_verbose >= 1)
- fprintf(stderr, "embed_model_load: BERT layers [%d, %d) loaded (%d-dim)\n", layer_start,
- layer_end, cfg->hidden_size);
- return model;
- }
-
- const char *weight_prefix = multi_safetensors_weight_prefix(ms, "embed_tokens.weight");
- if (!weight_prefix) {
- fprintf(stderr, "embed_model_load: neither embed_tokens.weight nor "
- "model.embed_tokens.weight was found\n");
- goto fail;
- }
-
- if (embed_verbose >= 1)
- fprintf(stderr, "embed_model_load: loading weights from %s\n", model_dir);
-
- char name[256];
-
- /* Only the first stage needs token embeddings. */
- embed_weights_t *w = &model->weights;
- if (layer_start == 0) {
- const int64_t embed_shape[2] = {cfg->vocab_size, cfg->hidden_size};
- snprintf(name, sizeof(name), "%sembed_tokens.weight", weight_prefix);
- w->embed_tokens = load_weight_direct(ms, name, embed_shape, 2);
- if (!w->embed_tokens.data)
- goto fail;
- }
-
- /* Allocate layer array */
- w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t));
- if (!w->layers)
- goto fail;
-
- /* Per-layer weights */
- for (int i = layer_start; i < layer_end; i++) {
- embed_layer_t *l = &w->layers[i];
-
-#define LOAD_NORM(field, fmt, d0) \
- do { \
- const int64_t expect[1] = {(d0)}; \
- snprintf(name, sizeof(name), fmt, weight_prefix, i); \
- l->field = load_norm_f32(ms, name, expect, 1); \
- if (!l->field) \
- goto fail; \
- } while (0)
-
-#define LOAD2(field, fmt, d0, d1) \
- do { \
- const int64_t expect[2] = {(d0), (d1)}; \
- snprintf(name, sizeof(name), fmt, weight_prefix, i); \
- l->field = load_weight_direct(ms, name, expect, 2); \
- if (!l->field.data) \
- goto fail; \
- } while (0)
-
- LOAD2(wq, "%slayers.%d.self_attn.q_proj.weight", cfg->q_dim, cfg->hidden_size);
- LOAD2(wk, "%slayers.%d.self_attn.k_proj.weight", cfg->kv_dim, cfg->hidden_size);
- LOAD2(wv, "%slayers.%d.self_attn.v_proj.weight", cfg->kv_dim, cfg->hidden_size);
- LOAD2(wo, "%slayers.%d.self_attn.o_proj.weight", cfg->hidden_size, cfg->q_dim);
- if (cfg->qkv_bias) {
- LOAD_NORM(q_bias, "%slayers.%d.self_attn.q_proj.bias", cfg->q_dim);
- LOAD_NORM(k_bias, "%slayers.%d.self_attn.k_proj.bias", cfg->kv_dim);
- LOAD_NORM(v_bias, "%slayers.%d.self_attn.v_proj.bias", cfg->kv_dim);
- }
- if (cfg->qk_norm) {
- LOAD_NORM(q_norm, "%slayers.%d.self_attn.q_norm.weight", cfg->head_dim);
- LOAD_NORM(k_norm, "%slayers.%d.self_attn.k_norm.weight", cfg->head_dim);
- }
- LOAD_NORM(input_norm, "%slayers.%d.input_layernorm.weight", cfg->hidden_size);
- LOAD_NORM(post_attn_norm, "%slayers.%d.post_attention_layernorm.weight", cfg->hidden_size);
- LOAD2(gate_proj, "%slayers.%d.mlp.gate_proj.weight", cfg->intermediate_size,
- cfg->hidden_size);
- LOAD2(up_proj, "%slayers.%d.mlp.up_proj.weight", cfg->intermediate_size, cfg->hidden_size);
- LOAD2(down_proj, "%slayers.%d.mlp.down_proj.weight", cfg->hidden_size,
- cfg->intermediate_size);
-
-#undef LOAD2
-#undef LOAD_NORM
-
- if (embed_verbose >= 2)
- fprintf(stderr, " layer %d loaded\n", i);
- }
-
- /* Only the final stage applies the output RMSNorm. */
- if (layer_end == cfg->n_layers) {
- const int64_t norm_shape[1] = {cfg->hidden_size};
- snprintf(name, sizeof(name), "%snorm.weight", weight_prefix);
- w->norm = load_norm_f32(ms, name, norm_shape, 1);
- if (!w->norm)
- goto fail;
- }
-
- if (embed_verbose >= 1)
- fprintf(stderr,
- "embed_model_load: layers [%d, %d) loaded "
- "(%d-dim embeddings)\n",
- layer_start, layer_end, cfg->hidden_size);
-
- return model;
-
-fail:
- embed_model_free(model);
- return NULL;
-}
-
-embed_model_t *embed_model_load(const char *model_dir) {
- return model_load_range_ex(model_dir, 0, -1, 0);
-}
-
-/* Release a weight ref's backing store. Only F16-widened refs own their buffer;
- * borrowed F32/BF16 views into the mmap are a no-op (see weight_ref_from_tensor). */
-static void free_weight_ref(embed_weight_ref_t *w) {
- if (w->owned)
- free((void *)w->data);
- w->data = NULL;
- w->owned = 0;
-}
-
-void embed_model_free(embed_model_t *model) {
- if (!model)
- return;
- if (model->weights.layers) {
- for (int i = 0; i < model->config.n_layers; i++) {
- embed_layer_t *l = &model->weights.layers[i];
- free_weight_ref(&l->wq);
- free_weight_ref(&l->wk);
- free_weight_ref(&l->wv);
- free_weight_ref(&l->wo);
- free_weight_ref(&l->gate_proj);
- free_weight_ref(&l->up_proj);
- free_weight_ref(&l->down_proj);
- free((void *)l->q_bias);
- free((void *)l->k_bias);
- free((void *)l->v_bias);
- free((void *)l->q_norm);
- free((void *)l->k_norm);
- free((void *)l->input_norm);
- free((void *)l->post_attn_norm);
- free((void *)l->o_bias);
- free((void *)l->attn_ln_bias);
- free((void *)l->ffn_inter_bias);
- free((void *)l->ffn_out_bias);
- free((void *)l->ffn_ln_bias);
- }
- }
- free_weight_ref(&model->weights.embed_tokens);
- free((void *)model->weights.norm);
- free((void *)model->weights.position_embeddings);
- free((void *)model->weights.token_type_embeddings);
- free((void *)model->weights.embed_ln_w);
- free((void *)model->weights.embed_ln_b);
- if (model->safetensors)
- multi_safetensors_close((multi_safetensors_t *)model->safetensors);
- free(model->weights.layers);
- free(model);
-}
-
-embed_workspace_t *embed_workspace_new(const embed_model_t *model) {
- if (!model)
- return NULL;
- embed_workspace_t *ws = (embed_workspace_t *)calloc(1, sizeof(embed_workspace_t));
- if (!ws)
- return NULL;
- ws->model = model;
- return ws;
-}
-
-void embed_workspace_free(embed_workspace_t *ws) {
- if (!ws)
- return;
- free(ws->x);
- free(ws->x_norm);
- free(ws->q);
- free(ws->k);
- free(ws->v);
- free(ws->attn_out);
- free(ws->proj_out);
- free(ws->ffn_gate);
- free(ws->ffn_up);
- free(ws->attn_scores);
- free(ws->offsets);
- free(ws->rope_cos);
- free(ws->rope_sin);
- free(ws->bf16_widen);
- free(ws);
-}
-
-const embed_config_t *embed_model_config(const embed_model_t *model) {
- return model ? &model->config : NULL;
-}
-
-/* ========================================================================
- * Workspace buffer growth
- *
- * Lazy per-call growth of the forward-pass scratch. Lives here, beside the
- * workspace lifecycle and embed_workspace_nbytes below: nbytes reproduces these
- * allocation sizes, so the allocator and the accountant stay in one file.
- * ======================================================================== */
-
-int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq) {
- if (seq <= ws->buf_seq_cap)
- return 0;
-
- int cap;
- if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0)
- return -1;
-
-#define R(ptr, n) \
- do { \
- if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \
- return -1; \
- } while (0)
-
- R(ws->x, c->hidden_size);
- R(ws->x_norm, c->hidden_size);
- R(ws->q, c->q_dim);
- R(ws->k, c->kv_dim);
- R(ws->v, c->kv_dim);
- R(ws->attn_out, c->q_dim);
- R(ws->proj_out, c->hidden_size);
- R(ws->ffn_gate, c->intermediate_size);
- R(ws->ffn_up, c->intermediate_size);
-
-#undef R
-
- ws->buf_seq_cap = cap;
- return 0;
-}
-
-void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch) {
- size_t bytes = embed_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- if (bytes <= ws->attn_scores_bytes)
- return;
-
- void *p = realloc(ws->attn_scores, bytes);
- if (!p)
- return;
- ws->attn_scores = (float *)p;
- ws->attn_scores_bytes = bytes;
-}
-
-int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos) {
- if (n_pos <= ws->rope_cache_cap)
- return 0;
-
- int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512;
- while (cap < n_pos) {
- if (cap > INT_MAX / 2)
- return -1;
- cap *= 2;
- }
-
- int head_dim = cfg->head_dim;
- int half = head_dim / 2;
- float theta = cfg->rope_theta;
-
- size_t count;
- if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0)
- return -1;
- if (realloc_floats(&ws->rope_cos, count) != 0)
- return -1;
- if (realloc_floats(&ws->rope_sin, count) != 0)
- return -1;
-
- for (int pos = ws->rope_cache_cap; pos < cap; pos++) {
- float fp = (float)pos;
- float *c = ws->rope_cos + (size_t)pos * head_dim;
- float *s = ws->rope_sin + (size_t)pos * head_dim;
- for (int d = 0; d < half; d++) {
- float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
- float angle = fp * inv;
- float cv = cosf(angle), sv = sinf(angle);
- c[d] = cv;
- c[half + d] = cv;
- s[d] = sv;
- s[half + d] = sv;
- }
- }
-
- ws->rope_cache_cap = cap;
- return 0;
-}
-
-int ensure_offsets(embed_workspace_t *ws, int batch) {
- if (!ws || batch <= 0 || batch == INT_MAX)
- return -1;
-
- int needed = batch + 1;
- if (needed <= ws->offsets_cap)
- return 0;
-
- if (realloc_ints(&ws->offsets, (size_t)needed) != 0)
- return -1;
- ws->offsets_cap = needed;
- return 0;
-}
-
-size_t embed_workspace_nbytes(const embed_workspace_t *ws) {
- if (!ws)
- return 0;
-
- size_t total = sizeof(*ws);
- const embed_model_t *model = ws->model;
- if (!model)
- return total;
-
- const embed_config_t *c = &model->config;
- size_t row_floats = 0;
- const int dims[] = {
- c->hidden_size, /* x */
- c->hidden_size, /* x_norm */
- c->q_dim, /* q */
- c->kv_dim, /* k */
- c->kv_dim, /* v */
- c->q_dim, /* attn_out */
- c->hidden_size, /* proj_out */
- c->intermediate_size, /* ffn_gate */
- c->intermediate_size, /* ffn_up */
- };
-
- for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) {
- if (dims[i] < 0 || add_size(row_floats, (size_t)dims[i], &row_floats) != 0)
- return SIZE_MAX;
- }
-
- size_t buf_floats = 0;
- if (ws->buf_seq_cap > 0 && mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0)
- return SIZE_MAX;
-
- size_t rope_floats = 0;
- if (ws->rope_cache_cap > 0) {
- size_t rope_rows = 0;
- if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, &rope_rows) != 0 ||
- mul_size(rope_rows, 2, &rope_floats) != 0)
- return SIZE_MAX;
- }
-
- size_t all_floats = 0, bytes = 0;
- if (add_size(buf_floats, rope_floats, &all_floats) != 0 ||
- mul_size(all_floats, sizeof(float), &bytes) != 0 || add_size(total, bytes, &total) != 0)
- return SIZE_MAX;
-
- size_t offset_bytes = 0;
- if (ws->offsets_cap > 0 && mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0)
- return SIZE_MAX;
- if (add_size(total, offset_bytes, &total) != 0)
- return SIZE_MAX;
- if (add_size(total, ws->attn_scores_bytes, &total) != 0)
- return SIZE_MAX;
-
- size_t widen_bytes = 0;
- if (ws->bf16_widen_count > 0 &&
- mul_size(ws->bf16_widen_count, sizeof(float), &widen_bytes) != 0)
- return SIZE_MAX;
- if (add_size(total, widen_bytes, &total) != 0)
- return SIZE_MAX;
-
- return total;
-}
diff --git a/src/model_internal.h b/src/model_internal.h
deleted file mode 100644
index acbba07..0000000
--- a/src/model_internal.h
+++ /dev/null
@@ -1,76 +0,0 @@
-#ifndef EMBED_MODEL_INTERNAL_H
-#define EMBED_MODEL_INTERNAL_H
-
-#include "internal.h"
-#include "alloc.h"
-
-/* Cross-file function declarations private to the CPU model code. The companion
- * to internal.h: that header holds the shared struct definitions, this one the
- * shared functions. None are part of the public API (embed.h) and none carry
- * EMBED_API visibility; they are shared between the translation units (model
- * loading, the forward pass, and late interaction) that used to live in one
- * file. The frontends and the GPU backends do not include this header. */
-
-/* Validate that a safetensor matches an expected dtype/rank/shape and that its
- * bytes lie within the mapped file. bf16_ok permits either 16-bit float format
- * (BF16 or F16) in addition to F32. */
-int tensor_has_supported_shape(const safetensors_file_t *sf,
- const safetensor_t *t,
- const char *name,
- const int64_t *shape,
- int ndim,
- int bf16_ok);
-
-/* Build a weight ref from a located tensor: F32/BF16 borrow the mmap, F16 is
- * widened to an owned F32 buffer (no fused F16 kernel). Shared by the model
- * loader and the late-interaction projection loader; the owner frees ref.data
- * when ref.owned is set. Returns ref.data == NULL on allocation failure. */
-embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t);
-
-/* True if model_dir carries a 1_Dense projection (a late-interaction model). */
-int model_dir_has_late_projection(const char *model_dir);
-
-/* Load (a layer range of) a model. allow_late lets the late loader accept a
- * 1_Dense snapshot that the pooled loader rejects. layer_end == -1 means all. */
-embed_model_t *
-model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late);
-
-/* Workspace scratch growth, lazy per forward call. Defined in model.c beside the
- * workspace lifecycle (embed_workspace_nbytes mirrors their sizing); called from
- * the forward pass in forward.c. */
-int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq);
-void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch);
-int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos);
-int ensure_offsets(embed_workspace_t *ws, int batch);
-
-/* y[seq,out] = x[seq,in] @ w^T, no bias; dispatches F32/BF16 paths. Shared by
- * the forward pass and the late projection. */
-void linear_nobias_weight(embed_workspace_t *ws,
- float *y,
- const float *x,
- const embed_weight_ref_t *w,
- int seq_len,
- int in_dim,
- int out_dim);
-
-/* Compute packed sequence offsets [batch+1] plus total and max sequence length
- * over a batch of inputs. require_ids rejects inputs with a NULL id pointer. */
-int build_offsets(const embed_input_t *inputs,
- int batch,
- int require_ids,
- int *offsets,
- int *total_out,
- int *max_out);
-
-/* Run the full transformer forward over a packed batch in ws, leaving hidden
- * states in ws->x. Used by pooled encoding and by late-interaction encoding. */
-int forward_packed_inplace(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_input_t *inputs,
- int batch,
- const int *offsets,
- int total_seq,
- int max_seq,
- int apply_final_norm);
-
-#endif /* EMBED_MODEL_INTERNAL_H */
diff --git a/src/safetensors.c b/src/safetensors.c
deleted file mode 100644
index 65d0d7c..0000000
--- a/src/safetensors.c
+++ /dev/null
@@ -1,559 +0,0 @@
-#include "safetensors.h"
-#include "dtype.h"
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <fcntl.h>
-#include <unistd.h>
-#include <sys/mman.h>
-#include <sys/stat.h>
-#include <dirent.h>
-
-/* ========================================================================
- * Minimal JSON parser for safetensors header
- * ======================================================================== */
-
-static void skip_whitespace(const char **p) {
- while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t')
- (*p)++;
-}
-
-static int parse_string(const char **p, char *out, size_t max_len) {
- skip_whitespace(p);
- if (**p != '"')
- return -1;
- (*p)++;
- size_t i = 0;
- while (**p && **p != '"' && i < max_len - 1) {
- if (**p == '\\') {
- (*p)++;
- if (**p == 'n')
- out[i++] = '\n';
- else if (**p == 't')
- out[i++] = '\t';
- else if (**p == '"')
- out[i++] = '"';
- else if (**p == '\\')
- out[i++] = '\\';
- else
- out[i++] = **p;
- } else {
- out[i++] = **p;
- }
- (*p)++;
- }
- out[i] = '\0';
- if (**p != '"')
- return -1;
- (*p)++;
- return 0;
-}
-
-static int parse_int(const char **p, int64_t *out) {
- skip_whitespace(p);
- int64_t val = 0;
- int neg = 0;
- if (**p == '-') {
- neg = 1;
- (*p)++;
- }
- if (**p < '0' || **p > '9')
- return -1;
- while (**p >= '0' && **p <= '9') {
- val = val * 10 + (**p - '0');
- (*p)++;
- }
- *out = neg ? -val : val;
- return 0;
-}
-
-static safetensor_dtype_t parse_dtype(const char *s) {
- if (strcmp(s, "F32") == 0)
- return DTYPE_F32;
- if (strcmp(s, "F16") == 0)
- return DTYPE_F16;
- if (strcmp(s, "BF16") == 0)
- return DTYPE_BF16;
- if (strcmp(s, "I32") == 0)
- return DTYPE_I32;
- if (strcmp(s, "I64") == 0)
- return DTYPE_I64;
- if (strcmp(s, "BOOL") == 0)
- return DTYPE_BOOL;
- return DTYPE_UNKNOWN;
-}
-
-static int parse_tensor_entry(const char **p, safetensor_t *t) {
- skip_whitespace(p);
- if (**p != '{')
- return -1;
- (*p)++;
-
- t->dtype = DTYPE_UNKNOWN;
- t->ndim = 0;
- t->data_offset = 0;
- t->data_size = 0;
-
- while (**p && **p != '}') {
- skip_whitespace(p);
- if (**p == ',') {
- (*p)++;
- continue;
- }
-
- char key[64];
- if (parse_string(p, key, sizeof(key)) != 0)
- return -1;
- skip_whitespace(p);
- if (**p != ':')
- return -1;
- (*p)++;
- skip_whitespace(p);
-
- if (strcmp(key, "dtype") == 0) {
- char dtype_str[32];
- if (parse_string(p, dtype_str, sizeof(dtype_str)) != 0)
- return -1;
- t->dtype = parse_dtype(dtype_str);
- } else if (strcmp(key, "shape") == 0) {
- if (**p != '[')
- return -1;
- (*p)++;
- t->ndim = 0;
- while (**p && **p != ']') {
- skip_whitespace(p);
- if (**p == ',') {
- (*p)++;
- continue;
- }
- if (t->ndim >= 8)
- return -1;
- if (parse_int(p, &t->shape[t->ndim]) != 0)
- return -1;
- t->ndim++;
- }
- if (**p != ']')
- return -1;
- (*p)++;
- } else if (strcmp(key, "data_offsets") == 0) {
- if (**p != '[')
- return -1;
- (*p)++;
- skip_whitespace(p);
- int64_t start_i;
- if (parse_int(p, &start_i) != 0)
- return -1;
- skip_whitespace(p);
- if (**p != ',')
- return -1;
- (*p)++;
- skip_whitespace(p);
- int64_t end_i;
- if (parse_int(p, &end_i) != 0)
- return -1;
- if (start_i < 0 || end_i < start_i)
- return -1;
- t->data_offset = (size_t)start_i;
- t->data_size = (size_t)(end_i - start_i);
- skip_whitespace(p);
- if (**p != ']')
- return -1;
- (*p)++;
- } else {
- /* Skip unknown value */
- if (**p == '"') {
- (*p)++;
- while (**p && **p != '"') {
- if (**p == '\\')
- (*p)++;
- if (**p)
- (*p)++;
- }
- if (**p == '"')
- (*p)++;
- } else if (**p == '[') {
- int depth = 1;
- (*p)++;
- while (**p && depth > 0) {
- if (**p == '[')
- depth++;
- else if (**p == ']')
- depth--;
- (*p)++;
- }
- } else if (**p == '{') {
- int depth = 1;
- (*p)++;
- while (**p && depth > 0) {
- if (**p == '{')
- depth++;
- else if (**p == '}')
- depth--;
- (*p)++;
- }
- } else {
- while (**p && **p != ',' && **p != '}')
- (*p)++;
- }
- }
- }
- if (**p != '}')
- return -1;
- (*p)++;
- return 0;
-}
-
-static int parse_header(safetensors_file_t *sf) {
- const char *p = sf->header_json;
- skip_whitespace(&p);
- if (*p != '{')
- return -1;
- p++;
-
- sf->num_tensors = 0;
-
- while (*p && *p != '}') {
- skip_whitespace(&p);
- if (*p == ',') {
- p++;
- continue;
- }
- if (*p == '}')
- break;
-
- char name[256];
- if (parse_string(&p, name, sizeof(name)) != 0)
- return -1;
- skip_whitespace(&p);
- if (*p != ':')
- return -1;
- p++;
-
- if (strcmp(name, "__metadata__") == 0) {
- skip_whitespace(&p);
- if (*p == '{') {
- int depth = 1;
- p++;
- while (*p && depth > 0) {
- if (*p == '{')
- depth++;
- else if (*p == '}')
- depth--;
- p++;
- }
- }
- continue;
- }
-
- if (sf->num_tensors >= SAFETENSORS_MAX_TENSORS) {
- fprintf(stderr, "safetensors: too many tensors in %s (max %d)\n",
- sf->path ? sf->path : "<unknown>", SAFETENSORS_MAX_TENSORS);
- return -1;
- }
-
- safetensor_t *t = &sf->tensors[sf->num_tensors];
- snprintf(t->name, sizeof(t->name), "%s", name);
- if (parse_tensor_entry(&p, t) != 0)
- return -1;
- sf->num_tensors++;
- }
- if (*p != '}')
- return -1;
- return 0;
-}
-
-/* ========================================================================
- * Single file operations
- * ======================================================================== */
-
-safetensors_file_t *safetensors_open(const char *path) {
- int fd = open(path, O_RDONLY);
- if (fd < 0)
- return NULL;
-
- struct stat st;
- if (fstat(fd, &st) < 0) {
- close(fd);
- return NULL;
- }
-
- size_t file_size = (size_t)st.st_size;
- if (file_size < 8) {
- close(fd);
- return NULL;
- }
-
- void *data = mmap(NULL, file_size, PROT_READ, MAP_PRIVATE, fd, 0);
- close(fd);
- if (data == MAP_FAILED)
- return NULL;
-
- uint64_t header_size = 0;
- memcpy(&header_size, data, 8);
- if (header_size > file_size - 8) {
- munmap(data, file_size);
- return NULL;
- }
-
- safetensors_file_t *sf = calloc(1, sizeof(safetensors_file_t));
- if (!sf) {
- munmap(data, file_size);
- return NULL;
- }
-
- sf->path = strdup(path);
- sf->data = data;
- sf->file_size = file_size;
- sf->header_size = (size_t)header_size;
-
- sf->header_json = malloc(header_size + 1);
- if (!sf->header_json) {
- safetensors_close(sf);
- return NULL;
- }
- memcpy(sf->header_json, (char *)data + 8, header_size);
- sf->header_json[header_size] = '\0';
-
- if (parse_header(sf) != 0) {
- safetensors_close(sf);
- return NULL;
- }
-
- return sf;
-}
-
-void safetensors_close(safetensors_file_t *sf) {
- if (!sf)
- return;
- if (sf->data)
- munmap(sf->data, sf->file_size);
- free(sf->path);
- free(sf->header_json);
- free(sf);
-}
-
-const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t) {
- return (const char *)sf->data + 8 + sf->header_size + t->data_offset;
-}
-
-int64_t safetensor_numel(const safetensor_t *t) {
- int64_t n = 1;
- for (int i = 0; i < t->ndim; i++)
- n *= t->shape[i];
- return n;
-}
-
-float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) {
- int64_t n = safetensor_numel(t);
- if (n <= 0)
- return NULL;
-
- float *out = malloc(n * sizeof(float));
- if (!out)
- return NULL;
-
- const void *data = safetensors_data(sf, t);
-
- switch (t->dtype) {
- case DTYPE_F32:
- memcpy(out, data, n * sizeof(float));
- break;
- case DTYPE_BF16: {
- const uint16_t *src = (const uint16_t *)data;
- for (int64_t i = 0; i < n; i++)
- out[i] = embed_bf16_to_f32(src[i]);
- break;
- }
- case DTYPE_F16: {
- const uint16_t *src = (const uint16_t *)data;
- for (int64_t i = 0; i < n; i++)
- out[i] = embed_f16_to_f32(src[i]);
- break;
- }
- default:
- free(out);
- return NULL;
- }
- return out;
-}
-
-/* ========================================================================
- * Multi-shard operations
- * ======================================================================== */
-
-static int cmp_shard_names(const void *a, const void *b) {
- const char *sa = (const char *)a;
- const char *sb = (const char *)b;
- return strcmp(sa, sb);
-}
-
-static int parse_shard_name(const char *name, int *idx, int *total) {
- int i = 0, n = 0, consumed = 0;
- if (sscanf(name, "model-%d-of-%d.safetensors%n", &i, &n, &consumed) != 2)
- return -1;
- if (name[consumed] != '\0' || i <= 0 || n <= 0 || i > n)
- return -1;
- *idx = i;
- *total = n;
- return 0;
-}
-
-multi_safetensors_t *multi_safetensors_open(const char *model_dir) {
- multi_safetensors_t *ms = calloc(1, sizeof(multi_safetensors_t));
- if (!ms)
- return NULL;
-
- char path[4096];
-
- /* Try single file first */
- snprintf(path, sizeof(path), "%s/model.safetensors", model_dir);
- safetensors_file_t *sf = safetensors_open(path);
- if (sf) {
- ms->shards[0] = sf;
- ms->num_shards = 1;
- return ms;
- }
-
- /* Scan directory for shard files */
- DIR *dir = opendir(model_dir);
- if (!dir) {
- free(ms);
- return NULL;
- }
-
- struct dirent *entry;
- char shard_names[SAFETENSORS_MAX_SHARDS][256];
- int seen[SAFETENSORS_MAX_SHARDS] = {0};
- int n_shards = 0;
- int expected_total = 0;
-
- while ((entry = readdir(dir)) != NULL) {
- if (strncmp(entry->d_name, "model-", 6) == 0 &&
- strstr(entry->d_name, ".safetensors") != NULL) {
- int shard_idx = 0, shard_total = 0;
- if (parse_shard_name(entry->d_name, &shard_idx, &shard_total) != 0) {
- fprintf(stderr, "multi_safetensors_open: bad shard name: %s\n", entry->d_name);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (shard_total > SAFETENSORS_MAX_SHARDS) {
- fprintf(stderr, "multi_safetensors_open: too many shards (%d > %d)\n", shard_total,
- SAFETENSORS_MAX_SHARDS);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (expected_total == 0) {
- expected_total = shard_total;
- } else if (expected_total != shard_total) {
- fprintf(stderr, "multi_safetensors_open: inconsistent shard total in %s\n",
- entry->d_name);
- closedir(dir);
- free(ms);
- return NULL;
- }
- if (seen[shard_idx - 1]) {
- fprintf(stderr, "multi_safetensors_open: duplicate shard index %d\n", shard_idx);
- closedir(dir);
- free(ms);
- return NULL;
- }
- seen[shard_idx - 1] = 1;
-
- snprintf(shard_names[n_shards], sizeof(shard_names[n_shards]), "%s", entry->d_name);
- n_shards++;
- }
- }
- closedir(dir);
-
- if (n_shards == 0) {
- fprintf(stderr, "multi_safetensors_open: no safetensors files in %s\n", model_dir);
- free(ms);
- return NULL;
- }
- if (expected_total != n_shards) {
- fprintf(stderr, "multi_safetensors_open: expected %d shard(s), found %d in %s\n",
- expected_total, n_shards, model_dir);
- free(ms);
- return NULL;
- }
- for (int i = 0; i < expected_total; i++) {
- if (!seen[i]) {
- fprintf(stderr, "multi_safetensors_open: missing shard %d of %d in %s\n", i + 1,
- expected_total, model_dir);
- free(ms);
- return NULL;
- }
- }
-
- /* Sort shard names to ensure consistent ordering */
- qsort(shard_names, n_shards, sizeof(shard_names[0]), cmp_shard_names);
-
- /* Open each shard */
- for (int i = 0; i < n_shards; i++) {
- snprintf(path, sizeof(path), "%s/%s", model_dir, shard_names[i]);
- ms->shards[i] = safetensors_open(path);
- if (!ms->shards[i]) {
- fprintf(stderr, "multi_safetensors_open: failed to open %s\n", path);
- multi_safetensors_close(ms);
- return NULL;
- }
- }
- ms->num_shards = n_shards;
- return ms;
-}
-
-void multi_safetensors_close(multi_safetensors_t *ms) {
- if (!ms)
- return;
- for (int i = 0; i < ms->num_shards; i++) {
- safetensors_close(ms->shards[i]);
- }
- free(ms);
-}
-
-int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, size_t *out_nbytes) {
- if (!ms || !out_nbytes)
- return -1;
- size_t total = 0;
- for (int s = 0; s < ms->num_shards; s++) {
- const safetensors_file_t *sf = ms->shards[s];
- for (int i = 0; i < sf->num_tensors; i++) {
- size_t n = sf->tensors[i].data_size;
- if (n > SIZE_MAX - total)
- return -1;
- total += n;
- }
- }
- *out_nbytes = total;
- return 0;
-}
-
-const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms,
- const char *name,
- safetensors_file_t **out_sf) {
- for (int s = 0; s < ms->num_shards; s++) {
- safetensors_file_t *sf = ms->shards[s];
- for (int i = 0; i < sf->num_tensors; i++) {
- if (strcmp(sf->tensors[i].name, name) == 0) {
- if (out_sf)
- *out_sf = sf;
- return &sf->tensors[i];
- }
- }
- }
- return NULL;
-}
-
-const char *multi_safetensors_weight_prefix(const multi_safetensors_t *ms,
- const char *tensor_name) {
- if (multi_safetensors_find(ms, tensor_name, NULL))
- return "";
-
- char prefixed[sizeof(((safetensor_t *)0)->name)];
- int n = snprintf(prefixed, sizeof(prefixed), "model.%s", tensor_name);
- if (n < 0 || (size_t)n >= sizeof(prefixed))
- return NULL;
- return multi_safetensors_find(ms, prefixed, NULL) ? "model." : NULL;
-}
diff --git a/src/safetensors.h b/src/safetensors.h
deleted file mode 100644
index 8c9b33f..0000000
--- a/src/safetensors.h
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * embed_safetensors.h - Safetensors file format reader (multi-shard support)
- * Adapted from voxtral-realtime project.
- */
-
-#ifndef EMBED_SAFETENSORS_H
-#define EMBED_SAFETENSORS_H
-
-#include <stddef.h>
-#include <stdint.h>
-
-#define SAFETENSORS_MAX_TENSORS 1024
-#define SAFETENSORS_MAX_SHARDS 8
-
-typedef enum {
- DTYPE_F32 = 0,
- DTYPE_F16 = 1,
- DTYPE_BF16 = 2,
- DTYPE_I32 = 3,
- DTYPE_I64 = 4,
- DTYPE_BOOL = 5,
- DTYPE_UNKNOWN = -1
-} safetensor_dtype_t;
-
-typedef struct {
- char name[256];
- safetensor_dtype_t dtype;
- int ndim;
- int64_t shape[8];
- size_t data_offset;
- size_t data_size;
-} safetensor_t;
-
-typedef struct {
- char *path;
- void *data;
- size_t file_size;
- size_t header_size;
- char *header_json;
- int num_tensors;
- safetensor_t tensors[SAFETENSORS_MAX_TENSORS];
-} safetensors_file_t;
-
-/* Multi-shard wrapper: opens all shard files and provides unified tensor lookup */
-typedef struct {
- safetensors_file_t *shards[SAFETENSORS_MAX_SHARDS];
- int num_shards;
-} multi_safetensors_t;
-
-/* Open a single safetensors file (memory-mapped) */
-safetensors_file_t *safetensors_open(const char *path);
-void safetensors_close(safetensors_file_t *sf);
-
-/* Open model from directory (auto-detects single file or multi-shard) */
-multi_safetensors_t *multi_safetensors_open(const char *model_dir);
-void multi_safetensors_close(multi_safetensors_t *ms);
-int multi_safetensors_data_nbytes(const multi_safetensors_t *ms, size_t *out_nbytes);
-
-/* Find a tensor by name across all shards */
-const safetensor_t *multi_safetensors_find(const multi_safetensors_t *ms,
- const char *name,
- safetensors_file_t **out_sf);
-
-/* Resolve an optional Hugging Face "model." namespace for a backbone tensor.
- * Returns "", "model.", or NULL when neither form exists. */
-const char *multi_safetensors_weight_prefix(const multi_safetensors_t *ms, const char *tensor_name);
-
-/* Get raw pointer to tensor data (within mmap'd region) */
-const void *safetensors_data(const safetensors_file_t *sf, const safetensor_t *t);
-
-/* Get tensor data as float32 (allocates, caller must free) */
-float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t);
-
-int64_t safetensor_numel(const safetensor_t *t);
-
-#endif /* EMBED_SAFETENSORS_H */
diff --git a/src/sbuf.c b/src/sbuf.c
deleted file mode 100644
index f286d8c..0000000
--- a/src/sbuf.c
+++ /dev/null
@@ -1,74 +0,0 @@
-#include "sbuf.h"
-#include "server_util.h"
-
-#include <stdarg.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-void sbuf_reserve(sbuf *b, size_t add) {
- if (add > SIZE_MAX - b->len - 1)
- die_oom();
- size_t need = b->len + add + 1;
- if (need <= b->cap)
- return;
- size_t cap = b->cap ? b->cap : 256;
- while (cap < need) {
- if (cap > SIZE_MAX / 2) {
- cap = need;
- break;
- }
- cap *= 2;
- }
- b->ptr = xrealloc(b->ptr, cap);
- b->cap = cap;
-}
-
-void sbuf_append(sbuf *b, const void *p, size_t n) {
- sbuf_reserve(b, n);
- memcpy(b->ptr + b->len, p, n);
- b->len += n;
- b->ptr[b->len] = '\0';
-}
-
-void sbuf_putc(sbuf *b, char c) {
- sbuf_reserve(b, 1);
- b->ptr[b->len++] = c;
- b->ptr[b->len] = '\0';
-}
-
-void sbuf_puts(sbuf *b, const char *s) { sbuf_append(b, s, strlen(s)); }
-
-void sbuf_clear(sbuf *b) {
- b->len = 0;
- if (b->ptr)
- b->ptr[0] = '\0';
-}
-
-void sbuf_printf(sbuf *b, const char *fmt, ...) {
- va_list ap;
- va_start(ap, fmt);
- for (;;) {
- size_t avail = b->cap > b->len ? b->cap - b->len : 0;
- va_list aq;
- va_copy(aq, ap);
- int n = vsnprintf(avail ? b->ptr + b->len : NULL, avail, fmt, aq);
- va_end(aq);
- if (n < 0) {
- va_end(ap);
- die_oom();
- }
- if ((size_t)n < avail) {
- b->len += (size_t)n;
- va_end(ap);
- return;
- }
- sbuf_reserve(b, (size_t)n);
- }
-}
-
-void sbuf_free(sbuf *b) {
- free(b->ptr);
- memset(b, 0, sizeof(*b));
-}
diff --git a/src/sbuf.h b/src/sbuf.h
deleted file mode 100644
index 0c8929e..0000000
--- a/src/sbuf.h
+++ /dev/null
@@ -1,24 +0,0 @@
-#ifndef EMBED_SBUF_H
-#define EMBED_SBUF_H
-
-#include <stddef.h>
-
-/* Growable, always-NUL-terminated byte buffer used to build HTTP responses and
- * JSON bodies. Growth is die-on-OOM (see server_util.h), so the mutators never
- * fail; callers do not check return values. */
-
-typedef struct {
- char *ptr;
- size_t len;
- size_t cap;
-} sbuf;
-
-void sbuf_reserve(sbuf *b, size_t add);
-void sbuf_append(sbuf *b, const void *p, size_t n);
-void sbuf_putc(sbuf *b, char c);
-void sbuf_puts(sbuf *b, const char *s);
-void sbuf_clear(sbuf *b);
-void sbuf_printf(sbuf *b, const char *fmt, ...) __attribute__((format(printf, 2, 3)));
-void sbuf_free(sbuf *b);
-
-#endif /* EMBED_SBUF_H */
diff --git a/src/server.c b/src/server.c
deleted file mode 100644
index 808c6f8..0000000
--- a/src/server.c
+++ /dev/null
@@ -1,1042 +0,0 @@
-#ifndef _GNU_SOURCE
-#define _GNU_SOURCE
-#endif
-
-#include "build.h"
-#include "server.h"
-#include "server_internal.h"
-#include "server_util.h"
-#include "sbuf.h"
-#include "embed.h"
-#include "safetensors.h"
-#include "kernels.h"
-#include "tokenizer_bpe.h"
-#include "tokenizer_wordpiece.h"
-
-#ifdef USE_MLX
-#include "mlx.h"
-#endif
-#ifdef USE_CUDA
-#include "cuda.h"
-#endif
-
-#include "deps/ae/ae.h"
-#include "deps/ae/anet.h"
-
-#include <arpa/inet.h>
-#include <ctype.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <limits.h>
-#include <netinet/in.h>
-#include <pthread.h>
-#include <signal.h>
-#include <stdbool.h>
-#include <stdint.h>
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <strings.h>
-#include <sys/socket.h>
-#include <sys/time.h>
-#include <sys/types.h>
-#include <time.h>
-#include <unistd.h>
-
-#ifdef __APPLE__
-#include <sys/sysctl.h>
-#endif
-
-#include <cjson/cJSON.h>
-
-static volatile sig_atomic_t g_stop_requested = 0;
-static volatile sig_atomic_t g_signal_wfd = -1;
-
-static void stop_signal_handler(int sig) {
- (void)sig;
- if (g_stop_requested)
- _exit(130);
- int saved = errno;
- g_stop_requested = 1;
- int fd = (int)g_signal_wfd;
- if (fd >= 0) {
- const unsigned char byte = 1;
- if (write(fd, &byte, 1) < 0) {
- /* best-effort stop wakeup; a failed self-pipe write is ignored */
- }
- }
- errno = saved;
-}
-
-static bool auth_ok(http_server *s, const char *auth) {
- if (!s->api_key || !s->api_key[0])
- return true;
- if (!auth)
- return false;
- const char prefix[] = "Bearer ";
- return !strncmp(auth, prefix, sizeof(prefix) - 1) &&
- !strcmp(auth + sizeof(prefix) - 1, s->api_key);
-}
-
-void dispatch_request(client *c) {
- aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
-
- if (!strcmp(c->req.method, "OPTIONS")) {
- append_http_response(c, 204, NULL, "", 0);
- if (queue_write(c) != AE_OK)
- close_client(c);
- return;
- }
-
- if (!route_is_inference(c->req.method, c->req.path)) {
- size_t len;
- char *body = json_error_body("unknown endpoint", "invalid_request_error", &len);
- append_http_response(c, 404, "application/json", body, len);
- free(body);
- if (queue_write(c) != AE_OK)
- close_client(c);
- return;
- }
-
- if (!auth_ok(c->srv, c->req.auth)) {
- size_t len;
- char *body =
- json_error_body("missing or invalid bearer token", "authentication_error", &len);
- append_http_response(c, 401, "application/json", body, len);
- free(body);
- if (queue_write(c) != AE_OK)
- close_client(c);
- return;
- }
-
- job *j = xcalloc(1, sizeof(*j));
- j->srv = c->srv;
- j->c = c;
- snprintf(j->method, sizeof(j->method), "%s", c->req.method);
- snprintf(j->path, sizeof(j->path), "%s", c->req.path);
- j->body = xstrndup(c->req.body ? c->req.body : "", c->req.body_len);
- j->body_len = c->req.body_len;
- j->auth = c->req.auth ? xstrdup(c->req.auth) : NULL;
- j->created_ns = nstime();
- client_incref(c);
- enqueue_job(j);
-}
-
-static void accept_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop;
- (void)mask;
- http_server *s = clientData;
- char cip[64];
- int cport = 0;
- for (;;) {
- int cfd = anetTcpAccept(NULL, fd, cip, sizeof(cip), &cport);
- if (cfd == ANET_ERR) {
- if (errno == EAGAIN || errno == EWOULDBLOCK)
- return;
- if (errno == EINTR)
- continue;
- return;
- }
- set_nonblock(cfd);
- client *c = xcalloc(1, sizeof(*c));
- c->srv = s;
- c->fd = cfd;
- c->refcount = 1;
- c->last_active_ms = mstime();
- c->linked = 1;
- c->next = s->clients;
- if (s->clients)
- s->clients->prev = c;
- s->clients = c;
- s->n_clients++;
- if (aeCreateFileEvent(s->loop, cfd, AE_READABLE, read_cb, c) != AE_OK)
- close_client(c);
- }
-}
-
-static int timeout_cb(aeEventLoop *loop, long long id, void *clientData) {
- (void)loop;
- (void)id;
- http_server *s = clientData;
- uint64_t now = mstime();
- client *c = s->clients;
- while (c) {
- client *next = c->next;
- /* refcount > 1 means a request from this client is queued or being
- * processed by the worker; long batch queues (e.g. 4B long-document
- * batches) must not be reaped as idle. */
- if (!c->cancelled && c->refcount <= 1 &&
- now - c->last_active_ms > EMBED_HTTP_CLIENT_TIMEOUT_MS)
- close_client(c);
- c = next;
- }
- return EMBED_HTTP_SWEEP_MS;
-}
-
-static void signal_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)fd;
- (void)mask;
- http_server *s = clientData;
- unsigned char tmp[128];
- while (read(s->signal_pipe[0], tmp, sizeof(tmp)) > 0) {
- }
- aeStop(loop);
-}
-
-static void *worker_main(void *arg) {
- http_server *s = arg;
-#ifdef USE_MLX
- if (s->use_mlx) {
- int rc = 0;
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- embed_mlx_options_t mlx_opts = {
- .quantize_bits = s->mlx_quantize_bits,
- .quantize_group_size = s->mlx_quantize_group_size,
- };
- if (m->info->kind == MODEL_KIND_LATE)
- m->mlx_late_ctx = embed_mlx_late_load_with_options(m->path, &mlx_opts);
- else
- m->mlx_ctx = embed_mlx_load_with_options(m->path, &mlx_opts);
- if ((!m->mlx_ctx && m->info->kind != MODEL_KIND_LATE) ||
- (!m->mlx_late_ctx && m->info->kind == MODEL_KIND_LATE)) {
- server_log("embed-server: failed to load MLX model on worker: %s", m->path);
- rc = 1;
- break;
- }
- const embed_config_t *config = m->info->kind == MODEL_KIND_LATE
- ? embed_mlx_late_config(m->mlx_late_ctx)
- : embed_mlx_config(m->mlx_ctx);
- int token_dim =
- m->info->kind == MODEL_KIND_LATE ? embed_mlx_late_token_dim(m->mlx_late_ctx) : 0;
- if (configure_loaded_model(m, config, token_dim) != 0 ||
- (m->info->kind == MODEL_KIND_LATE &&
- (m->late_mask_id >= config->vocab_size ||
- m->late_query_prefix_id >= config->vocab_size ||
- m->late_document_prefix_id >= config->vocab_size))) {
- rc = 1;
- break;
- }
- }
- pthread_mutex_lock(&s->mu);
- s->worker_init_rc = rc;
- s->worker_ready = 1;
- if (rc)
- s->stopping = 1;
- pthread_cond_broadcast(&s->cv);
- pthread_mutex_unlock(&s->mu);
- if (rc) {
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- if (m->mlx_ctx) {
- embed_mlx_free(m->mlx_ctx);
- m->mlx_ctx = NULL;
- }
- if (m->mlx_late_ctx) {
- embed_mlx_late_free(m->mlx_late_ctx);
- m->mlx_late_ctx = NULL;
- }
- }
- return NULL;
- }
- } else
-#endif
-#ifdef USE_CUDA
- if (s->use_cuda) {
- int rc = 0;
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- if (m->info->kind == MODEL_KIND_LATE)
- m->cuda_late_ctx = embed_cuda_late_load(m->path);
- else
- m->cuda_ctx = embed_cuda_load(m->path);
- if ((!m->cuda_ctx && m->info->kind != MODEL_KIND_LATE) ||
- (!m->cuda_late_ctx && m->info->kind == MODEL_KIND_LATE)) {
- server_log("embed-server: failed to load CUDA model on worker: %s", m->path);
- rc = 1;
- break;
- }
- const embed_config_t *config = m->info->kind == MODEL_KIND_LATE
- ? embed_cuda_late_config(m->cuda_late_ctx)
- : embed_cuda_config(m->cuda_ctx);
- int token_dim =
- m->info->kind == MODEL_KIND_LATE ? embed_cuda_late_token_dim(m->cuda_late_ctx) : 0;
- if (configure_loaded_model(m, config, token_dim) != 0 ||
- (m->info->kind == MODEL_KIND_LATE &&
- (m->late_mask_id >= config->vocab_size ||
- m->late_query_prefix_id >= config->vocab_size ||
- m->late_document_prefix_id >= config->vocab_size))) {
- rc = 1;
- break;
- }
- }
- pthread_mutex_lock(&s->mu);
- s->worker_init_rc = rc;
- s->worker_ready = 1;
- if (rc)
- s->stopping = 1;
- pthread_cond_broadcast(&s->cv);
- pthread_mutex_unlock(&s->mu);
- if (rc) {
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- if (m->cuda_ctx) {
- embed_cuda_free(m->cuda_ctx);
- m->cuda_ctx = NULL;
- }
- if (m->cuda_late_ctx) {
- embed_cuda_late_free(m->cuda_late_ctx);
- m->cuda_late_ctx = NULL;
- }
- }
- return NULL;
- }
- } else
-#endif
- {
- pthread_mutex_lock(&s->mu);
- s->worker_ready = 1;
- pthread_cond_broadcast(&s->cv);
- pthread_mutex_unlock(&s->mu);
- }
-
- for (;;) {
- job *jobs[EMBED_SERVER_MICROBATCH_MAX_JOBS];
- int n_jobs = collect_job_batch(s, jobs, EMBED_SERVER_MICROBATCH_MAX_JOBS);
- if (n_jobs == 0)
- break;
- process_job_group(s, jobs, n_jobs);
- }
-#ifdef USE_MLX
- if (s->use_mlx) {
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- if (m->mlx_ctx) {
- embed_mlx_free(m->mlx_ctx);
- m->mlx_ctx = NULL;
- }
- if (m->mlx_late_ctx) {
- embed_mlx_late_free(m->mlx_late_ctx);
- m->mlx_late_ctx = NULL;
- }
- }
- }
-#endif
-#ifdef USE_CUDA
- if (s->use_cuda) {
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- if (m->cuda_ctx) {
- embed_cuda_free(m->cuda_ctx);
- m->cuda_ctx = NULL;
- }
- if (m->cuda_late_ctx) {
- embed_cuda_late_free(m->cuda_late_ctx);
- m->cuda_late_ctx = NULL;
- }
- }
- }
-#endif
- return NULL;
-}
-
-static int listen_on(const char *host, int port) {
- char portbuf[32];
- snprintf(portbuf, sizeof(portbuf), "%d", port);
- char err[ANET_ERR_LEN];
- int fd = anetTcpServer(err, port, (char *)host, EMBED_HTTP_BACKLOG);
- (void)portbuf;
- if (fd == ANET_ERR) {
- server_log("embed-server: listen failed on %s:%d: %s", host, port, err);
- return -1;
- }
- set_nonblock(fd);
- return fd;
-}
-
-static uint64_t physical_memory_nbytes(void) {
-#ifdef __APPLE__
- uint64_t bytes = 0;
- size_t len = sizeof(bytes);
- if (sysctlbyname("hw.memsize", &bytes, &len, NULL, 0) == 0 && len == sizeof(bytes) && bytes > 0)
- return bytes;
-#endif
- long pages = sysconf(_SC_PHYS_PAGES);
- long page_size = sysconf(_SC_PAGESIZE);
- if (pages <= 0 || page_size <= 0 || (uint64_t)pages > UINT64_MAX / (uint64_t)page_size)
- return 0;
- return (uint64_t)pages * (uint64_t)page_size;
-}
-
-static int validate_model_specs(const embed_server_config_t *cfg) {
- for (int i = 0; i < cfg->n_models; i++) {
- const embed_server_model_spec_t *spec = &cfg->models[i];
- if (!spec->id || !spec->id[0]) {
- fprintf(stderr, "embed-server: model label must not be empty\n");
- return -1;
- }
- if (!spec->path || !spec->path[0]) {
- fprintf(stderr, "embed-server: model %s has an empty path\n", spec->id);
- return -1;
- }
- if (spec->kind < EMBED_SERVER_MODEL_STANDARD || spec->kind > EMBED_SERVER_MODEL_LATE) {
- fprintf(stderr, "embed-server: model %s has an invalid kind\n", spec->id);
- return -1;
- }
- if (spec->api < EMBED_SERVER_API_DEFAULT || spec->api > EMBED_SERVER_API_PERPLEXITY) {
- fprintf(stderr, "embed-server: model %s has an invalid api\n", spec->id);
- return -1;
- }
- if (spec->min_dim < 0) {
- fprintf(stderr, "embed-server: model %s has an invalid min_dim\n", spec->id);
- return -1;
- }
- for (int j = 0; j < i; j++) {
- if (spec->id && cfg->models[j].id && !strcmp(cfg->models[j].id, spec->id)) {
- fprintf(stderr, "embed-server: duplicate model label: %s\n", spec->id);
- return -1;
- }
- }
- }
- return 0;
-}
-
-static int mlx_memory_preflight(const embed_server_config_t *cfg) {
- if (!cfg->use_mlx)
- return 0;
-
- uint64_t payload = 0;
- for (int i = 0; i < cfg->n_models; i++) {
- multi_safetensors_t *ms = multi_safetensors_open(cfg->models[i].path);
- size_t model_payload = 0;
- if (!ms || multi_safetensors_data_nbytes(ms, &model_payload) != 0) {
- multi_safetensors_close(ms);
- fprintf(stderr, "embed-server: failed to inspect MLX model: %s\n", cfg->models[i].path);
- return -1;
- }
- multi_safetensors_close(ms);
- if ((uint64_t)model_payload > UINT64_MAX - payload) {
- fprintf(stderr, "embed-server: MLX model payload size overflow\n");
- return -1;
- }
- payload += (uint64_t)model_payload;
- }
-
- if (payload > UINT64_MAX / EMBED_MLX_RESIDENT_MULTIPLIER) {
- fprintf(stderr, "embed-server: MLX resident-memory estimate overflow\n");
- return -1;
- }
- uint64_t estimated = payload * EMBED_MLX_RESIDENT_MULTIPLIER;
- uint64_t physical = physical_memory_nbytes();
- if (physical == 0) {
- server_log("embed-server: warning: could not determine physical memory; "
- "skipping MLX memory preflight");
- return 0;
- }
- double utilization = cfg->memory_utilization > 0.0 ? cfg->memory_utilization
- : EMBED_MLX_MEMORY_BUDGET_PERCENT / 100.0;
- uint64_t budget = (uint64_t)((double)physical * utilization);
- const double gib = 1024.0 * 1024.0 * 1024.0;
- server_log("embed-server: MLX memory preflight: %.1f GiB tensors, "
- "%.1f GiB estimated resident, %.1f GiB budget "
- "(%.2f of physical memory)",
- (double)payload / gib, (double)estimated / gib, (double)budget / gib, utilization);
- if (cfg->mlx_quantize_bits) {
- server_log("embed-server: MLX %d-bit quantization enabled; preflight "
- "uses source payload as a conservative peak estimate",
- cfg->mlx_quantize_bits);
- }
- if (estimated <= budget)
- return 0;
- server_log("embed-server: refusing MLX model set above the host-memory "
- "budget");
- server_log("embed-server: use BF16 artifacts, load fewer models, or raise "
- "--memory-utilization (values above 1.0 overcommit)");
- return -1;
-}
-
-int embed_run_server(const embed_server_config_t *cfg) {
- if (!cfg || !cfg->models || cfg->n_models <= 0) {
- fprintf(stderr, "embed-server: at least one model label and path is required\n");
- return 1;
- }
- if (validate_model_specs(cfg) != 0 || mlx_memory_preflight(cfg) != 0)
- return 1;
-
- http_server s;
- memset(&s, 0, sizeof(s));
- s.host = cfg->host && cfg->host[0] ? cfg->host : "127.0.0.1";
- s.port = cfg->port > 0 ? cfg->port : 8000;
- s.batch_size = cfg->batch_size > 0 ? cfg->batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE;
- s.max_batch_tokens =
- cfg->max_batch_tokens > 0 ? cfg->max_batch_tokens : EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS;
- s.batch_wait_us = cfg->batch_wait_us >= 0 ? cfg->batch_wait_us
- : (cfg->use_cuda ? EMBED_SERVER_CUDA_BATCH_WAIT_US
- : EMBED_SERVER_BATCH_WAIT_US);
- s.use_mlx = cfg->use_mlx;
- s.use_cuda = cfg->use_cuda;
- s.mlx_quantize_bits = cfg->mlx_quantize_bits;
- s.mlx_quantize_group_size =
- cfg->mlx_quantize_group_size > 0 ? cfg->mlx_quantize_group_size : 64;
- if (cfg->api_key && cfg->api_key[0])
- s.api_key = xstrdup(cfg->api_key);
- else {
- const char *env_key = getenv("EMBED_API_KEY");
- if (env_key && env_key[0])
- s.api_key = xstrdup(env_key);
- }
-
- pthread_mutex_init(&s.mu, NULL);
- pthread_cond_init(&s.cv, NULL);
-
- for (int i = 0; i < cfg->n_models; i++) {
- if (load_one_model(&s, &cfg->models[i]) != 0) {
- free_models(&s);
- return 1;
- }
- }
-
- if (pipe(s.completion_pipe) != 0 || pipe(s.signal_pipe) != 0) {
- perror("pipe");
- free_models(&s);
- return 1;
- }
- set_nonblock(s.completion_pipe[0]);
- set_nonblock(s.completion_pipe[1]);
- set_nonblock(s.signal_pipe[0]);
- set_nonblock(s.signal_pipe[1]);
-
- struct sigaction sa;
- memset(&sa, 0, sizeof(sa));
- sa.sa_handler = stop_signal_handler;
- sigemptyset(&sa.sa_mask);
- sigaction(SIGINT, &sa, NULL);
- sigaction(SIGTERM, &sa, NULL);
- signal(SIGPIPE, SIG_IGN);
- g_signal_wfd = s.signal_pipe[1];
-
- s.loop = aeCreateEventLoop(EMBED_HTTP_SETSIZE);
- if (!s.loop) {
- fprintf(stderr, "embed-server: failed to create event loop\n");
- free_models(&s);
- return 1;
- }
-
- if (pthread_create(&s.worker, NULL, worker_main, &s) != 0) {
- fprintf(stderr, "embed-server: failed to start worker\n");
- aeDeleteEventLoop(s.loop);
- free_models(&s);
- return 1;
- }
- pthread_mutex_lock(&s.mu);
- while (!s.worker_ready)
- pthread_cond_wait(&s.cv, &s.mu);
- int worker_init_rc = s.worker_init_rc;
- pthread_mutex_unlock(&s.mu);
- if (worker_init_rc) {
- pthread_join(s.worker, NULL);
- aeDeleteEventLoop(s.loop);
- free_models(&s);
- free(s.api_key);
- pthread_cond_destroy(&s.cv);
- pthread_mutex_destroy(&s.mu);
- return 1;
- }
-
- s.listen_fd = listen_on(s.host, s.port);
- if (s.listen_fd < 0) {
- pthread_mutex_lock(&s.mu);
- s.stopping = 1;
- pthread_cond_signal(&s.cv);
- pthread_mutex_unlock(&s.mu);
- pthread_join(s.worker, NULL);
- aeDeleteEventLoop(s.loop);
- free_models(&s);
- return 1;
- }
-
- aeCreateFileEvent(s.loop, s.listen_fd, AE_READABLE, accept_cb, &s);
- aeCreateFileEvent(s.loop, s.completion_pipe[0], AE_READABLE, completion_cb, &s);
- aeCreateFileEvent(s.loop, s.signal_pipe[0], AE_READABLE, signal_cb, &s);
- aeCreateTimeEvent(s.loop, EMBED_HTTP_SWEEP_MS, timeout_cb, &s, NULL);
-
- server_log("embed-server: listening on http://%s:%d", s.host, s.port);
- aeMain(s.loop);
-
- server_log("embed-server: shutdown requested");
- if (s.listen_fd >= 0)
- close(s.listen_fd);
- client *c = s.clients;
- while (c) {
- client *next = c->next;
- close_client(c);
- c = next;
- }
-
- pthread_mutex_lock(&s.mu);
- s.stopping = 1;
- pthread_cond_signal(&s.cv);
- pthread_mutex_unlock(&s.mu);
- pthread_join(s.worker, NULL);
-
- aeDeleteEventLoop(s.loop);
- close(s.completion_pipe[0]);
- close(s.completion_pipe[1]);
- close(s.signal_pipe[0]);
- close(s.signal_pipe[1]);
- free_models(&s);
- free(s.api_key);
- pthread_cond_destroy(&s.cv);
- pthread_mutex_destroy(&s.mu);
- return 0;
-}
-
-/* ====================================================================
- * embed-server entry point
- * ==================================================================== */
-#ifndef EMBED_SERVER_TEST
-
-static void print_usage(const char *prog) {
- fprintf(stderr,
- "Usage: %s --model LABEL=DIR [options]\n"
- "\n"
- "Options:\n"
- " --model LABEL=DIR[:key=val]\n"
- " Standard embedding model (repeatable)\n"
- " --contextual-model LABEL=DIR[:key=val]\n"
- " Contextual embedding model (repeatable)\n"
- " --late-model LABEL=DIR[:key=val]\n"
- " Late-interaction rerank model (repeatable)\n"
- " keys: api=openai|perplexity, min_dim=N,\n"
- " query=TEXT (last key; \\n escapes newline)\n"
-#ifdef USE_MLX
- " --mlx Use Apple MLX GPU backend\n"
-#endif
-#ifdef USE_CUDA
- " --cuda Use NVIDIA CUDA GPU backend\n"
-#endif
- " --host HOST Bind host (default: 127.0.0.1)\n"
- " --port N Bind port (default: 8000)\n"
- " --api-key K Require Authorization: Bearer K\n"
- " -b, --batch-size N Max texts or documents per inference batch\n"
- " (default: 32)\n"
- " --max-batch-tokens N Max tokens per inference batch (default: 16384)\n"
- " --batch-wait-us N First-arrival micro-batch deadline in us\n"
-#ifdef USE_CUDA
- " (default: CUDA 1000; CPU 0)\n"
-#else
- " (default: 0)\n"
-#endif
-#ifdef USE_MLX
- " --memory-utilization F Fraction of physical memory the MLX model-set\n"
- " preflight may plan for (default: 0.90; values\n"
- " above 1.0 overcommit)\n"
- " --mlx-quant-bits N Quantize MLX linear weights to 8 bits at load\n"
- " --mlx-quant-group-size N MLX quantization group size (default: 64)\n"
-#endif
-#ifdef USE_CUDA
- " --cuda-gemm-mode MODE CUDA GEMM compute: f32, tf32, bf16, or 16f\n"
- " (default: f32, exact)\n"
- " --cuda-weight-dtype DTYPE CUDA weight storage: f32 or bf16 (default:\n"
- " f32); bf16 halves weight memory\n"
-#endif
- " -t, --threads N CPU threads (default: available cores, cgroup-aware)\n"
- " -v, --verbose Verbose (-vv for debug)\n"
- " -V, --version Print version and exit\n"
- " --build-info Print build details and exit\n"
- " -h, --help Show this help\n"
- "\n"
- "Examples:\n"
- " %s --model docs=./model --port 8000\n"
- " %s --model pplx=./model:api=perplexity:min_dim=128 --port 8000\n"
-#ifdef USE_MLX
- " %s --mlx --mlx-quant-bits 8 \\\n"
- " --model pplx=./model-4b-bf16:api=perplexity:min_dim=128\n",
- prog, prog, prog, prog);
-#else
- ,
- prog, prog, prog);
-#endif
-}
-
-typedef struct {
- embed_server_model_spec_t *v;
- int n;
- int cap;
-} model_specs_t;
-
-static char *dup_range(const char *p, size_t n) {
- char *s = (char *)malloc(n + 1);
- if (!s)
- return NULL;
- memcpy(s, p, n);
- s[n] = '\0';
- return s;
-}
-
-static char *unescape_model_option(const char *s) {
- size_t n = strlen(s);
- char *out = (char *)malloc(n + 1);
- if (!out)
- return NULL;
- size_t w = 0;
- for (size_t r = 0; r < n; r++) {
- if (s[r] == '\\' && r + 1 < n) {
- r++;
- if (s[r] == 'n')
- out[w++] = '\n';
- else if (s[r] == 't')
- out[w++] = '\t';
- else if (s[r] == 'r')
- out[w++] = '\r';
- else
- out[w++] = s[r];
- } else {
- out[w++] = s[r];
- }
- }
- out[w] = '\0';
- return out;
-}
-
-static int parse_model_option(embed_server_model_spec_t *spec, const char *opt, const char *flag) {
- const char *eq = strchr(opt, '=');
- if (!eq || eq == opt || !eq[1]) {
- fprintf(stderr, "%s option expects key=value: %s\n", flag, opt);
- return -1;
- }
- size_t key_len = (size_t)(eq - opt);
- const char *val = eq + 1;
- if (key_len == 3 && strncmp(opt, "api", key_len) == 0) {
- if (!strcmp(val, "openai")) {
- spec->api = EMBED_SERVER_API_OPENAI;
- } else if (!strcmp(val, "perplexity") || !strcmp(val, "pplx")) {
- spec->api = EMBED_SERVER_API_PERPLEXITY;
- } else if (!strcmp(val, "default")) {
- spec->api = EMBED_SERVER_API_DEFAULT;
- } else {
- fprintf(stderr, "%s api must be openai or perplexity\n", flag);
- return -1;
- }
- return 0;
- }
- if (key_len == 7 && strncmp(opt, "min_dim", key_len) == 0) {
- char *end = NULL;
- long v = strtol(val, &end, 10);
- if (!end || *end || v <= 0 || v > INT_MAX) {
- fprintf(stderr, "%s min_dim must be a positive integer\n", flag);
- return -1;
- }
- spec->min_dim = (int)v;
- return 0;
- }
- fprintf(stderr, "%s unknown model option: %.*s\n", flag, (int)key_len, opt);
- return -1;
-}
-
-static int append_model_spec(model_specs_t *specs,
- const char *arg,
- embed_server_model_kind_t kind,
- const char *flag) {
- const char *eq = strchr(arg, '=');
- if (!eq || eq == arg || !eq[1]) {
- fprintf(stderr, "%s expects LABEL=DIR\n", flag);
- return -1;
- }
- if (specs->n == specs->cap) {
- int new_cap = specs->cap ? specs->cap * 2 : 4;
- void *p = realloc(specs->v, (size_t)new_cap * sizeof(specs->v[0]));
- if (!p)
- return -1;
- specs->v = p;
- specs->cap = new_cap;
- }
- size_t id_len = (size_t)(eq - arg);
- const char *path_start = eq + 1;
- const char *opts = strchr(path_start, ':');
- size_t path_len = opts ? (size_t)(opts - path_start) : strlen(path_start);
- if (path_len == 0) {
- fprintf(stderr, "%s expects a non-empty DIR\n", flag);
- return -1;
- }
- embed_server_model_spec_t spec;
- memset(&spec, 0, sizeof(spec));
- spec.id = dup_range(arg, id_len);
- spec.path = dup_range(path_start, path_len);
- spec.kind = kind;
- if (!spec.id || !spec.path) {
- free((char *)spec.id);
- free((char *)spec.path);
- return -1;
- }
- const char *p = opts ? opts + 1 : NULL;
- while (p && *p) {
- if (!strncmp(p, "query=", 6) || !strncmp(p, "query_instruct=", 15)) {
- const char *val = p + (!strncmp(p, "query=", 6) ? 6 : 15);
- spec.query_instruct = unescape_model_option(val);
- if (!spec.query_instruct)
- goto fail;
- break;
- }
- const char *next = strchr(p, ':');
- size_t opt_len = next ? (size_t)(next - p) : strlen(p);
- char *opt = dup_range(p, opt_len);
- if (!opt)
- goto fail;
- int rc = parse_model_option(&spec, opt, flag);
- free(opt);
- if (rc != 0)
- goto fail;
- p = next ? next + 1 : NULL;
- }
- specs->v[specs->n] = spec;
- specs->n++;
- return 0;
-
-fail:
- free((char *)spec.id);
- free((char *)spec.path);
- free((char *)spec.query_instruct);
- return -1;
-}
-
-static void free_model_specs(model_specs_t *specs) {
- if (!specs)
- return;
- for (int i = 0; i < specs->n; i++) {
- free((char *)specs->v[i].id);
- free((char *)specs->v[i].path);
- free((char *)specs->v[i].query_instruct);
- }
- free(specs->v);
-}
-
-int main(int argc, char *argv[]) {
- int n_threads = 0;
- int verbose = 0;
- int batch_size = 0;
- int max_batch_tokens = 0;
- int batch_wait_us = -1;
-#ifdef USE_MLX
- int use_mlx = 0;
- int mlx_quantize_bits = 0;
- int mlx_quantize_group_size = 64;
- double memory_utilization = 0.0;
-#endif
-#ifdef USE_CUDA
- int use_cuda = 0;
- const char *cuda_fast_gemm = NULL;
- const char *cuda_weights = NULL;
-#endif
- const char *host = "127.0.0.1";
- const char *api_key = NULL;
- int port = 8000;
- model_specs_t model_specs = {0};
-
- const char *prog = embed_prog_name(argv[0]);
- int arg = 1;
- while (arg < argc && argv[arg][0] == '-') {
- const char *f = argv[arg];
- if (!strcmp(f, "-V") || !strcmp(f, "--version")) {
- embed_print_version(prog);
- free_model_specs(&model_specs);
- return 0;
- } else if (!strcmp(f, "--build-info")) {
- embed_print_build_info(prog);
- free_model_specs(&model_specs);
- return 0;
- } else if (!strcmp(f, "--model")) {
- if (arg + 1 >= argc ||
- append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_STANDARD, f) !=
- 0) {
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "--contextual-model")) {
- if (arg + 1 >= argc ||
- append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_CONTEXTUAL, f) !=
- 0) {
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "--late-model")) {
- if (arg + 1 >= argc ||
- append_model_spec(&model_specs, argv[++arg], EMBED_SERVER_MODEL_LATE, f) != 0) {
- free_model_specs(&model_specs);
- return 1;
- }
- }
-#ifdef USE_MLX
- else if (!strcmp(f, "--mlx")) {
- use_mlx = 1;
- } else if (!strcmp(f, "--mlx-quant-bits")) {
- mlx_quantize_bits = atoi(argv[++arg]);
- if (mlx_quantize_bits != 0 && mlx_quantize_bits != 8) {
- fprintf(stderr, "--mlx-quant-bits must be 0 or 8\n");
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "--mlx-quant-group-size")) {
- mlx_quantize_group_size = atoi(argv[++arg]);
- if (mlx_quantize_group_size <= 0) {
- fprintf(stderr, "--mlx-quant-group-size must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "--memory-utilization")) {
- memory_utilization = atof(argv[++arg]);
- if (memory_utilization <= 0.0) {
- fprintf(stderr, "--memory-utilization must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
-#endif
-#ifdef USE_CUDA
- else if (!strcmp(f, "--cuda")) {
- use_cuda = 1;
- } else if (!strcmp(f, "--cuda-gemm-mode")) {
- cuda_fast_gemm = argv[++arg];
- } else if (!strcmp(f, "--cuda-weight-dtype")) {
- cuda_weights = argv[++arg];
- }
-#endif
- else if (!strcmp(f, "--host")) {
- host = argv[++arg];
- } else if (!strcmp(f, "--port")) {
- port = atoi(argv[++arg]);
- } else if (!strcmp(f, "--api-key")) {
- api_key = argv[++arg];
- } else if (!strcmp(f, "-b") || !strcmp(f, "--batch-size")) {
- batch_size = atoi(argv[++arg]);
- } else if (!strcmp(f, "--max-batch-tokens")) {
- max_batch_tokens = atoi(argv[++arg]);
- if (max_batch_tokens <= 0) {
- fprintf(stderr, "--max-batch-tokens must be > 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "--batch-wait-us")) {
- batch_wait_us = atoi(argv[++arg]);
- if (batch_wait_us < 0) {
- fprintf(stderr, "--batch-wait-us must be >= 0\n");
- free_model_specs(&model_specs);
- return 1;
- }
- } else if (!strcmp(f, "-t") || !strcmp(f, "--threads")) {
- n_threads = atoi(argv[++arg]);
- } else if (!strcmp(f, "-v") || !strcmp(f, "--verbose")) {
- verbose++;
- } else if (!strcmp(f, "-vv")) {
- verbose = 2;
- } else if (!strcmp(f, "-h") || !strcmp(f, "--help")) {
- print_usage(prog);
- free_model_specs(&model_specs);
- return 0;
- } else {
- fprintf(stderr, "unknown option: %s\n", f);
- print_usage(prog);
- free_model_specs(&model_specs);
- return 1;
- }
- arg++;
- }
-
- embed_verbose = verbose;
-
- if (arg < argc) {
- fprintf(stderr, "unexpected argument: %s\n", argv[arg]);
- print_usage(prog);
- free_model_specs(&model_specs);
- return 1;
- }
- if (model_specs.n == 0) {
- fprintf(stderr, "at least one --model LABEL=DIR is required\n");
- print_usage(prog);
- free_model_specs(&model_specs);
- return 1;
- }
-#ifdef USE_MLX
- if (mlx_quantize_bits && !use_mlx) {
- fprintf(stderr, "--mlx-quant-bits requires --mlx\n");
- free_model_specs(&model_specs);
- return 1;
- }
-#endif
-#ifdef USE_CUDA
- if (cuda_fast_gemm) {
- if (!use_cuda) {
- fprintf(stderr, "--cuda-gemm-mode requires --cuda\n");
- free_model_specs(&model_specs);
- return 1;
- }
- if (embed_cuda_set_fast_gemm(cuda_fast_gemm) != 0) {
- fprintf(stderr, "--cuda-gemm-mode must be f32, tf32, bf16, or 16f\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
- if (cuda_weights) {
- if (!use_cuda) {
- fprintf(stderr, "--cuda-weight-dtype requires --cuda\n");
- free_model_specs(&model_specs);
- return 1;
- }
- if (!strcmp(cuda_weights, "bf16")) {
- embed_cuda_set_weights_bf16(1);
- } else if (!strcmp(cuda_weights, "f32")) {
- embed_cuda_set_weights_bf16(0);
- } else {
- fprintf(stderr, "--cuda-weight-dtype must be f32 or bf16\n");
- free_model_specs(&model_specs);
- return 1;
- }
- }
-#endif
-
-#ifdef USE_MLX
- if (use_mlx) {
- if (verbose >= 1)
- fprintf(stderr, "Using MLX GPU backend\n");
- } else
-#endif
-#ifdef USE_CUDA
- if (use_cuda) {
- if (verbose >= 1)
- fprintf(stderr, "Using CUDA GPU backend\n");
- } else
-#endif
- {
- if (n_threads <= 0)
- n_threads = embed_get_num_cpus();
- embed_set_threads(n_threads);
- if (verbose >= 1)
- fprintf(stderr, "Using %d CPU thread(s)\n", n_threads);
- }
-
- embed_server_config_t scfg = {
- .models = model_specs.v,
- .n_models = model_specs.n,
- .host = host,
- .port = port,
- .batch_size = batch_size > 0 ? batch_size : EMBED_SERVER_DEFAULT_BATCH_SIZE,
- .max_batch_tokens = max_batch_tokens,
- .batch_wait_us = batch_wait_us,
-#ifdef USE_MLX
- .use_mlx = use_mlx,
- .mlx_quantize_bits = mlx_quantize_bits,
- .mlx_quantize_group_size = mlx_quantize_group_size,
- .memory_utilization = memory_utilization,
-#endif
-#ifdef USE_CUDA
- .use_cuda = use_cuda,
-#endif
- .api_key = api_key,
- };
- int rc = embed_run_server(&scfg);
- free_model_specs(&model_specs);
- return rc;
-}
-
-#endif /* EMBED_SERVER_TEST */
diff --git a/src/server.h b/src/server.h
deleted file mode 100644
index 5ad2b12..0000000
--- a/src/server.h
+++ /dev/null
@@ -1,58 +0,0 @@
-#ifndef EMBED_SERVER_H
-#define EMBED_SERVER_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-#include <stdbool.h>
-
-typedef enum {
- EMBED_SERVER_MODEL_STANDARD = 0,
- EMBED_SERVER_MODEL_CONTEXTUAL = 1,
- EMBED_SERVER_MODEL_LATE = 2,
-} embed_server_model_kind_t;
-
-typedef enum {
- EMBED_SERVER_API_DEFAULT = 0, /* currently OpenAI-compatible float output */
- EMBED_SERVER_API_OPENAI = 1,
- EMBED_SERVER_API_PERPLEXITY = 2,
-} embed_server_embedding_api_t;
-
-typedef struct {
- const char *id; /* served label; requests match this string exactly */
- const char *path;
- embed_server_model_kind_t kind;
- embed_server_embedding_api_t api;
- int min_dim; /* 0 means the loaded model's full hidden size */
- const char *query_instruct; /* NULL means use config_sentence_transformers.json if present */
-} embed_server_model_spec_t;
-
-typedef struct {
- const embed_server_model_spec_t *models;
- int n_models;
- const char *host;
- int port;
- int batch_size;
- int max_batch_tokens;
- int batch_wait_us;
- int use_mlx;
- int use_cuda;
- int mlx_quantize_bits;
- int mlx_quantize_group_size;
- double memory_utilization; /* fraction of physical memory the MLX
- preflight may plan for; 0 = default 0.90,
- values above 1.0 allow overcommit */
- const char *api_key;
-} embed_server_config_t;
-
-EMBED_API int embed_run_server(const embed_server_config_t *cfg);
-
-#endif
diff --git a/src/server_encode.c b/src/server_encode.c
deleted file mode 100644
index ef4e3ec..0000000
--- a/src/server_encode.c
+++ /dev/null
@@ -1,152 +0,0 @@
-#include "server_internal.h"
-#include "server_util.h"
-#include "sbuf.h"
-#include "base64.h"
-#include "embed.h"
-
-#include <math.h>
-#include <stdlib.h>
-#include <string.h>
-
-/* ========================================================================
- * Embedding output encoding
- *
- * Turns a pooled float vector into the requested wire form. OpenAI-compatible
- * models emit true float32 arrays by default. Perplexity-compatible models
- * emit tanh-quantized int8 by default, and their "float" view is the decoded
- * int8 value for SDK compatibility.
- * ======================================================================== */
-
-signed char quantize_int8_tanh(float x) {
- int v = (int)lrintf(tanhf(x) * 127.0f);
- if (v > 127)
- v = 127;
- if (v < -128)
- v = -128;
- return (signed char)v;
-}
-
-char *encode_embedding(const float *emb, int dims, const char *encoding) {
- if (!strcmp(encoding, "base64")) {
- /* OpenAI/DashScope (Qwen3): base64 of the raw little-endian float32
- * vector. Copy through a byte buffer instead of reinterpret-casting the
- * float pointer; x86_64 and aarch64 are little-endian, so the bytes
- * match the wire format with no swap. */
- size_t nbytes = (size_t)dims * sizeof(float);
- unsigned char *buf = xmalloc(nbytes);
- memcpy(buf, emb, nbytes);
- char *out = base64_encode(buf, nbytes);
- free(buf);
- return out;
- }
- if (!strcmp(encoding, "base64_binary")) {
- size_t bytes = (size_t)dims / 8;
- unsigned char *bits = xcalloc(bytes, 1);
- for (int i = 0; i < dims; i++) {
- if (emb[i] >= 0.0f)
- bits[(size_t)i >> 3] |= (unsigned char)(1u << (i & 7));
- }
- char *out = base64_encode(bits, bytes);
- free(bits);
- return out;
- }
-
- signed char *q = xmalloc((size_t)dims);
- for (int i = 0; i < dims; i++)
- q[i] = quantize_int8_tanh(emb[i]);
- char *out = base64_encode((const unsigned char *)q, (size_t)dims);
- free(q);
- return out;
-}
-
-static void append_embedding_object(sbuf *b, int index, const char *embedding) {
- sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":\"", index);
- sbuf_puts(b, embedding);
- sbuf_puts(b, "\"}");
-}
-
-void append_embedding_value(
- sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api) {
- if (!strcmp(encoding, "float")) {
- sbuf_printf(b, "{\"object\":\"embedding\",\"index\":%d,\"embedding\":[", index);
- for (int i = 0; i < dims; i++) {
- if (i)
- sbuf_putc(b, ',');
- float v =
- api == EMBED_API_PERPLEXITY ? (float)quantize_int8_tanh(emb[i]) / 128.0f : emb[i];
- sbuf_printf(b, "%.9g", (double)v);
- }
- sbuf_puts(b, "]}");
- return;
- }
- char *encoded = encode_embedding(emb, dims, encoding);
- append_embedding_object(b, index, encoded);
- free(encoded);
-}
-
-void set_response_from_buf(job *j, sbuf *b) {
- j->status = 200;
- j->content_type = "application/json";
- j->response = b->ptr;
- j->response_len = b->len;
- memset(b, 0, sizeof(*b));
-}
-
-int render_embedding_response(embedding_request *r, const float *embs) {
- const int full_dim = r->model->info->dim;
- const float *render_embs = embs;
- float *truncated = NULL;
- if (r->model->renormalize_truncated && r->dims < full_dim) {
- truncated = xmalloc((size_t)r->n_inputs * r->dims * sizeof(*truncated));
- for (int i = 0; i < r->n_inputs; i++) {
- float *dst = truncated + (size_t)i * r->dims;
- memcpy(dst, embs + (size_t)i * full_dim, (size_t)r->dims * sizeof(*dst));
- if (embed_l2_normalize(dst, r->dims) != 0) {
- free(truncated);
- return -1;
- }
- }
- render_embs = truncated;
- }
- int render_stride = truncated ? r->dims : full_dim;
- embedding_api_t api = r->model->info->api;
-
- sbuf b = {0};
- sbuf_puts(&b, "{\"object\":\"list\",\"data\":[");
- for (int i = 0; i < r->n_inputs; i++) {
- if (i)
- sbuf_putc(&b, ',');
- append_embedding_value(&b, i, render_embs + (size_t)i * render_stride, r->dims, r->encoding,
- api);
- }
- sbuf_printf(&b,
- "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,"
- "\"total_tokens\":%d}}",
- r->model->info->id, r->total_tokens, r->total_tokens);
- set_response_from_buf(r->j, &b);
-
- free(truncated);
- return 0;
-}
-
-void render_contextual_response(contextual_request *r, const float *embs) {
- sbuf b = {0};
- sbuf_puts(&b, "{\"object\":\"list\",\"data\":[");
- int span_offset = 0;
- for (int di = 0; di < r->n_docs; di++) {
- if (di)
- sbuf_putc(&b, ',');
- sbuf_printf(&b, "{\"object\":\"list\",\"index\":%d,\"data\":[", di);
- for (int ci = 0; ci < r->docs[di].n_spans; ci++) {
- if (ci)
- sbuf_putc(&b, ',');
- const float *emb = embs + (size_t)span_offset * r->model->info->dim;
- append_embedding_value(&b, ci, emb, r->dims, r->encoding, r->model->info->api);
- span_offset++;
- }
- sbuf_puts(&b, "]}");
- }
- sbuf_printf(&b, "],\"model\":\"%s\",\"usage\":{\"prompt_tokens\":%d,\"total_tokens\":%d}}",
- r->model->info->id, r->total_tokens, r->total_tokens);
- set_response_from_buf(r->j, &b);
-}
diff --git a/src/server_handlers.c b/src/server_handlers.c
deleted file mode 100644
index b3476d9..0000000
--- a/src/server_handlers.c
+++ /dev/null
@@ -1,907 +0,0 @@
-#include "server_internal.h"
-#include "server_util.h"
-#include "embed.h"
-
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-/* ========================================================================
- * Endpoint handlers: prepare, batch, execute, render
- *
- * One prepare_* per endpoint validates the request JSON and tokenizes its
- * inputs into a *_request; the worker (server_schedule.c) groups compatible
- * ready requests and calls the matching execute_*_request_list, which runs the
- * batched forward pass through model_embed_* and renders each response via
- * server_encode.c. Rerank runs MaxSim over late-interaction token vectors.
- * ======================================================================== */
-
-int embedding_request_compatible(const embedding_request *a, const embedding_request *b) {
- return a->ready && b->ready && a->model == b->model && a->dims == b->dims &&
- !strcmp(a->encoding, b->encoding);
-}
-
-typedef struct {
- embed_input_t input;
- int output_index;
-} embedding_batch_item;
-
-static int embedding_batch_item_cmp(const void *a, const void *b) {
- const embedding_batch_item *ia = a;
- const embedding_batch_item *ib = b;
- if (ia->input.n_tokens < ib->input.n_tokens)
- return -1;
- if (ia->input.n_tokens > ib->input.n_tokens)
- return 1;
- return ia->output_index - ib->output_index;
-}
-
-int embedding_request_fits_group(const embedding_request *r,
- int group_inputs,
- int group_tokens,
- int max_batch,
- int max_batch_tokens) {
- if (group_inputs == 0)
- return 1;
- return r->n_inputs <= max_batch - group_inputs &&
- r->total_tokens <= max_batch_tokens - group_tokens;
-}
-
-void execute_embedding_request_list(embedding_request **reqs, int n_reqs) {
- if (n_reqs <= 0)
- return;
- loaded_model *m = reqs[0]->model;
- int dim = m->info->dim;
- int total_inputs = 0;
- for (int i = 0; i < n_reqs; i++)
- total_inputs += reqs[i]->n_inputs;
-
- embedding_batch_item *items = xmalloc((size_t)total_inputs * sizeof(*items));
- float *embs = xmalloc((size_t)total_inputs * dim * sizeof(float));
-
- int pos = 0;
- for (int i = 0; i < n_reqs; i++) {
- for (int k = 0; k < reqs[i]->n_inputs; k++) {
- items[pos].input = reqs[i]->inputs[k];
- items[pos].output_index = pos;
- pos++;
- }
- }
- qsort(items, (size_t)total_inputs, sizeof(*items), embedding_batch_item_cmp);
-
- int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_inputs;
- if (max_batch > total_inputs)
- max_batch = total_inputs;
- int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
- embed_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
- float *batch_embs = xmalloc((size_t)max_batch * dim * sizeof(float));
-
- int failed = 0;
- uint64_t infer_ns = 0;
- for (int start = 0; start < total_inputs;) {
- int cur = 0;
- int tokens = 0;
- while (start + cur < total_inputs && cur < max_batch) {
- int n_tokens = items[start + cur].input.n_tokens;
- if (!inference_batch_accepts_input(m, cur, tokens, n_tokens, max_batch_tokens))
- break;
- inputs[cur] = items[start + cur].input;
- tokens += n_tokens;
- cur++;
- }
- uint64_t t0 = nstime();
- int embed_rc = model_embed_batch(m, inputs, cur, batch_embs);
- infer_ns += nstime() - t0;
- if (embed_rc != 0) {
- failed = 1;
- break;
- }
- for (int i = 0; i < cur; i++)
- memcpy(embs + (size_t)items[start + i].output_index * dim, batch_embs + (size_t)i * dim,
- (size_t)dim * sizeof(float));
- start += cur;
- }
-
- pos = 0;
- for (int i = 0; i < n_reqs; i++) {
- reqs[i]->j->infer_ns += infer_ns;
- if (failed) {
- job_set_error(reqs[i]->j, 500, "embedding failed", "server_error");
- } else {
- uint64_t t0 = nstime();
- if (render_embedding_response(reqs[i], embs + (size_t)pos * dim) != 0)
- job_set_error(reqs[i]->j, 500, "embedding normalization failed", "server_error");
- reqs[i]->j->encode_ns += nstime() - t0;
- }
- pos += reqs[i]->n_inputs;
- }
-
- free(batch_embs);
- free(inputs);
- free(embs);
- free(items);
-}
-
-int contextual_request_compatible(const contextual_request *a, const contextual_request *b) {
- return a->ready && b->ready && a->model == b->model && a->dims == b->dims &&
- !strcmp(a->encoding, b->encoding);
-}
-
-int contextual_request_fits_group(const contextual_request *r,
- int group_docs,
- int group_tokens,
- int max_batch,
- int max_batch_tokens) {
- if (group_docs == 0)
- return 1;
- return r->n_docs <= max_batch - group_docs &&
- r->total_tokens <= max_batch_tokens - group_tokens;
-}
-
-typedef struct {
- embed_context_input_t input;
- int output_span_index;
- int order;
-} contextual_batch_item;
-
-static int contextual_batch_item_cmp(const void *a, const void *b) {
- const contextual_batch_item *ia = a;
- const contextual_batch_item *ib = b;
- if (ia->input.input.n_tokens < ib->input.input.n_tokens)
- return -1;
- if (ia->input.input.n_tokens > ib->input.input.n_tokens)
- return 1;
- return ia->order - ib->order;
-}
-
-void execute_contextual_request_list(contextual_request **reqs, int n_reqs) {
- if (n_reqs <= 0)
- return;
- loaded_model *m = reqs[0]->model;
- int dim = m->info->dim;
- int total_docs = 0;
- int total_chunks = 0;
- for (int i = 0; i < n_reqs; i++) {
- if (total_docs > INT_MAX - reqs[i]->n_docs ||
- total_chunks > INT_MAX - reqs[i]->total_chunks) {
- for (int k = 0; k < n_reqs; k++)
- job_set_error(reqs[k]->j, 500, "contextual batch is too large", "server_error");
- return;
- }
- total_docs += reqs[i]->n_docs;
- total_chunks += reqs[i]->total_chunks;
- }
- if ((size_t)total_chunks > SIZE_MAX / (size_t)dim / sizeof(float)) {
- for (int i = 0; i < n_reqs; i++)
- job_set_error(reqs[i]->j, 500, "contextual batch is too large", "server_error");
- return;
- }
-
- contextual_batch_item *items = xmalloc((size_t)total_docs * sizeof(*items));
- float *embs = xmalloc((size_t)total_chunks * dim * sizeof(*embs));
- int pos = 0;
- int span_offset = 0;
- for (int i = 0; i < n_reqs; i++) {
- for (int d = 0; d < reqs[i]->n_docs; d++) {
- contextual_doc *doc = &reqs[i]->docs[d];
- items[pos].input.input.ids = doc->ids;
- items[pos].input.input.n_tokens = doc->n_tokens;
- items[pos].input.spans = doc->spans;
- items[pos].input.n_spans = doc->n_spans;
- items[pos].output_span_index = span_offset;
- items[pos].order = pos;
- span_offset += doc->n_spans;
- pos++;
- }
- }
- qsort(items, (size_t)total_docs, sizeof(*items), contextual_batch_item_cmp);
-
- int max_batch = reqs[0]->j->srv->batch_size > 0 ? reqs[0]->j->srv->batch_size : total_docs;
- if (max_batch > total_docs)
- max_batch = total_docs;
- int max_batch_tokens = reqs[0]->j->srv->max_batch_tokens;
- embed_context_input_t *inputs = xmalloc((size_t)max_batch * sizeof(*inputs));
- float *batch_embs = xmalloc(sizeof(*batch_embs));
- size_t batch_emb_cap = 0;
-
- int failed = 0;
- uint64_t infer_ns = 0;
- for (int start = 0; start < total_docs;) {
- int cur = 0;
- int tokens = 0;
- int chunks = 0;
- while (start + cur < total_docs && cur < max_batch) {
- const embed_context_input_t *input = &items[start + cur].input;
- if (!inference_batch_accepts_input(m, cur, tokens, input->input.n_tokens,
- max_batch_tokens))
- break;
- inputs[cur] = *input;
- tokens += input->input.n_tokens;
- chunks += input->n_spans;
- cur++;
- }
- size_t needed = (size_t)chunks * dim;
- if (needed > batch_emb_cap) {
- batch_embs = xrealloc(batch_embs, needed * sizeof(*batch_embs));
- batch_emb_cap = needed;
- }
- uint64_t t0 = nstime();
- int embed_rc = model_embed_spans_batch(m, inputs, cur, batch_embs);
- infer_ns += nstime() - t0;
- if (embed_rc != 0) {
- failed = 1;
- break;
- }
- int batch_span = 0;
- for (int i = 0; i < cur; i++) {
- int n_spans = inputs[i].n_spans;
- memcpy(embs + (size_t)items[start + i].output_span_index * dim,
- batch_embs + (size_t)batch_span * dim, (size_t)n_spans * dim * sizeof(*embs));
- batch_span += n_spans;
- }
- start += cur;
- }
-
- span_offset = 0;
- for (int i = 0; i < n_reqs; i++) {
- reqs[i]->j->infer_ns += infer_ns;
- if (failed) {
- job_set_error(reqs[i]->j, 500, "contextual embedding failed", "server_error");
- } else {
- uint64_t t0 = nstime();
- render_contextual_response(reqs[i], embs + (size_t)span_offset * dim);
- reqs[i]->j->encode_ns += nstime() - t0;
- }
- span_offset += reqs[i]->total_chunks;
- }
-
- free(batch_embs);
- free(inputs);
- free(embs);
- free(items);
-}
-
-static int validate_common(cJSON *root,
- cJSON *detail,
- model_kind expected_kind,
- loaded_model **out_model,
- int *out_dims,
- const char **out_encoding,
- http_server *s) {
- cJSON *model_item = cJSON_GetObjectItemCaseSensitive(root, "model");
- const char *model_id = NULL;
- loaded_model *m = NULL;
- if (!model_item) {
- ve_add(detail, "[\"body\",\"model\"]", "field required", "missing");
- } else if (!cJSON_IsString(model_item) || !model_item->valuestring) {
- ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string");
- } else {
- model_id = model_item->valuestring;
- m = loaded_model_for_label(s, model_id);
- if (!m) {
- ve_add(detail, "[\"body\",\"model\"]",
- "value is not a valid enum member for this endpoint", "enum");
- } else if (m->info->kind != expected_kind) {
- ve_add(detail, "[\"body\",\"model\"]",
- "model is not available on this endpoint", "enum");
- m = NULL;
- }
- }
- *out_model = m;
- const char *enc = encoding_from_root(root, detail, m ? m->info->api : EMBED_API_OPENAI);
- int min_dim = m ? m->info->min_dim : 128;
- int max_dim = m ? m->info->dim : 2560;
- int dims = dimensions_from_root(root, detail, min_dim, max_dim, enc);
- *out_encoding = enc;
- *out_dims = dims;
- return 0;
-}
-
-void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out) {
- memset(out, 0, sizeof(*out));
- out->j = j;
- out->root = root;
-
- cJSON *detail = cJSON_CreateArray();
- loaded_model *m = NULL;
- int dims = 0;
- const char *encoding = NULL;
- validate_common(root, detail, MODEL_KIND_STANDARD, &m, &dims, &encoding, s);
- out->model = m;
- out->dims = dims;
- out->encoding = encoding;
-
- /* The query instruction, when requested, applies to every input. */
- const char *query_instruct = m && text_type_is_query(root, detail, m->info->query_instruct)
- ? m->info->query_instruct
- : NULL;
-
- cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
- int n_inputs = 0;
- if (!input) {
- ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
- } else if (cJSON_IsString(input)) {
- if (!input->valuestring || input->valuestring[0] == '\0')
- ve_add(detail, "[\"body\",\"input\"]", "input must not be empty", "value_error.empty");
- n_inputs = 1;
- } else if (cJSON_IsArray(input)) {
- n_inputs = cJSON_GetArraySize(input);
- if (n_inputs < 1)
- ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 item",
- "value_error.list.min_items");
- else if (n_inputs > EMBED_API_MAX_STANDARD_INPUTS)
- ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 items",
- "value_error.list.max_items");
- cJSON *item;
- int i = 0;
- cJSON_ArrayForEach(item, input) {
- char loc[64];
- snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", i);
- if (!cJSON_IsString(item)) {
- ve_add(detail, loc, "input item must be a string", "type_error.string");
- } else if (!item->valuestring || item->valuestring[0] == '\0') {
- ve_add(detail, loc, "input item must not be empty", "value_error.empty");
- }
- i++;
- }
- } else {
- ve_add(detail, "[\"body\",\"input\"]", "input must be a string or an array of strings",
- "type_error");
- }
-
- if (cJSON_GetArraySize(detail) == 0 && m && input) {
- out->n_inputs = n_inputs;
- out->tokens = xcalloc((size_t)n_inputs, sizeof(*out->tokens));
- out->inputs = xmalloc((size_t)n_inputs * sizeof(*out->inputs));
-
- int idx = 0;
- if (input && cJSON_IsString(input) && input->valuestring) {
- if (tokenize_input(m, j, input->valuestring, query_instruct, &out->tokens[0]) != 0) {
- ve_add(detail, "[\"body\",\"input\"]", "tokenization failed",
- "value_error.tokenization");
- } else {
- out->inputs[0].ids = out->tokens[0].ids;
- out->inputs[0].n_tokens = out->tokens[0].n_tokens;
- out->total_tokens = out->tokens[0].n_tokens;
- if (out->tokens[0].n_tokens > EMBED_API_MAX_ITEM_TOKENS)
- ve_add(detail, "[\"body\",\"input\"]", "input exceeds 32768 token limit",
- "value_error.context_length");
- }
- } else {
- cJSON *item;
- cJSON_ArrayForEach(item, input) {
- char loc[64];
- snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", idx);
- if (!cJSON_IsString(item) || !item->valuestring) {
- ve_add(detail, loc, "input item must be a string", "type_error.string");
- idx++;
- continue;
- }
- if (tokenize_input(m, j, item->valuestring, query_instruct, &out->tokens[idx]) !=
- 0) {
- ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
- } else {
- out->inputs[idx].ids = out->tokens[idx].ids;
- out->inputs[idx].n_tokens = out->tokens[idx].n_tokens;
- out->total_tokens += out->tokens[idx].n_tokens;
- if (out->tokens[idx].n_tokens > EMBED_API_MAX_ITEM_TOKENS)
- ve_add(detail, loc, "input exceeds 32768 token limit",
- "value_error.context_length");
- }
- idx++;
- }
- }
-
- if (out->total_tokens > EMBED_API_MAX_TOTAL_TOKENS)
- ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
- "value_error.context_length");
- }
-
- if (cJSON_GetArraySize(detail) > 0) {
- job_set_422(j, detail);
- embedding_request_free(out);
- out->j = j;
- } else {
- out->ready = 1;
- }
- cJSON_Delete(detail);
-}
-
-void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out) {
- memset(out, 0, sizeof(*out));
- out->j = j;
- out->root = root;
-
- cJSON *detail = cJSON_CreateArray();
- loaded_model *m = NULL;
- int dims = 0;
- const char *encoding = NULL;
- validate_common(root, detail, MODEL_KIND_CONTEXTUAL, &m, &dims, &encoding, s);
- out->model = m;
- out->dims = dims;
- out->encoding = encoding;
-
- cJSON *input = cJSON_GetObjectItemCaseSensitive(root, "input");
- int n_docs = 0;
- int total_chunks = 0;
- if (!input) {
- ve_add(detail, "[\"body\",\"input\"]", "field required", "missing");
- } else if (!cJSON_IsArray(input)) {
- ve_add(detail, "[\"body\",\"input\"]", "input must be an array of document chunk arrays",
- "type_error.array");
- } else {
- n_docs = cJSON_GetArraySize(input);
- if (n_docs < 1)
- ve_add(detail, "[\"body\",\"input\"]", "input must contain at least 1 document",
- "value_error.list.min_items");
- else if (n_docs > EMBED_API_MAX_CONTEXT_DOCS)
- ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 512 documents",
- "value_error.list.max_items");
- cJSON *doc_arr;
- int di = 0;
- cJSON_ArrayForEach(doc_arr, input) {
- if (!cJSON_IsArray(doc_arr)) {
- char loc[64];
- snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
- ve_add(detail, loc, "document must be an array of strings", "type_error.array");
- di++;
- continue;
- }
- int doc_chunks = cJSON_GetArraySize(doc_arr);
- if (doc_chunks > EMBED_API_MAX_CONTEXT_CHUNKS - total_chunks)
- total_chunks = EMBED_API_MAX_CONTEXT_CHUNKS + 1;
- else
- total_chunks += doc_chunks;
- if (doc_chunks < 1) {
- char loc[64];
- snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
- ve_add(detail, loc, "document must contain at least 1 chunk",
- "value_error.list.min_items");
- }
- cJSON *chunk;
- int ci = 0;
- cJSON_ArrayForEach(chunk, doc_arr) {
- char loc[80];
- snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci);
- if (!cJSON_IsString(chunk)) {
- ve_add(detail, loc, "chunk must be a string", "type_error.string");
- } else if (!chunk->valuestring || chunk->valuestring[0] == '\0') {
- ve_add(detail, loc, "chunk must not be empty", "value_error.empty");
- }
- ci++;
- }
- di++;
- }
- if (total_chunks > EMBED_API_MAX_CONTEXT_CHUNKS)
- ve_add(detail, "[\"body\",\"input\"]", "input must contain at most 16000 chunks",
- "value_error.list.max_items");
- }
-
- if (cJSON_GetArraySize(detail) == 0 && m) {
- out->n_docs = n_docs;
- out->docs = xcalloc((size_t)n_docs, sizeof(*out->docs));
- int64_t request_tokens = 0;
- cJSON *doc_arr;
- int di = 0;
- cJSON_ArrayForEach(doc_arr, input) {
- contextual_doc *doc = &out->docs[di];
- int n_chunks = cJSON_GetArraySize(doc_arr);
- token_buf *chunk_tokens = xcalloc((size_t)n_chunks, sizeof(*chunk_tokens));
- doc->spans = xcalloc((size_t)n_chunks, sizeof(*doc->spans));
- doc->n_spans = n_chunks;
- int64_t doc_tokens = n_chunks > 1 ? n_chunks - 1 : 0;
- int doc_valid = 1;
-
- cJSON *chunk;
- int ci = 0;
- cJSON_ArrayForEach(chunk, doc_arr) {
- char loc[80];
- snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d,%d]", di, ci);
- if (tokenize_one(m, j, chunk->valuestring, &chunk_tokens[ci]) != 0) {
- ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
- doc_valid = 0;
- } else {
- doc_tokens += chunk_tokens[ci].n_tokens;
- }
- ci++;
- }
-
- if (doc_tokens > EMBED_API_MAX_ITEM_TOKENS) {
- char loc[64];
- snprintf(loc, sizeof(loc), "[\"body\",\"input\",%d]", di);
- ve_add(detail, loc, "document exceeds 32768 token limit",
- "value_error.context_length");
- doc_valid = 0;
- }
- request_tokens += doc_tokens;
-
- if (doc_valid) {
- doc->n_tokens = (int)doc_tokens;
- doc->ids = xmalloc((size_t)doc->n_tokens * sizeof(*doc->ids));
- int pos = 0;
- for (ci = 0; ci < n_chunks; ci++) {
- doc->spans[ci].start = pos;
- doc->spans[ci].n_tokens = chunk_tokens[ci].n_tokens;
- /* n_tokens > 0 implies ids != NULL (tokenize_one sets both
- * together), and the guard avoids a memcpy(dst, NULL, 0) for
- * an empty chunk that the static analyzer reports as UB. */
- if (chunk_tokens[ci].n_tokens > 0)
- memcpy(doc->ids + pos, chunk_tokens[ci].ids,
- (size_t)chunk_tokens[ci].n_tokens * sizeof(*doc->ids));
- pos += chunk_tokens[ci].n_tokens;
- if (ci + 1 < n_chunks) {
- doc->ids[pos++] = m->context_separator_id;
- }
- }
- }
- free_token_bufs(chunk_tokens, n_chunks);
- free(chunk_tokens);
- di++;
- }
- if (request_tokens > EMBED_API_MAX_TOTAL_TOKENS) {
- ve_add(detail, "[\"body\",\"input\"]", "request exceeds 120000 token limit",
- "value_error.context_length");
- } else {
- out->total_tokens = (int)request_tokens;
- out->total_chunks = total_chunks;
- }
- }
-
- if (cJSON_GetArraySize(detail) > 0) {
- job_set_422(j, detail);
- contextual_request_free(out);
- out->j = j;
- } else {
- out->ready = 1;
- }
- cJSON_Delete(detail);
-}
-
-static int
-validate_rerank_model(cJSON *root, cJSON *detail, http_server *s, loaded_model **out_model) {
- cJSON *item = cJSON_GetObjectItemCaseSensitive(root, "model");
- if (!item) {
- ve_add(detail, "[\"body\",\"model\"]", "field required", "missing");
- return 0;
- }
- if (!cJSON_IsString(item) || !item->valuestring) {
- ve_add(detail, "[\"body\",\"model\"]", "model must be a string", "type_error.string");
- return 0;
- }
- loaded_model *m = loaded_model_for_label(s, item->valuestring);
- if (!m) {
- ve_add(detail, "[\"body\",\"model\"]", "value is not a valid enum member for this endpoint",
- "enum");
- return 0;
- }
- if (m->info->kind != MODEL_KIND_LATE) {
- ve_add(detail, "[\"body\",\"model\"]", "model is not available on this endpoint", "enum");
- return 0;
- }
- *out_model = m;
- return 0;
-}
-
-void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out) {
- memset(out, 0, sizeof(*out));
- out->j = j;
- out->root = root;
-
- cJSON *detail = cJSON_CreateArray();
- validate_rerank_model(root, detail, s, &out->model);
- cJSON *query = cJSON_GetObjectItemCaseSensitive(root, "query");
- cJSON *documents = cJSON_GetObjectItemCaseSensitive(root, "documents");
- /* query_str is set only once query passes every check; the tokenize step
- * below runs only when detail is empty, so it is non-NULL there. Hoisting it
- * also keeps the validated value explicit for the static analyzer. */
- const char *query_str = NULL;
- if (!query) {
- ve_add(detail, "[\"body\",\"query\"]", "field required", "missing");
- } else if (!cJSON_IsString(query)) {
- ve_add(detail, "[\"body\",\"query\"]", "query must be a string", "type_error.string");
- } else if (!query->valuestring || !query->valuestring[0]) {
- ve_add(detail, "[\"body\",\"query\"]", "query must not be empty", "value_error.empty");
- } else {
- query_str = query->valuestring;
- }
-
- int n_documents = 0;
- if (!documents) {
- ve_add(detail, "[\"body\",\"documents\"]", "field required", "missing");
- } else if (!cJSON_IsArray(documents)) {
- ve_add(detail, "[\"body\",\"documents\"]", "documents must be an array of strings",
- "type_error.array");
- } else {
- n_documents = cJSON_GetArraySize(documents);
- if (n_documents < 1)
- ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at least 1 item",
- "value_error.list.min_items");
- else if (n_documents > EMBED_API_MAX_RERANK_DOCUMENTS)
- ve_add(detail, "[\"body\",\"documents\"]", "documents must contain at most 1000 items",
- "value_error.list.max_items");
- cJSON *document;
- int i = 0;
- cJSON_ArrayForEach(document, documents) {
- char loc[72];
- snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i++);
- if (!cJSON_IsString(document))
- ve_add(detail, loc, "document must be a string", "type_error.string");
- else if (!document->valuestring || !document->valuestring[0])
- ve_add(detail, loc, "document must not be empty", "value_error.empty");
- }
- }
-
- cJSON *top_n = cJSON_GetObjectItemCaseSensitive(root, "top_n");
- cJSON *top_k = cJSON_GetObjectItemCaseSensitive(root, "top_k");
- if (top_n && top_k) {
- ve_add(detail, "[\"body\",\"top_n\"]", "top_n and top_k are aliases; provide only one",
- "value_error.conflict");
- }
- cJSON *top = top_n ? top_n : top_k;
- out->top_n = n_documents;
- if (top) {
- if (!cjson_is_integer(top)) {
- ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]",
- "top_n must be an integer", "type_error.integer");
- } else {
- out->top_n = top->valueint;
- if (out->top_n < 1 || out->top_n > n_documents)
- ve_add(detail, top_n ? "[\"body\",\"top_n\"]" : "[\"body\",\"top_k\"]",
- "top_n must be between 1 and the number of documents", "value_error.range");
- }
- }
-
- cJSON *return_documents = cJSON_GetObjectItemCaseSensitive(root, "return_documents");
- if (return_documents) {
- if (!cJSON_IsBool(return_documents))
- ve_add(detail, "[\"body\",\"return_documents\"]", "return_documents must be a boolean",
- "type_error.bool");
- else
- out->return_documents = cJSON_IsTrue(return_documents);
- }
-
- if (cJSON_GetArraySize(detail) == 0 && out->model) {
- out->n_documents = n_documents;
- out->documents = xcalloc((size_t)n_documents, sizeof(*out->documents));
- if (tokenize_late_text(out->model, j, query_str, 1, &out->query) != 0) {
- ve_add(detail, "[\"body\",\"query\"]", "tokenization failed",
- "value_error.tokenization");
- } else {
- out->query_tokens = out->query.n_tokens;
- if (out->query_tokens > EMBED_API_MAX_ITEM_TOKENS)
- ve_add(detail, "[\"body\",\"query\"]", "query exceeds 32768 token limit",
- "value_error.context_length");
- }
-
- cJSON *document;
- int i = 0;
- cJSON_ArrayForEach(document, documents) {
- char loc[72];
- snprintf(loc, sizeof(loc), "[\"body\",\"documents\",%d]", i);
- if (tokenize_late_text(out->model, j, document->valuestring, 0, &out->documents[i]) !=
- 0) {
- ve_add(detail, loc, "tokenization failed", "value_error.tokenization");
- } else {
- int n = out->documents[i].n_tokens;
- if (n > EMBED_API_MAX_ITEM_TOKENS)
- ve_add(detail, loc, "document exceeds 32768 token limit",
- "value_error.context_length");
- if (out->document_tokens <= INT_MAX - n)
- out->document_tokens += n;
- else
- out->document_tokens = INT_MAX;
- }
- i++;
- }
- if (out->query_tokens > EMBED_API_MAX_TOTAL_TOKENS - out->document_tokens) {
- ve_add(detail, "[\"body\",\"documents\"]", "request exceeds 120000 token limit",
- "value_error.context_length");
- }
- }
-
- if (cJSON_GetArraySize(detail) > 0) {
- job_set_422(j, detail);
- rerank_request_free(out);
- out->j = j;
- } else {
- out->ready = 1;
- }
- cJSON_Delete(detail);
-}
-
-typedef struct {
- int index;
- float score;
-} rerank_result;
-
-static int rerank_result_cmp(const void *a, const void *b) {
- const rerank_result *ra = a;
- const rerank_result *rb = b;
- if (ra->score > rb->score)
- return -1;
- if (ra->score < rb->score)
- return 1;
- return ra->index - rb->index;
-}
-
-static int execute_rerank_cpu(rerank_request *r, float *scores) {
- loaded_model *m = r->model;
- int dim = m->info->token_dim;
- int total_doc_vecs = 0;
- for (int i = 0; i < r->n_documents; i++)
- total_doc_vecs += r->documents[i].n_keep;
-
- float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query));
- float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs));
- int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
-
- /* Encode every candidate in one block-diagonal forward. The encoder packs
- * each document's kept token vectors back-to-back and fills offsets, so the
- * cost of N short documents collapses to a single packed forward. */
- const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
- const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
- int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
- int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep));
- for (int i = 0; i < r->n_documents; i++) {
- doc_ids[i] = r->documents[i].ids;
- keep[i] = r->documents[i].keep;
- n_tokens[i] = r->documents[i].n_tokens;
- n_keep[i] = r->documents[i].n_keep;
- }
-
- int rc = embed_late_model_encode_tokens(m->cpu_late_model, m->cpu_late_ws, r->query.ids,
- r->query.n_tokens, 1, query);
- if (rc == 0)
- rc = embed_late_model_encode_docs(m->cpu_late_model, m->cpu_late_ws, doc_ids, n_tokens,
- keep, n_keep, r->n_documents, 1, docs, offsets);
- if (rc == 0)
- rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim,
- scores);
- free(n_keep);
- free(n_tokens);
- free(keep);
- free(doc_ids);
- free(offsets);
- free(docs);
- free(query);
- return rc;
-}
-
-#ifdef USE_MLX
-static int execute_rerank_mlx(rerank_request *r, float *scores) {
- loaded_model *m = r->model;
- embed_mlx_late_vectors_t *query =
- embed_mlx_late_encode_tokens_device(m->mlx_late_ctx, r->query.ids, r->query.n_tokens, 1);
-
- /* Encode every candidate in one padded transformer pass. The encoder packs
- * each document's kept token vectors back-to-back and fills offsets, so the
- * cost of N short documents collapses to a single batched forward. */
- const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
- const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
- int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
- int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep));
- int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
- for (int i = 0; i < r->n_documents; i++) {
- doc_ids[i] = r->documents[i].ids;
- keep[i] = r->documents[i].keep;
- n_tokens[i] = r->documents[i].n_tokens;
- n_keep[i] = r->documents[i].n_keep;
- }
- embed_mlx_late_vectors_t *packed =
- query ? embed_mlx_late_encode_docs_device(m->mlx_late_ctx, doc_ids, n_tokens, keep, n_keep,
- r->n_documents, 1, offsets)
- : NULL;
- int rc = packed ? embed_mlx_late_maxsim_batch_device(m->mlx_late_ctx, query, packed, offsets,
- r->n_documents, scores)
- : -1;
- embed_mlx_late_vectors_free(packed);
- embed_mlx_late_vectors_free(query);
- free(offsets);
- free(n_keep);
- free(n_tokens);
- free(keep);
- free(doc_ids);
- return rc;
-}
-#endif
-
-#ifdef USE_CUDA
-static int execute_rerank_cuda(rerank_request *r, float *scores) {
- loaded_model *m = r->model;
- int dim = m->info->token_dim;
- int total_doc_vecs = 0;
- for (int i = 0; i < r->n_documents; i++)
- total_doc_vecs += r->documents[i].n_keep;
-
- float *query = xmalloc((size_t)r->query.n_tokens * dim * sizeof(*query));
- float *docs = xmalloc((size_t)total_doc_vecs * dim * sizeof(*docs));
- int *offsets = xmalloc((size_t)(r->n_documents + 1) * sizeof(*offsets));
-
- /* GPU encodes the query and every candidate (one packed forward); MaxSim
- * runs on the host, where the grouped-GEMM scorer beats a device graph. */
- const int **doc_ids = xmalloc((size_t)r->n_documents * sizeof(*doc_ids));
- const int **keep = xmalloc((size_t)r->n_documents * sizeof(*keep));
- int *n_tokens = xmalloc((size_t)r->n_documents * sizeof(*n_tokens));
- int *n_keep = xmalloc((size_t)r->n_documents * sizeof(*n_keep));
- for (int i = 0; i < r->n_documents; i++) {
- doc_ids[i] = r->documents[i].ids;
- keep[i] = r->documents[i].keep;
- n_tokens[i] = r->documents[i].n_tokens;
- n_keep[i] = r->documents[i].n_keep;
- }
-
- int rc =
- embed_cuda_late_encode_tokens(m->cuda_late_ctx, r->query.ids, r->query.n_tokens, 1, query);
- if (rc == 0)
- rc = embed_cuda_late_encode_docs(m->cuda_late_ctx, doc_ids, n_tokens, keep, n_keep,
- r->n_documents, 1, docs, offsets);
- if (rc == 0)
- rc = embed_late_maxsim_batch(query, r->query.n_keep, docs, offsets, r->n_documents, dim,
- scores);
- free(n_keep);
- free(n_tokens);
- free(keep);
- free(doc_ids);
- free(offsets);
- free(docs);
- free(query);
- return rc;
-}
-#endif
-
-void execute_rerank_request(rerank_request *r) {
- if (!r || !r->model) {
- if (r && r->j)
- job_set_error(r->j, 500, "late-interaction model is unavailable", "server_error");
- return;
- }
- float *scores = xmalloc((size_t)r->n_documents * sizeof(*scores));
- uint64_t t0 = nstime();
-#if defined(USE_MLX)
- int rc = r->model->mlx_late_ctx ? execute_rerank_mlx(r, scores) : execute_rerank_cpu(r, scores);
-#elif defined(USE_CUDA)
- int rc =
- r->model->cuda_late_ctx ? execute_rerank_cuda(r, scores) : execute_rerank_cpu(r, scores);
-#else
- int rc = execute_rerank_cpu(r, scores);
-#endif
- r->j->infer_ns += nstime() - t0;
- if (rc != 0) {
- free(scores);
- job_set_error(r->j, 500, "late-interaction reranking failed", "server_error");
- return;
- }
-
- rerank_result *ranked = xmalloc((size_t)r->n_documents * sizeof(*ranked));
- for (int i = 0; i < r->n_documents; i++) {
- ranked[i].index = i;
- ranked[i].score = scores[i];
- }
- qsort(ranked, (size_t)r->n_documents, sizeof(*ranked), rerank_result_cmp);
-
- t0 = nstime();
- sbuf b = {0};
- sbuf_printf(&b, "{\"object\":\"list\",\"model\":\"%s\",\"results\":[", r->model->info->id);
- cJSON *documents = cJSON_GetObjectItemCaseSensitive(r->root, "documents");
- for (int i = 0; i < r->top_n; i++) {
- if (i)
- sbuf_putc(&b, ',');
- sbuf_printf(&b, "{\"index\":%d,\"relevance_score\":%.9g", ranked[i].index,
- (double)ranked[i].score);
- if (r->return_documents) {
- cJSON *document = cJSON_GetArrayItem(documents, ranked[i].index);
- sbuf_puts(&b, ",\"document\":");
- append_json_string(&b, document->valuestring);
- }
- sbuf_putc(&b, '}');
- }
- sbuf_printf(&b,
- "],\"usage\":{\"query_tokens\":%d,\"document_tokens\":%d,"
- "\"total_tokens\":%d}}",
- r->query_tokens, r->document_tokens, r->query_tokens + r->document_tokens);
- set_response_from_buf(r->j, &b);
- r->j->encode_ns += nstime() - t0;
- free(ranked);
- free(scores);
-}
diff --git a/src/server_http.c b/src/server_http.c
deleted file mode 100644
index f155aec..0000000
--- a/src/server_http.c
+++ /dev/null
@@ -1,399 +0,0 @@
-#include "server_internal.h"
-#include "server_util.h"
-
-#include <ctype.h>
-#include <errno.h>
-#include <fcntl.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <strings.h>
-#include <sys/types.h>
-#include <unistd.h>
-
-/* ========================================================================
- * HTTP/1.1 connection lifecycle, header parsing, and response framing
- *
- * One client per connection, refcounted: the event loop holds a reference
- * while the socket is open and the worker thread holds one while a job derived
- * from the connection is in flight, so a request that completes after the peer
- * disconnects frees the client safely. read_cb accumulates bytes until a full
- * request is framed, then hands off to dispatch_request (server.c); write_cb
- * drains the response and either closes or arms the next pipelined request.
- * ======================================================================== */
-
-int set_nonblock(int fd) {
- int flags = fcntl(fd, F_GETFL, 0);
- if (flags < 0)
- return -1;
- return fcntl(fd, F_SETFL, flags | O_NONBLOCK);
-}
-
-static void http_request_free(http_request *r) {
- free(r->body);
- free(r->auth);
- memset(r, 0, sizeof(*r));
-}
-
-static void client_free(client *c) {
- if (!c)
- return;
- http_request_free(&c->req);
- sbuf_free(&c->in);
- sbuf_free(&c->out);
- free(c);
-}
-
-void client_incref(client *c) {
- pthread_mutex_lock(&c->srv->mu);
- c->refcount++;
- pthread_mutex_unlock(&c->srv->mu);
-}
-
-static void client_decref(client *c) {
- int free_now = 0;
- pthread_mutex_lock(&c->srv->mu);
- if (--c->refcount == 0)
- free_now = 1;
- pthread_mutex_unlock(&c->srv->mu);
- if (free_now)
- client_free(c);
-}
-
-void client_decref_n(client *c, int n) {
- int free_now = 0;
- if (n <= 0)
- return;
- pthread_mutex_lock(&c->srv->mu);
- c->refcount -= n;
- if (c->refcount <= 0)
- free_now = 1;
- pthread_mutex_unlock(&c->srv->mu);
- if (free_now)
- client_free(c);
-}
-
-static void client_unlink(client *c) {
- http_server *s = c->srv;
- if (!c->linked)
- return;
- if (c->prev)
- c->prev->next = c->next;
- else
- s->clients = c->next;
- if (c->next)
- c->next->prev = c->prev;
- c->prev = c->next = NULL;
- c->linked = 0;
- if (s->n_clients > 0)
- s->n_clients--;
-}
-
-int close_client_unlink(client *c) {
- if (!c || c->cancelled)
- return 0;
- c->cancelled = 1;
- if (c->fd >= 0) {
- aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
- aeDeleteFileEvent(c->srv->loop, c->fd, AE_WRITABLE);
- close(c->fd);
- c->fd = -1;
- }
- client_unlink(c);
- return 1;
-}
-
-void close_client(client *c) {
- if (!close_client_unlink(c))
- return;
- client_decref(c);
-}
-
-void append_http_response_ex(client *c,
- int status,
- const char *ctype,
- const char *extra_headers,
- const char *body,
- size_t body_len) {
- const char *reason = "OK";
- if (status == 204)
- reason = "No Content";
- else if (status == 400)
- reason = "Bad Request";
- else if (status == 401)
- reason = "Unauthorized";
- else if (status == 404)
- reason = "Not Found";
- else if (status == 405)
- reason = "Method Not Allowed";
- else if (status == 422)
- reason = "Unprocessable Entity";
- else if (status == 503)
- reason = "Service Unavailable";
- else if (status >= 500)
- reason = "Internal Server Error";
-
- sbuf_printf(&c->out,
- "HTTP/1.1 %d %s\r\n"
- "Content-Length: %zu\r\n"
- "Connection: %s\r\n",
- status, reason, body_len, c->close_after_write ? "close" : "keep-alive");
- if (ctype)
- sbuf_printf(&c->out, "Content-Type: %s\r\n", ctype);
-
- if (extra_headers)
- sbuf_puts(&c->out, extra_headers);
- sbuf_puts(&c->out, "\r\n");
- if (body_len)
- sbuf_append(&c->out, body, body_len);
-}
-
-void append_http_response(
- client *c, int status, const char *ctype, const char *body, size_t body_len) {
- append_http_response_ex(c, status, ctype, NULL, body, body_len);
-}
-
-static ssize_t find_header_end(const char *p, size_t n) {
- for (size_t i = 3; i < n; i++) {
- if (p[i - 3] == '\r' && p[i - 2] == '\n' && p[i - 1] == '\r' && p[i] == '\n')
- return (ssize_t)(i + 1);
- }
- for (size_t i = 1; i < n; i++) {
- if (p[i - 1] == '\n' && p[i] == '\n')
- return (ssize_t)(i + 1);
- }
- return -1;
-}
-
-static char *header_value_dup(const char *h, size_t n, const char *name) {
- size_t name_len = strlen(name);
- const char *p = h, *end = h + n;
- while (p < end) {
- const char *line = p;
- while (p < end && *p != '\n')
- p++;
- size_t len = (size_t)(p - line);
- if (len && line[len - 1] == '\r')
- len--;
- if (len > name_len && !strncasecmp(line, name, name_len) && line[name_len] == ':') {
- const char *v = line + name_len + 1;
- while (v < line + len && isspace((unsigned char)*v))
- v++;
- const char *e = line + len;
- while (e > v && isspace((unsigned char)e[-1]))
- e--;
- return xstrndup(v, (size_t)(e - v));
- }
- if (p < end)
- p++;
- }
- return NULL;
-}
-
-static int header_has_token(const char *value, const char *token) {
- size_t token_len = strlen(token);
- const char *p = value;
- while (p && *p) {
- while (*p == ',' || isspace((unsigned char)*p))
- p++;
- const char *start = p;
- while (*p && *p != ',')
- p++;
- const char *end = p;
- while (end > start && isspace((unsigned char)end[-1]))
- end--;
- if ((size_t)(end - start) == token_len && !strncasecmp(start, token, token_len))
- return 1;
- }
- return 0;
-}
-
-static int parse_request_headers(client *c) {
- char line[512];
- size_t i = 0;
- while (i < c->header_len && c->in.ptr[i] != '\n' && i + 1 < sizeof(line)) {
- line[i] = c->in.ptr[i];
- i++;
- }
- line[i] = '\0';
- int fields = sscanf(line, "%7s %255s %15s", c->req.method, c->req.path, c->req.version);
- if (fields < 2)
- return -1;
- if (fields < 3)
- snprintf(c->req.version, sizeof(c->req.version), "HTTP/1.0");
- char *q = strchr(c->req.path, '?');
- if (q)
- *q = '\0';
-
- c->close_after_write = strcasecmp(c->req.version, "HTTP/1.1") != 0;
- char *conn = header_value_dup(c->in.ptr, c->header_len, "Connection");
- if (conn) {
- if (header_has_token(conn, "close"))
- c->close_after_write = 1;
- else if (header_has_token(conn, "keep-alive"))
- c->close_after_write = 0;
- free(conn);
- }
-
- char *cl = header_value_dup(c->in.ptr, c->header_len, "Content-Length");
- if (cl) {
- char *end = NULL;
- errno = 0;
- unsigned long long v = strtoull(cl, &end, 10);
- if (errno || !end || *end || v > EMBED_HTTP_MAX_BODY) {
- free(cl);
- return -1;
- }
- c->content_length = (size_t)v;
- free(cl);
- }
- c->req.auth = header_value_dup(c->in.ptr, c->header_len, "Authorization");
- return 0;
-}
-
-static int complete_request(client *c) {
- if (!c->header_done) {
- ssize_t hend = find_header_end(c->in.ptr, c->in.len);
- if (hend < 0) {
- if (c->in.len > EMBED_HTTP_MAX_HEADER)
- return -1;
- return 0;
- }
- c->header_done = 1;
- c->header_len = (size_t)hend;
- if (parse_request_headers(c) != 0)
- return -1;
- }
- if (c->content_length > EMBED_HTTP_MAX_BODY)
- return -1;
- if (c->in.len < c->header_len + c->content_length)
- return 0;
- c->req.body_len = c->content_length;
- c->req.body = xmalloc(c->content_length + 1);
- memcpy(c->req.body, c->in.ptr + c->header_len, c->content_length);
- c->req.body[c->content_length] = '\0';
- return 1;
-}
-
-static void send_bad_http_request(client *c) {
- size_t len;
- char *body = json_error_body("bad HTTP request", "invalid_request_error", &len);
- c->close_after_write = 1;
- append_http_response(c, 400, "application/json", body, len);
- free(body);
- aeDeleteFileEvent(c->srv->loop, c->fd, AE_READABLE);
- if (queue_write(c) != AE_OK)
- close_client(c);
-}
-
-static void reset_client_for_next_request(client *c) {
- size_t consumed = c->header_len + c->content_length;
- if (consumed > c->in.len)
- consumed = c->in.len;
- size_t remain = c->in.len - consumed;
- if (remain)
- memmove(c->in.ptr, c->in.ptr + consumed, remain);
- c->in.len = remain;
- if (c->in.ptr)
- c->in.ptr[remain] = '\0';
-
- http_request_free(&c->req);
- sbuf_clear(&c->out);
- c->sent = 0;
- c->header_done = 0;
- c->header_len = 0;
- c->content_length = 0;
- c->close_after_write = 0;
-}
-
-static void arm_next_request(client *c) {
- reset_client_for_next_request(c);
- if (c->cancelled || c->fd < 0)
- return;
-
- if (c->in.len) {
- int done = complete_request(c);
- if (done < 0) {
- send_bad_http_request(c);
- return;
- }
- if (done > 0) {
- dispatch_request(c);
- return;
- }
- }
-
- if (aeCreateFileEvent(c->srv->loop, c->fd, AE_READABLE, read_cb, c) != AE_OK)
- close_client(c);
-}
-
-static void write_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop;
- (void)mask;
- client *c = clientData;
- while (c->sent < c->out.len) {
- ssize_t n = write(fd, c->out.ptr + c->sent, c->out.len - c->sent);
- if (n > 0) {
- c->sent += (size_t)n;
- c->last_active_ms = mstime();
- continue;
- }
- if (n < 0 && errno == EINTR)
- continue;
- if (n < 0 && (errno == EAGAIN || errno == EWOULDBLOCK))
- return;
- close_client(c);
- return;
- }
- aeDeleteFileEvent(c->srv->loop, fd, AE_WRITABLE);
- if (c->close_after_write)
- close_client(c);
- else
- arm_next_request(c);
-}
-
-int queue_write(client *c) {
- if (!c || c->cancelled || c->fd < 0)
- return -1;
- return aeCreateFileEvent(c->srv->loop, c->fd, AE_WRITABLE, write_cb, c);
-}
-
-bool route_is_inference(const char *method, const char *path) {
- return !strcmp(method, "POST") &&
- (!strcmp(path, "/v1/embeddings") || !strcmp(path, "/v1/contextualizedembeddings") ||
- !strcmp(path, "/v1/rerank"));
-}
-
-void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop;
- (void)mask;
- client *c = clientData;
- char tmp[EMBED_HTTP_IO_BUF];
- for (;;) {
- ssize_t n = read(fd, tmp, sizeof(tmp));
- if (n > 0) {
- sbuf_append(&c->in, tmp, (size_t)n);
- c->last_active_ms = mstime();
- int done = complete_request(c);
- if (done < 0) {
- send_bad_http_request(c);
- return;
- }
- if (done > 0) {
- dispatch_request(c);
- return;
- }
- continue;
- }
- if (n == 0) {
- close_client(c);
- return;
- }
- if (errno == EINTR)
- continue;
- if (errno == EAGAIN || errno == EWOULDBLOCK)
- return;
- close_client(c);
- return;
- }
-}
diff --git a/src/server_internal.h b/src/server_internal.h
deleted file mode 100644
index 1f106ca..0000000
--- a/src/server_internal.h
+++ /dev/null
@@ -1,361 +0,0 @@
-#ifndef EMBED_SERVER_INTERNAL_H
-#define EMBED_SERVER_INTERNAL_H
-
-/* Declarations shared across the embed-server translation units (HTTP framing,
- * JSON validation, output encoding, scheduling, model lifecycle, endpoint
- * handlers). The server is split by concern but compiles as several objects
- * linked into one binary; this header is what they share. Not a public API -
- * the CLI and library do not include it. */
-
-#include "embed.h"
-#include "server.h"
-#include "sbuf.h"
-#include "tokenizer_bpe.h"
-#include "tokenizer_sentencepiece.h"
-#include "tokenizer_wordpiece.h"
-
-#ifdef USE_MLX
-#include "mlx.h"
-#endif
-#ifdef USE_CUDA
-#include "cuda.h"
-#endif
-
-#include "deps/ae/ae.h"
-
-#include <cjson/cJSON.h>
-#include <pthread.h>
-#include <stdbool.h>
-#include <stddef.h>
-#include <stdint.h>
-
-#define EMBED_HTTP_IO_BUF 8192
-#define EMBED_HTTP_MAX_HEADER (64u * 1024u)
-#define EMBED_HTTP_MAX_BODY (64u * 1024u * 1024u)
-#define EMBED_HTTP_MAX_PATH 255u
-#define EMBED_HTTP_BACKLOG 128
-#define EMBED_HTTP_SETSIZE 10240
-#define EMBED_HTTP_CLIENT_TIMEOUT_MS 30000
-#define EMBED_HTTP_SWEEP_MS 1000
-
-#define EMBED_API_MAX_STANDARD_INPUTS 512
-#define EMBED_API_MAX_CONTEXT_DOCS 512
-#define EMBED_API_MAX_CONTEXT_CHUNKS 16000
-#define EMBED_API_MAX_RERANK_DOCUMENTS 1000
-#define EMBED_API_MAX_ITEM_TOKENS 32768
-#define EMBED_API_MAX_TOTAL_TOKENS 120000
-#define EMBED_LATE_QUERY_TOKENS 32
-#define EMBED_LATE_MASK_TOKEN_ID 151642
-#define EMBED_LATE_QUERY_PREFIX_ID 151669
-#define EMBED_LATE_DOCUMENT_PREFIX_ID 151670
-/* Chosen from the L4 scheduler sweep: -b 32 gained ~24% concurrent
- * short-request throughput over 8 with no long-document penalty, while 128
- * was no faster and inflated long-document tail latency. */
-#define EMBED_SERVER_DEFAULT_BATCH_SIZE 32
-#define EMBED_SERVER_DEFAULT_MAX_BATCH_TOKENS 16384
-/* Microseconds the worker waits for more requests before dispatching a batch.
- * CUDA uses a 1 ms window so arrivals group into one launch and keep the GPU
- * busy; MLX and CPU dispatch immediately, where waiting only adds latency.
- * Override with --batch-wait-us. */
-#define EMBED_SERVER_BATCH_WAIT_US 0
-#define EMBED_SERVER_CUDA_BATCH_WAIT_US 1000
-#define EMBED_SERVER_MICROBATCH_MAX_JOBS 128
-#define EMBED_MLX_MEMORY_BUDGET_PERCENT 90
-#define EMBED_MLX_RESIDENT_MULTIPLIER 2
-
-typedef enum { MODEL_KIND_STANDARD, MODEL_KIND_CONTEXTUAL, MODEL_KIND_LATE } model_kind;
-
-typedef enum {
- EMBED_API_OPENAI = 0,
- EMBED_API_PERPLEXITY = 1,
-} embedding_api_t;
-
-typedef struct {
- char *id;
- model_kind kind;
- int dim;
- int min_dim;
- int token_dim;
- embed_attention_mode_t attention_mode;
- embed_pooling_mode_t pooling_mode;
- int normalize_embeddings;
- embedding_api_t api;
- char *query_instruct;
-} model_info;
-
-typedef struct {
- model_info *info;
- char *path;
- embed_tokenizer_t *tok;
- embed_tokenizer_workspace_t *tok_ws;
- /* BERT-family WordPiece tokenizer, selected by file probe in load_one_model.
- * When wp_tok is set, tokenize_one wraps the ids with [CLS]/[SEP] and the
- * Qwen byte-level BPE fields above stay NULL. */
- wordpiece_tokenizer_t *wp_tok;
- wordpiece_workspace_t *wp_tok_ws;
- sentencepiece_tokenizer_t *sp_tok;
- sentencepiece_workspace_t *sp_tok_ws;
- int cls_id;
- int sep_id;
- int context_separator_id;
- int late_mask_id;
- int late_query_prefix_id;
- int late_document_prefix_id;
- int late_skip_ids[64];
- int n_late_skip_ids;
- int append_terminal_token;
- int terminal_token_id;
- int renormalize_truncated;
- embed_model_t *cpu_model;
- embed_workspace_t *cpu_ws;
- embed_late_model_t *cpu_late_model;
- embed_late_workspace_t *cpu_late_ws;
-#ifdef USE_MLX
- embed_mlx_ctx_t *mlx_ctx;
- embed_mlx_late_ctx_t *mlx_late_ctx;
-#endif
-#ifdef USE_CUDA
- embed_cuda_ctx_t *cuda_ctx;
- embed_cuda_late_ctx_t *cuda_late_ctx;
-#endif
-} loaded_model;
-
-typedef struct client client;
-typedef struct job job;
-
-typedef struct {
- aeEventLoop *loop;
- int listen_fd;
- int completion_pipe[2];
- int signal_pipe[2];
- const char *host;
- int port;
- int batch_size;
- int max_batch_tokens;
- int batch_wait_us;
- char *api_key;
- loaded_model *models;
- int n_models;
- int use_mlx;
- int use_cuda;
- int mlx_quantize_bits;
- int mlx_quantize_group_size;
-
- pthread_t worker;
- pthread_mutex_t mu;
- pthread_cond_t cv;
- int stopping;
- int worker_ready;
- int worker_init_rc;
- job *job_head;
- job *job_tail;
- job *done_head;
- job *done_tail;
-
- client *clients;
- int n_clients;
-} http_server;
-
-typedef struct {
- char method[8];
- char path[EMBED_HTTP_MAX_PATH + 1];
- char version[16];
- char *body;
- size_t body_len;
- char *auth;
-} http_request;
-
-struct client {
- http_server *srv;
- int fd;
- int refcount;
- int cancelled;
- int linked;
- uint64_t last_active_ms;
- sbuf in;
- sbuf out;
- size_t sent;
- int header_done;
- size_t header_len;
- size_t content_length;
- int close_after_write;
- http_request req;
- client *prev;
- client *next;
-};
-
-struct job {
- http_server *srv;
- client *c;
- char method[8];
- char path[EMBED_HTTP_MAX_PATH + 1];
- char *body;
- size_t body_len;
- char *auth;
- int status;
- char *content_type;
- char *extra_headers;
- char *response;
- size_t response_len;
- uint64_t created_ns;
- uint64_t started_ns;
- uint64_t parse_ns;
- uint64_t tokenize_ns;
- uint64_t infer_ns;
- uint64_t encode_ns;
- job *next;
-};
-
-/* Parsed request state, shared by the prepare (handlers), execute (schedule),
- * and render (encode) stages. Each carries its source cJSON root (freed when
- * the request is freed), the resolved model, and the tokenized inputs. */
-typedef struct {
- int *ids;
- int n_tokens;
-} token_buf;
-
-typedef struct {
- job *j;
- cJSON *root;
- loaded_model *model;
- int dims;
- const char *encoding;
- int n_inputs;
- token_buf *tokens;
- embed_input_t *inputs;
- int total_tokens;
- int ready;
-} embedding_request;
-
-typedef struct {
- int *ids;
- int n_tokens;
- embed_span_t *spans;
- int n_spans;
-} contextual_doc;
-
-typedef struct {
- job *j;
- cJSON *root;
- loaded_model *model;
- int dims;
- const char *encoding;
- contextual_doc *docs;
- int n_docs;
- int total_chunks;
- int total_tokens;
- int ready;
-} contextual_request;
-
-typedef struct {
- int *ids;
- int n_tokens;
- int *keep;
- int n_keep;
-} late_tokens;
-
-typedef struct {
- job *j;
- cJSON *root;
- loaded_model *model;
- late_tokens query;
- late_tokens *documents;
- int n_documents;
- int top_n;
- int return_documents;
- int query_tokens;
- int document_tokens;
- int ready;
-} rerank_request;
-
-/* ---- server_json.c: request validation and error bodies ---- */
-void append_json_string(sbuf *b, const char *s);
-char *json_error_body(const char *message, const char *type, size_t *len);
-void job_set_error(job *j, int status, const char *message, const char *type);
-bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type);
-void job_set_422(job *j, cJSON *detail);
-cJSON *parse_json_body(job *j, cJSON *detail);
-bool cjson_is_integer(cJSON *item);
-const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api);
-int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct);
-int dimensions_from_root(
- cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding);
-
-/* ---- server_http.c: connection lifecycle, header parse, response framing ---- */
-int set_nonblock(int fd);
-void client_incref(client *c);
-void client_decref_n(client *c, int n);
-int close_client_unlink(client *c);
-void close_client(client *c);
-void append_http_response_ex(client *c,
- int status,
- const char *ctype,
- const char *extra_headers,
- const char *body,
- size_t body_len);
-void append_http_response(
- client *c, int status, const char *ctype, const char *body, size_t body_len);
-int queue_write(client *c);
-bool route_is_inference(const char *method, const char *path);
-void read_cb(aeEventLoop *loop, int fd, void *clientData, int mask);
-/* Defined in server.c (request routing): framed requests are handed here. */
-void dispatch_request(client *c);
-
-/* ---- server_encode.c: embedding output encoding ---- */
-signed char quantize_int8_tanh(float x);
-char *encode_embedding(const float *emb, int dims, const char *encoding);
-void append_embedding_value(
- sbuf *b, int index, const float *emb, int dims, const char *encoding, embedding_api_t api);
-void set_response_from_buf(job *j, sbuf *b);
-int render_embedding_response(embedding_request *r, const float *embs);
-void render_contextual_response(contextual_request *r, const float *embs);
-
-/* ---- server_models.c: model load/lifecycle, tokenize/inference dispatch ---- */
-loaded_model *loaded_model_for_label(http_server *s, const char *label);
-void free_token_bufs(token_buf *t, int n);
-void embedding_request_free(embedding_request *r);
-void contextual_request_free(contextual_request *r);
-void late_tokens_free(late_tokens *t);
-void rerank_request_free(rerank_request *r);
-int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out);
-int tokenize_input(
- loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out);
-int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out);
-int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out);
-int model_embed_spans_batch(loaded_model *m,
- const embed_context_input_t *inputs,
- int batch,
- float *out);
-int inference_batch_accepts_input(
- const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens);
-int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim);
-int load_one_model(http_server *s, const embed_server_model_spec_t *spec);
-void free_models(http_server *s);
-
-/* ---- server_handlers.c: per-endpoint prepare / batch-group / execute ---- */
-void prepare_embedding_request(job *j, cJSON *root, http_server *s, embedding_request *out);
-int embedding_request_compatible(const embedding_request *a, const embedding_request *b);
-int embedding_request_fits_group(const embedding_request *r,
- int group_inputs,
- int group_tokens,
- int max_batch,
- int max_batch_tokens);
-void execute_embedding_request_list(embedding_request **reqs, int n_reqs);
-void prepare_contextual_request(job *j, cJSON *root, http_server *s, contextual_request *out);
-int contextual_request_compatible(const contextual_request *a, const contextual_request *b);
-int contextual_request_fits_group(const contextual_request *r,
- int group_docs,
- int group_tokens,
- int max_batch,
- int max_batch_tokens);
-void execute_contextual_request_list(contextual_request **reqs, int n_reqs);
-void prepare_rerank_request(job *j, cJSON *root, http_server *s, rerank_request *out);
-void execute_rerank_request(rerank_request *r);
-
-/* ---- server_schedule.c: job queue, micro-batching, completion ---- */
-void enqueue_job(job *j);
-int collect_job_batch(http_server *s, job **jobs, int max_jobs);
-void process_job_group(http_server *s, job **jobs, int n_jobs);
-void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask);
-
-#endif /* EMBED_SERVER_INTERNAL_H */
diff --git a/src/server_json.c b/src/server_json.c
deleted file mode 100644
index cca8fd3..0000000
--- a/src/server_json.c
+++ /dev/null
@@ -1,221 +0,0 @@
-#include "server_internal.h"
-#include "server_util.h"
-
-#include <cjson/cJSON.h>
-#include <ctype.h>
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-/* ========================================================================
- * Request validation and error bodies
- *
- * Two JSON shapes are produced here: the OpenAI-style {"error":{message,type}}
- * body (append_json_error / json_error_body / job_set_error) for transport and
- * auth failures, and the Perplexity-style {"detail":[...]} 422 body built from
- * ve_add entries for field validation. The parse_* / *_from_root helpers read
- * and validate the request object; append_json_string escapes a value for
- * inclusion in a response. */
-
-static void append_json_error(sbuf *b, const char *message, const char *type) {
- sbuf_puts(b, "{\"error\":{\"message\":");
- append_json_string(b, message ? message : "");
- sbuf_puts(b, ",\"type\":");
- append_json_string(b, type ? type : "server_error");
- sbuf_puts(b, "}}");
-}
-
-void append_json_string(sbuf *b, const char *s) {
- sbuf_putc(b, '"');
- for (const unsigned char *p = (const unsigned char *)(s ? s : ""); *p; p++) {
- switch (*p) {
- case '"':
- sbuf_puts(b, "\\\"");
- break;
- case '\\':
- sbuf_puts(b, "\\\\");
- break;
- case '\b':
- sbuf_puts(b, "\\b");
- break;
- case '\f':
- sbuf_puts(b, "\\f");
- break;
- case '\n':
- sbuf_puts(b, "\\n");
- break;
- case '\r':
- sbuf_puts(b, "\\r");
- break;
- case '\t':
- sbuf_puts(b, "\\t");
- break;
- default:
- if (*p < 0x20)
- sbuf_printf(b, "\\u%04x", (unsigned)*p);
- else
- sbuf_putc(b, (char)*p);
- }
- }
- sbuf_putc(b, '"');
-}
-
-char *json_error_body(const char *message, const char *type, size_t *len) {
- sbuf b = {0};
- append_json_error(&b, message, type);
- *len = b.len;
- return b.ptr;
-}
-
-void job_set_error(job *j, int status, const char *message, const char *type) {
- free(j->response);
- j->status = status;
- j->content_type = "application/json";
- j->response = json_error_body(message, type, &j->response_len);
-}
-
-bool ve_add(cJSON *detail, const char *loc_json, const char *msg, const char *type) {
- cJSON *obj = cJSON_CreateObject();
- cJSON *loc = cJSON_Parse(loc_json);
- if (!obj || !loc) {
- cJSON_Delete(obj);
- cJSON_Delete(loc);
- return false;
- }
- cJSON_AddItemToObject(obj, "loc", loc);
- cJSON_AddStringToObject(obj, "msg", msg ? msg : "");
- cJSON_AddStringToObject(obj, "type", type ? type : "");
- cJSON_AddItemToArray(detail, obj);
- return true;
-}
-
-void job_set_422(job *j, cJSON *detail) {
- cJSON *root = cJSON_CreateObject();
- cJSON_AddItemToObject(root, "detail", cJSON_Duplicate(detail, 1));
- char *s = cJSON_PrintUnformatted(root);
- cJSON_Delete(root);
- if (!s) {
- job_set_error(j, 500, "failed to render validation error", "server_error");
- return;
- }
- free(j->response);
- j->status = 422;
- j->content_type = "application/json";
- j->response = xstrdup(s);
- j->response_len = strlen(j->response);
- cJSON_free(s);
-}
-
-cJSON *parse_json_body(job *j, cJSON *detail) {
- if (!j->body || j->body_len == 0) {
- ve_add(detail, "[\"body\"]", "body required", "missing");
- return NULL;
- }
- const char *end = NULL;
- cJSON *root = cJSON_ParseWithLengthOpts(j->body, j->body_len, &end, false);
- if (!root) {
- ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid");
- return NULL;
- }
- const char *body_end = j->body + j->body_len;
- while (end && end < body_end && isspace((unsigned char)*end))
- end++;
- if (!end || end != body_end) {
- ve_add(detail, "[\"body\"]", "invalid JSON", "json_invalid");
- cJSON_Delete(root);
- return NULL;
- }
- if (!cJSON_IsObject(root)) {
- ve_add(detail, "[\"body\"]", "body must be a JSON object", "type_error.object");
- cJSON_Delete(root);
- return NULL;
- }
- return root;
-}
-
-bool cjson_is_integer(cJSON *item) {
- if (!cJSON_IsNumber(item))
- return false;
- double d = item->valuedouble;
- int i = item->valueint;
- return d >= (double)INT_MIN && d <= (double)INT_MAX && d == (double)i;
-}
-
-const char *encoding_from_root(cJSON *root, cJSON *detail, embedding_api_t api) {
- const char *dflt = api == EMBED_API_PERPLEXITY ? "base64_int8" : "float";
- cJSON *encoding = cJSON_GetObjectItemCaseSensitive(root, "encoding_format");
- if (!encoding)
- return dflt;
- if (!cJSON_IsString(encoding) || !encoding->valuestring) {
- ve_add(detail, "[\"body\",\"encoding_format\"]", "encoding_format must be a string",
- "type_error.string");
- return dflt;
- }
- const char *v = encoding->valuestring;
- int ok = api == EMBED_API_PERPLEXITY
- ? (!strcmp(v, "base64_int8") || !strcmp(v, "base64_binary") ||
- !strcmp(v, "float"))
- : (!strcmp(v, "float") || !strcmp(v, "base64"));
- if (!ok) {
- ve_add(detail, "[\"body\",\"encoding_format\"]",
- api == EMBED_API_PERPLEXITY
- ? "value is not a valid enum member; permitted: "
- "'base64_int8', 'base64_binary', 'float'"
- : "value is not a valid enum member; permitted: 'float', 'base64'",
- "enum");
- return dflt;
- }
- return v;
-}
-
-/* Parse the `text_type` hint. Returns 1 when the model's query instruction
- * should be prepended, 0 otherwise (document / absent). Models without a
- * published query instruction reject the field. */
-int text_type_is_query(cJSON *root, cJSON *detail, const char *query_instruct) {
- cJSON *tt = cJSON_GetObjectItemCaseSensitive(root, "text_type");
- if (!tt)
- return 0;
- if (!query_instruct) {
- ve_add(detail, "[\"body\",\"text_type\"]",
- "text_type is only supported for instruction embedding models", "extra_fields");
- return 0;
- }
- if (!cJSON_IsString(tt) || !tt->valuestring) {
- ve_add(detail, "[\"body\",\"text_type\"]", "text_type must be a string",
- "type_error.string");
- return 0;
- }
- const char *v = tt->valuestring;
- if (!strcmp(v, "query"))
- return 1;
- if (!strcmp(v, "document"))
- return 0;
- ve_add(detail, "[\"body\",\"text_type\"]",
- "value is not a valid enum member; permitted: 'query', 'document'", "enum");
- return 0;
-}
-
-int dimensions_from_root(
- cJSON *root, cJSON *detail, int min_dim, int max_dim, const char *encoding) {
- int dims = max_dim;
- cJSON *dimensions = cJSON_GetObjectItemCaseSensitive(root, "dimensions");
- if (dimensions) {
- if (!cjson_is_integer(dimensions)) {
- ve_add(detail, "[\"body\",\"dimensions\"]", "dimensions must be an integer",
- "type_error.integer");
- } else {
- dims = dimensions->valueint;
- if (dims < min_dim || dims > max_dim) {
- char msg[96];
- snprintf(msg, sizeof(msg), "dimensions must be between %d and %d", min_dim,
- max_dim);
- ve_add(detail, "[\"body\",\"dimensions\"]", msg, "value_error.range");
- }
- }
- }
- if (!strcmp(encoding, "base64_binary") && dims % 8 != 0)
- ve_add(detail, "[\"body\",\"dimensions\"]",
- "dimensions must be divisible by 8 for base64_binary", "value_error.divisible");
- return dims;
-}
diff --git a/src/server_models.c b/src/server_models.c
deleted file mode 100644
index 320840b..0000000
--- a/src/server_models.c
+++ /dev/null
@@ -1,627 +0,0 @@
-#include "server_internal.h"
-#include "server_util.h"
-
-#include <errno.h>
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <unistd.h>
-
-/* ========================================================================
- * Model load/lifecycle, and tokenize/inference dispatch
- *
- * Each loaded model owns resolved serving metadata for the operator-chosen
- * label. Transformer behavior comes from the model files through embed_config;
- * server-only facts absent from those files, such as output API family and
- * Matryoshka floor, come from the load spec. The tokenize_* and model_embed_*
- * helpers are the single dispatch point the request handlers call, hiding the
- * CPU/MLX/CUDA backend choice behind one signature.
- * ======================================================================== */
-
-loaded_model *loaded_model_for_label(http_server *s, const char *label) {
- if (!label)
- return NULL;
- for (int i = 0; i < s->n_models; i++) {
- if (s->models[i].info && !strcmp(label, s->models[i].info->id))
- return &s->models[i];
- }
- return NULL;
-}
-
-static model_kind model_kind_from_public(embed_server_model_kind_t kind) {
- switch (kind) {
- case EMBED_SERVER_MODEL_CONTEXTUAL:
- return MODEL_KIND_CONTEXTUAL;
- case EMBED_SERVER_MODEL_LATE:
- return MODEL_KIND_LATE;
- case EMBED_SERVER_MODEL_STANDARD:
- default:
- return MODEL_KIND_STANDARD;
- }
-}
-
-static embedding_api_t embedding_api_from_public(embed_server_embedding_api_t api) {
- return api == EMBED_SERVER_API_PERPLEXITY ? EMBED_API_PERPLEXITY : EMBED_API_OPENAI;
-}
-
-static int read_optional_text_file(const char *model_dir, const char *relative, char **out) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/%s", model_dir, relative);
- if (n < 0 || (size_t)n >= sizeof(path)) {
- server_log("embed-server: model path too long");
- return -1;
- }
- FILE *f = fopen(path, "rb");
- if (!f && errno == ENOENT)
- return 0;
- if (!f) {
- server_log("embed-server: failed to open %s", path);
- return -1;
- }
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return -1;
- }
- long sz = ftell(f);
- if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return -1;
- }
- char *buf = (char *)malloc((size_t)sz + 1);
- if (!buf) {
- fclose(f);
- return -1;
- }
- if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
- free(buf);
- fclose(f);
- return -1;
- }
- fclose(f);
- buf[sz] = '\0';
- *out = buf;
- return 1;
-}
-
-static int read_query_instruct_from_files(const char *model_dir, char **out) {
- char *buf = NULL;
- int rc = read_optional_text_file(model_dir, "config_sentence_transformers.json", &buf);
- if (rc <= 0)
- return rc;
- cJSON *root = cJSON_Parse(buf);
- free(buf);
- if (!root) {
- server_log("embed-server: invalid config_sentence_transformers.json: %s", model_dir);
- return -1;
- }
- cJSON *prompts = cJSON_GetObjectItemCaseSensitive(root, "prompts");
- cJSON *query = prompts ? cJSON_GetObjectItemCaseSensitive(prompts, "query") : NULL;
- if (query && !cJSON_IsString(query)) {
- cJSON_Delete(root);
- server_log("embed-server: prompts.query must be a string: %s", model_dir);
- return -1;
- }
- if (query && query->valuestring && query->valuestring[0]) {
- *out = xstrdup(query->valuestring);
- cJSON_Delete(root);
- return *out ? 1 : -1;
- }
- cJSON_Delete(root);
- return 0;
-}
-
-static int init_model_info(model_info **out, const embed_server_model_spec_t *spec) {
- model_info *info = xcalloc(1, sizeof(*info));
- info->id = xstrdup(spec->id);
- info->kind = model_kind_from_public(spec->kind);
- info->api = embedding_api_from_public(spec->api);
- info->min_dim = spec->min_dim;
- if (spec->query_instruct) {
- info->query_instruct = xstrdup(spec->query_instruct);
- } else if (read_query_instruct_from_files(spec->path, &info->query_instruct) < 0) {
- free(info->id);
- free(info);
- return -1;
- }
- if (!info->id || (spec->query_instruct && !info->query_instruct)) {
- free(info->query_instruct);
- free(info->id);
- free(info);
- return -1;
- }
- *out = info;
- return 0;
-}
-
-static void free_model_info(model_info *info) {
- if (!info)
- return;
- free(info->id);
- free(info->query_instruct);
- free(info);
-}
-
-void free_token_bufs(token_buf *t, int n) {
- if (!t)
- return;
- for (int i = 0; i < n; i++)
- free(t[i].ids);
-}
-
-void embedding_request_free(embedding_request *r) {
- if (!r)
- return;
- free_token_bufs(r->tokens, r->n_inputs);
- free(r->tokens);
- free(r->inputs);
- if (r->root)
- cJSON_Delete(r->root);
- memset(r, 0, sizeof(*r));
-}
-
-void contextual_request_free(contextual_request *r) {
- if (!r)
- return;
- for (int i = 0; i < r->n_docs; i++) {
- free(r->docs[i].ids);
- free(r->docs[i].spans);
- }
- free(r->docs);
- if (r->root)
- cJSON_Delete(r->root);
- memset(r, 0, sizeof(*r));
-}
-
-void late_tokens_free(late_tokens *t) {
- if (!t)
- return;
- free(t->ids);
- free(t->keep);
- memset(t, 0, sizeof(*t));
-}
-
-void rerank_request_free(rerank_request *r) {
- if (!r)
- return;
- late_tokens_free(&r->query);
- for (int i = 0; i < r->n_documents; i++)
- late_tokens_free(&r->documents[i]);
- free(r->documents);
- if (r->root)
- cJSON_Delete(r->root);
- memset(r, 0, sizeof(*r));
-}
-
-int tokenize_one(loaded_model *m, job *j, const char *text, token_buf *out) {
- memset(out, 0, sizeof(*out));
- uint64_t t0 = nstime();
- if (m->wp_tok) {
- int n = 0;
- int *core = wordpiece_tokenizer_encode_with_workspace(m->wp_tok, m->wp_tok_ws, text, &n);
- if (j)
- j->tokenize_ns += nstime() - t0;
- /* BERT wraps the WordPiece ids with [CLS] ... [SEP], exactly as the BPE
- * branch below appends its terminal token. A whitespace-only input
- * cleans to zero core tokens (core == NULL, n == 0); the bare
- * [CLS][SEP] pair is what Hugging Face produces for it too. Grow the
- * returned buffer in place so only one allocation is touched. */
- if (n < 0 || n > INT_MAX - 2) {
- free(core);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = m->cls_id;
- ids[n + 1] = m->sep_id;
- out->ids = ids;
- out->n_tokens = n + 2;
- return 0;
- }
- if (m->sp_tok) {
- int n = 0;
- int *core =
- sentencepiece_tokenizer_encode_with_workspace(m->sp_tok, m->sp_tok_ws, text, &n);
- if (j)
- j->tokenize_ns += nstime() - t0;
- if (n < 0 || n > INT_MAX - 2) {
- free(core);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- int *ids = realloc(core, (size_t)(n + 2) * sizeof(*ids));
- if (!ids) {
- free(core);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- memmove(ids + 1, ids, (size_t)n * sizeof(*ids));
- ids[0] = m->cls_id;
- ids[n + 1] = m->sep_id;
- out->ids = ids;
- out->n_tokens = n + 2;
- return 0;
- }
- out->ids = embed_tokenizer_encode_with_workspace(m->tok, m->tok_ws, text, &out->n_tokens);
- if (j)
- j->tokenize_ns += nstime() - t0;
- if (!out->ids || out->n_tokens <= 0) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- if (m->append_terminal_token) {
- if (out->n_tokens == INT_MAX) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- int *ids = realloc(out->ids, (size_t)(out->n_tokens + 1) * sizeof(*out->ids));
- if (!ids) {
- free(out->ids);
- memset(out, 0, sizeof(*out));
- return -1;
- }
- out->ids = ids;
- out->ids[out->n_tokens++] = m->terminal_token_id;
- }
- return 0;
-}
-
-/* Tokenize one embedding input, optionally prepending the model's query
- * instruction. The instruction and input are tokenized in one pass. */
-int tokenize_input(
- loaded_model *m, job *j, const char *text, const char *query_instruct, token_buf *out) {
- if (!query_instruct)
- return tokenize_one(m, j, text, out);
- size_t plen = strlen(query_instruct);
- size_t tlen = strlen(text);
- char *buf = (char *)malloc(plen + tlen + 1);
- if (!buf) {
- memset(out, 0, sizeof(*out));
- return -1;
- }
- memcpy(buf, query_instruct, plen);
- memcpy(buf + plen, text, tlen + 1);
- int rc = tokenize_one(m, j, buf, out);
- free(buf);
- return rc;
-}
-
-static int late_id_is_skipped(const loaded_model *m, int id) {
- for (int i = 0; i < m->n_late_skip_ids; i++) {
- if (m->late_skip_ids[i] == id)
- return 1;
- }
- return 0;
-}
-
-int tokenize_late_text(loaded_model *m, job *j, const char *text, int is_query, late_tokens *out) {
- token_buf raw = {0};
- if (tokenize_one(m, j, text, &raw) != 0)
- return -1;
-
- int raw_tokens = raw.n_tokens;
- int target = raw_tokens + 1;
- if (is_query && target < EMBED_LATE_QUERY_TOKENS)
- target = EMBED_LATE_QUERY_TOKENS;
-
- out->ids = xmalloc((size_t)target * sizeof(*out->ids));
- out->ids[0] = raw.ids[0];
- out->ids[1] = is_query ? m->late_query_prefix_id : m->late_document_prefix_id;
- if (raw_tokens > 1) {
- memcpy(out->ids + 2, raw.ids + 1, (size_t)(raw_tokens - 1) * sizeof(*out->ids));
- }
- out->n_tokens = raw_tokens + 1;
- if (is_query) {
- while (out->n_tokens < EMBED_LATE_QUERY_TOKENS)
- out->ids[out->n_tokens++] = m->late_mask_id;
- out->n_keep = out->n_tokens;
- } else {
- out->keep = xmalloc((size_t)out->n_tokens * sizeof(*out->keep));
- for (int i = 0; i < out->n_tokens; i++) {
- if (!late_id_is_skipped(m, out->ids[i]))
- out->keep[out->n_keep++] = i;
- }
- }
- free(raw.ids);
- return out->n_keep > 0 ? 0 : -1;
-}
-
-int model_embed_batch(loaded_model *m, const embed_input_t *inputs, int batch, float *out) {
-#ifdef USE_MLX
- if (m->mlx_ctx)
- return embed_mlx_encode_batch(m->mlx_ctx, inputs, batch, out);
-#endif
-#ifdef USE_CUDA
- if (m->cuda_ctx)
- return embed_cuda_encode_batch(m->cuda_ctx, inputs, batch, out);
-#endif
- return embed_model_encode_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
-}
-
-int model_embed_spans_batch(loaded_model *m,
- const embed_context_input_t *inputs,
- int batch,
- float *out) {
-#ifdef USE_MLX
- if (m->mlx_ctx)
- return embed_mlx_encode_spans_batch(m->mlx_ctx, inputs, batch, out);
-#endif
-#ifdef USE_CUDA
- if (m->cuda_ctx)
- return embed_cuda_encode_spans_batch(m->cuda_ctx, inputs, batch, out);
-#endif
- return embed_model_encode_spans_batch(m->cpu_model, m->cpu_ws, inputs, batch, out);
-}
-
-static int model_uses_dense_batches(const loaded_model *m) {
-#ifdef USE_MLX
- return m->mlx_ctx != NULL;
-#else
- (void)m;
- return 0;
-#endif
-}
-
-/* Inputs are sorted by length before chunking. CPU packs real tokens while MLX
- * pads each dense row to the longest input in the chunk. */
-int inference_batch_accepts_input(
- const loaded_model *m, int batch, int packed_tokens, int next_tokens, int max_batch_tokens) {
- if (batch == 0)
- return 1;
- if (model_uses_dense_batches(m))
- return next_tokens <= max_batch_tokens / (batch + 1);
- return next_tokens <= max_batch_tokens - packed_tokens;
-}
-
-int configure_loaded_model(loaded_model *m, const embed_config_t *config, int token_dim) {
- if (!m || !m->info || !config || config->hidden_size <= 0) {
- if (m && m->info)
- server_log("embed-server: model %s has invalid config", m->info->id);
- return -1;
- }
- if (m->info->kind == MODEL_KIND_LATE) {
- if (token_dim <= 0) {
- server_log("embed-server: late model %s is missing token projection metadata",
- m->info->id);
- return -1;
- }
- } else if (token_dim != 0) {
- server_log("embed-server: non-late model %s unexpectedly has token projection metadata",
- m->info->id);
- return -1;
- }
-
- int min_dim = m->info->min_dim > 0 ? m->info->min_dim : config->hidden_size;
- if (min_dim <= 0 || min_dim > config->hidden_size) {
- server_log("embed-server: model %s has invalid min_dim=%d for dim=%d", m->info->id,
- min_dim, config->hidden_size);
- return -1;
- }
- m->info->dim = config->hidden_size;
- m->info->min_dim = min_dim;
- m->info->token_dim = token_dim;
- m->info->attention_mode = config->attention_mode;
- m->info->pooling_mode = config->pooling_mode;
- m->info->normalize_embeddings = config->normalize_embeddings;
- m->append_terminal_token = config->append_terminal_token;
- /* m->terminal_token_id is resolved from the tokenizer in load_one_model. */
- m->renormalize_truncated =
- config->normalize_embeddings && config->pooling_mode == EMBED_POOL_LAST_TOKEN;
- return 0;
-}
-
-int load_one_model(http_server *s, const embed_server_model_spec_t *spec) {
- /* Grow the model list by one entry. */
- loaded_model *nm = realloc(s->models, (size_t)(s->n_models + 1) * sizeof(*s->models));
- if (!nm)
- return -1;
- s->models = nm;
- loaded_model *m = &s->models[s->n_models];
- memset(m, 0, sizeof(*m));
- s->n_models++;
-
- if (init_model_info(&m->info, spec) != 0)
- return -1;
- const char *path = spec->path;
- m->path = xstrdup(path);
- m->terminal_token_id = -1;
- m->cls_id = -1;
- m->sep_id = -1;
- /* Pick the tokenizer by the files present: WordPiece vocab.txt, then a
- * SentencePiece .model, otherwise the Qwen byte-level BPE vocab.json. */
- char wp_path[1024];
- snprintf(wp_path, sizeof(wp_path), "%s/vocab.txt", path);
- if (access(wp_path, R_OK) == 0) {
- m->wp_tok = wordpiece_tokenizer_load(path);
- if (!m->wp_tok) {
- server_log("embed-server: failed to load WordPiece tokenizer: %s", wp_path);
- return -1;
- }
- m->wp_tok_ws = wordpiece_workspace_new();
- if (!m->wp_tok_ws) {
- server_log("embed-server: failed to allocate WordPiece workspace: %s", path);
- return -1;
- }
- /* [CLS]/[SEP] wrap every input; resolve their ids once from the vocab. */
- m->cls_id = wordpiece_tokenizer_token_id(m->wp_tok, "[CLS]");
- m->sep_id = wordpiece_tokenizer_token_id(m->wp_tok, "[SEP]");
- if (m->cls_id < 0 || m->sep_id < 0) {
- server_log("embed-server: WordPiece vocab missing [CLS]/[SEP]: %s", path);
- return -1;
- }
- } else {
- char sp_path[1024];
- snprintf(sp_path, sizeof(sp_path), "%s/sentencepiece.bpe.model", path);
- if (access(sp_path, R_OK) != 0) {
- snprintf(sp_path, sizeof(sp_path), "%s/tokenizer.model", path);
- if (access(sp_path, R_OK) != 0)
- snprintf(sp_path, sizeof(sp_path), "%s/spiece.model", path);
- }
- if (access(sp_path, R_OK) == 0) {
- m->sp_tok = sentencepiece_tokenizer_load(path);
- if (!m->sp_tok) {
- server_log("embed-server: failed to load SentencePiece tokenizer: %s", sp_path);
- return -1;
- }
- m->sp_tok_ws = sentencepiece_workspace_new();
- if (!m->sp_tok_ws) {
- server_log("embed-server: failed to allocate SentencePiece workspace: %s", path);
- return -1;
- }
- m->cls_id = sentencepiece_tokenizer_token_id(m->sp_tok, "<s>");
- m->sep_id = sentencepiece_tokenizer_token_id(m->sp_tok, "</s>");
- if (m->cls_id < 0 || m->sep_id < 0) {
- server_log("embed-server: SentencePiece vocab missing <s>/</s>: %s", path);
- return -1;
- }
- }
- }
- if (!m->wp_tok && !m->sp_tok) {
- char vocab_path[1024];
- snprintf(vocab_path, sizeof(vocab_path), "%s/vocab.json", path);
- m->tok = embed_tokenizer_load(vocab_path);
- if (!m->tok) {
- server_log("embed-server: failed to load tokenizer: %s", vocab_path);
- return -1;
- }
- m->tok_ws = embed_tokenizer_workspace_new();
- if (!m->tok_ws) {
- server_log("embed-server: failed to allocate tokenizer workspace: %s", path);
- return -1;
- }
- /* The contextual chunk separator is the tokenizer's <|endoftext|>. The
- * released models keep added special tokens out of vocab.json, so the
- * family constant is the normal case; a vocab that does define the
- * token (e.g. small test models) takes precedence so the id stays
- * inside that model's embedding table. */
- int sep_id = embed_tokenizer_token_id(m->tok, "<|endoftext|>");
- m->context_separator_id = sep_id >= 0 ? sep_id : EMBED_CONTEXT_SEPARATOR_TOKEN_ID;
- /* Qwen3-Embedding pools the last token, the tokenizer's <|endoftext|>
- * suffix - the same token as the separator, not the model's chat
- * eos_token_id (<|im_end|>). Resolve it here from the tokenizer. */
- m->terminal_token_id = m->context_separator_id;
- if (m->info->kind == MODEL_KIND_LATE) {
- int id = embed_tokenizer_token_id(m->tok, "[MASK]");
- m->late_mask_id = id >= 0 ? id : EMBED_LATE_MASK_TOKEN_ID;
- id = embed_tokenizer_token_id(m->tok, "[Q]");
- m->late_query_prefix_id = id >= 0 ? id : EMBED_LATE_QUERY_PREFIX_ID;
- id = embed_tokenizer_token_id(m->tok, "[D]");
- m->late_document_prefix_id = id >= 0 ? id : EMBED_LATE_DOCUMENT_PREFIX_ID;
-
- const char *punct = "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~";
- for (const char *p = punct;
- *p &&
- m->n_late_skip_ids < (int)(sizeof(m->late_skip_ids) / sizeof(m->late_skip_ids[0]));
- p++) {
- char text[2] = {*p, '\0'};
- int n_ids = 0;
- int *ids = embed_tokenizer_encode(m->tok, text, &n_ids);
- if (ids && n_ids == 1)
- m->late_skip_ids[m->n_late_skip_ids++] = ids[0];
- free(ids);
- }
- }
- }
-#ifdef USE_MLX
- if (s->use_mlx) {
- /* MLX streams are thread-local. The inference worker loads the MLX
- * model so all MLX arrays/streams are created, used, and freed by the
- * same thread. */
- return 0;
- }
-#endif
-#ifdef USE_CUDA
- if (s->use_cuda) {
- return 0;
- }
-#endif
- const embed_config_t *config = NULL;
- int token_dim = 0;
- if (m->info->kind == MODEL_KIND_LATE) {
- m->cpu_late_model = embed_late_model_load(path);
- if (!m->cpu_late_model) {
- server_log("embed-server: failed to load late model: %s", path);
- return -1;
- }
- m->cpu_late_ws = embed_late_workspace_new(m->cpu_late_model);
- if (!m->cpu_late_ws) {
- server_log("embed-server: failed to allocate late workspace: %s", path);
- return -1;
- }
- config = embed_late_model_config(m->cpu_late_model);
- token_dim = embed_late_model_token_dim(m->cpu_late_model);
- } else {
- m->cpu_model = embed_model_load(path);
- if (!m->cpu_model) {
- server_log("embed-server: failed to load model: %s", path);
- return -1;
- }
- m->cpu_ws = embed_workspace_new(m->cpu_model);
- if (!m->cpu_ws) {
- server_log("embed-server: failed to allocate workspace: %s", path);
- return -1;
- }
- config = embed_model_config(m->cpu_model);
- }
- if (configure_loaded_model(m, config, token_dim) != 0) {
- return -1;
- }
- if (m->info->kind == MODEL_KIND_LATE &&
- (m->late_mask_id >= config->vocab_size || m->late_query_prefix_id >= config->vocab_size ||
- m->late_document_prefix_id >= config->vocab_size)) {
- server_log("embed-server: late special-token ids exceed model vocab");
- return -1;
- }
- return 0;
-}
-
-void free_models(http_server *s) {
- for (int i = 0; i < s->n_models; i++) {
- loaded_model *m = &s->models[i];
- free_model_info(m->info);
- free(m->path);
- if (m->tok_ws)
- embed_tokenizer_workspace_free(m->tok_ws);
- if (m->tok)
- embed_tokenizer_free(m->tok);
- if (m->wp_tok_ws)
- wordpiece_workspace_free(m->wp_tok_ws);
- if (m->wp_tok)
- wordpiece_tokenizer_free(m->wp_tok);
- if (m->sp_tok_ws)
- sentencepiece_workspace_free(m->sp_tok_ws);
- if (m->sp_tok)
- sentencepiece_tokenizer_free(m->sp_tok);
- if (m->cpu_ws)
- embed_workspace_free(m->cpu_ws);
- if (m->cpu_model)
- embed_model_free(m->cpu_model);
- if (m->cpu_late_ws)
- embed_late_workspace_free(m->cpu_late_ws);
- if (m->cpu_late_model)
- embed_late_model_free(m->cpu_late_model);
-#ifdef USE_MLX
- if (m->mlx_ctx)
- embed_mlx_free(m->mlx_ctx);
- if (m->mlx_late_ctx)
- embed_mlx_late_free(m->mlx_late_ctx);
-#endif
-#ifdef USE_CUDA
- if (m->cuda_ctx)
- embed_cuda_free(m->cuda_ctx);
- if (m->cuda_late_ctx)
- embed_cuda_late_free(m->cuda_late_ctx);
-#endif
- }
- free(s->models);
- s->models = NULL;
- s->n_models = 0;
-}
diff --git a/src/server_schedule.c b/src/server_schedule.c
deleted file mode 100644
index 1daf8dd..0000000
--- a/src/server_schedule.c
+++ /dev/null
@@ -1,341 +0,0 @@
-#include "server_internal.h"
-#include "server_util.h"
-
-#include <errno.h>
-#include <pthread.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <unistd.h>
-
-/* ========================================================================
- * Job queue, micro-batching, and completion
- *
- * dispatch_request (server_http.c) enqueues one job per framed request; the
- * inference worker thread (worker_main, server.c) drains them with
- * collect_job_batch (a short first-arrival wait groups concurrent arrivals) and
- * runs process_job_group, which routes each job to its endpoint handler and
- * batches compatible ones. Finished jobs go on the done queue and wake the event
- * loop through the completion pipe; completion_cb writes each response back on
- * the loop thread, so sockets are only ever touched there.
- * ======================================================================== */
-
-static void enqueue_done(job *j) {
- http_server *s = j->srv;
- pthread_mutex_lock(&s->mu);
- if (s->done_tail)
- s->done_tail->next = j;
- else
- s->done_head = j;
- s->done_tail = j;
- pthread_mutex_unlock(&s->mu);
- const unsigned char byte = 1;
- if (write(s->completion_pipe[1], &byte, 1) < 0) {
- /* best-effort wakeup; the event loop also drains the done queue */
- }
-}
-
-static job *pop_job_locked(http_server *s) {
- job *j = s->job_head;
- if (j) {
- s->job_head = j->next;
- if (!s->job_head)
- s->job_tail = NULL;
- j->next = NULL;
- }
- return j;
-}
-
-static int cond_wait_until_ns(pthread_cond_t *cv, pthread_mutex_t *mu, uint64_t deadline_ns) {
- uint64_t now_ns = nstime();
- if (now_ns >= deadline_ns)
- return ETIMEDOUT;
-
- uint64_t remaining_ns = deadline_ns - now_ns;
- struct timespec abs;
- if (clock_gettime(CLOCK_REALTIME, &abs) != 0) {
- struct timeval tv;
- gettimeofday(&tv, NULL);
- abs.tv_sec = tv.tv_sec;
- abs.tv_nsec = (long)tv.tv_usec * 1000L;
- }
- abs.tv_sec += (time_t)(remaining_ns / 1000000000u);
- abs.tv_nsec += (long)(remaining_ns % 1000000000u);
- if (abs.tv_nsec >= 1000000000L) {
- abs.tv_sec++;
- abs.tv_nsec -= 1000000000L;
- }
- return pthread_cond_timedwait(cv, mu, &abs);
-}
-
-int collect_job_batch(http_server *s, job **jobs, int max_jobs) {
- if (max_jobs <= 0)
- return 0;
-
- pthread_mutex_lock(&s->mu);
- while (!s->job_head && !s->stopping)
- pthread_cond_wait(&s->cv, &s->mu);
-
- job *first = pop_job_locked(s);
- if (!first) {
- pthread_mutex_unlock(&s->mu);
- return 0;
- }
-
- int n = 1;
- jobs[0] = first;
- int batchable = s->batch_size > 1 && (!strcmp(first->path, "/v1/embeddings") ||
- !strcmp(first->path, "/v1/contextualizedembeddings"));
- if (!batchable) {
- pthread_mutex_unlock(&s->mu);
- return n;
- }
-
- /* Exact item and token budgets are applied after worker-side tokenization;
- * this queue stage only defines which arrival window may be grouped. */
- uint64_t wait_ns = (uint64_t)s->batch_wait_us * 1000u;
- uint64_t deadline_ns =
- first->created_ns > UINT64_MAX - wait_ns ? UINT64_MAX : first->created_ns + wait_ns;
- while (n < max_jobs) {
- while (!s->job_head && !s->stopping && wait_ns > 0) {
- int rc = cond_wait_until_ns(&s->cv, &s->mu, deadline_ns);
- if (rc == ETIMEDOUT)
- break;
- }
- if (!s->job_head)
- break;
- if (wait_ns > 0 && s->job_head->created_ns > deadline_ns)
- break;
- jobs[n++] = pop_job_locked(s);
- if (wait_ns == 0 && !s->job_head)
- break;
- }
- pthread_mutex_unlock(&s->mu);
- return n;
-}
-
-void enqueue_job(job *j) {
- http_server *s = j->srv;
- pthread_mutex_lock(&s->mu);
- if (s->job_tail)
- s->job_tail->next = j;
- else
- s->job_head = j;
- s->job_tail = j;
- pthread_cond_signal(&s->cv);
- pthread_mutex_unlock(&s->mu);
-}
-
-static job *pop_done(http_server *s) {
- pthread_mutex_lock(&s->mu);
- job *j = s->done_head;
- if (j) {
- s->done_head = j->next;
- if (!s->done_head)
- s->done_tail = NULL;
- j->next = NULL;
- }
- pthread_mutex_unlock(&s->mu);
- return j;
-}
-
-static void job_free(job *j) {
- if (!j)
- return;
- free(j->body);
- free(j->auth);
- free(j->extra_headers);
- free(j->response);
- free(j);
-}
-
-void completion_cb(aeEventLoop *loop, int fd, void *clientData, int mask) {
- (void)loop;
- (void)mask;
- http_server *s = clientData;
- unsigned char tmp[128];
- while (read(fd, tmp, sizeof(tmp)) > 0) {
- }
-
- job *j;
- while ((j = pop_done(s)) != NULL) {
- client *c = j->c;
- int refs_to_drop = 1; /* job reference */
- if (!c->cancelled) {
- append_http_response_ex(c, j->status, j->content_type, j->extra_headers,
- j->response ? j->response : "",
- j->response ? j->response_len : 0);
- if (queue_write(c) != AE_OK)
- refs_to_drop += close_client_unlink(c);
- }
- client_decref_n(c, refs_to_drop);
- job_free(j);
- }
-}
-
-static void job_set_timing_header(job *j) {
- if (!j || j->extra_headers || !j->created_ns)
- return;
- uint64_t done_ns = nstime();
- uint64_t queue_ns = j->started_ns > j->created_ns ? j->started_ns - j->created_ns : 0;
- uint64_t worker_ns = j->started_ns && done_ns > j->started_ns ? done_ns - j->started_ns : 0;
-
- char buf[384];
- int n = snprintf(buf, sizeof(buf),
- "Server-Timing: queue;dur=%.3f, parse;dur=%.3f, "
- "tokenize;dur=%.3f, infer;dur=%.3f, encode;dur=%.3f, "
- "worker;dur=%.3f\r\n",
- ns_to_ms(queue_ns), ns_to_ms(j->parse_ns), ns_to_ms(j->tokenize_ns),
- ns_to_ms(j->infer_ns), ns_to_ms(j->encode_ns), ns_to_ms(worker_ns));
- if (n > 0 && (size_t)n < sizeof(buf))
- j->extra_headers = xstrdup(buf);
-}
-
-static int parse_job_root(job *j, cJSON **out_root) {
- cJSON *detail = cJSON_CreateArray();
- cJSON *root = parse_json_body(j, detail);
- if (!root) {
- job_set_422(j, detail);
- cJSON_Delete(detail);
- *out_root = NULL;
- return -1;
- }
- cJSON_Delete(detail);
- *out_root = root;
- return 0;
-}
-
-static void process_unknown_job(job *j) {
- cJSON *root = NULL;
- if (!j->started_ns)
- j->started_ns = nstime();
- uint64_t t0 = nstime();
- int rc = parse_job_root(j, &root);
- j->parse_ns += nstime() - t0;
- if (rc != 0)
- return;
- job_set_error(j, 404, "unknown endpoint", "invalid_request_error");
- cJSON_Delete(root);
-}
-
-void process_job_group(http_server *s, job **jobs, int n_jobs) {
- embedding_request *std_reqs = xcalloc((size_t)n_jobs, sizeof(*std_reqs));
- contextual_request *ctx_reqs = xcalloc((size_t)n_jobs, sizeof(*ctx_reqs));
- rerank_request *rerank_reqs = xcalloc((size_t)n_jobs, sizeof(*rerank_reqs));
- int *kind = xcalloc((size_t)n_jobs, sizeof(*kind));
- int *done = xcalloc((size_t)n_jobs, sizeof(*done));
- embedding_request **std_group = xmalloc((size_t)n_jobs * sizeof(*std_group));
- contextual_request **ctx_group = xmalloc((size_t)n_jobs * sizeof(*ctx_group));
-
- for (int i = 0; i < n_jobs; i++) {
- cJSON *root = NULL;
- if (!jobs[i]->started_ns)
- jobs[i]->started_ns = nstime();
- if (!strcmp(jobs[i]->path, "/v1/embeddings")) {
- kind[i] = 1;
- } else if (!strcmp(jobs[i]->path, "/v1/contextualizedembeddings")) {
- kind[i] = 2;
- } else if (!strcmp(jobs[i]->path, "/v1/rerank")) {
- kind[i] = 3;
- } else {
- continue;
- }
- uint64_t t0 = nstime();
- int parse_rc = parse_job_root(jobs[i], &root);
- jobs[i]->parse_ns += nstime() - t0;
- if (parse_rc != 0)
- continue;
- if (kind[i] == 1)
- prepare_embedding_request(jobs[i], root, s, &std_reqs[i]);
- else if (kind[i] == 2)
- prepare_contextual_request(jobs[i], root, s, &ctx_reqs[i]);
- else
- prepare_rerank_request(jobs[i], root, s, &rerank_reqs[i]);
- }
-
- for (int i = 0; i < n_jobs; i++) {
- if (done[i])
- continue;
-
- if (!kind[i]) {
- process_unknown_job(jobs[i]);
- job_set_timing_header(jobs[i]);
- enqueue_done(jobs[i]);
- done[i] = 1;
- continue;
- }
-
- if ((kind[i] == 1 && !std_reqs[i].ready) || (kind[i] == 2 && !ctx_reqs[i].ready) ||
- (kind[i] == 3 && !rerank_reqs[i].ready)) {
- job_set_timing_header(jobs[i]);
- enqueue_done(jobs[i]);
- done[i] = 1;
- continue;
- }
-
- if (kind[i] == 3) {
- execute_rerank_request(&rerank_reqs[i]);
- job_set_timing_header(jobs[i]);
- enqueue_done(jobs[i]);
- done[i] = 1;
- } else if (kind[i] == 1) {
- int group_n = 1;
- int group_inputs = std_reqs[i].n_inputs;
- int group_tokens = std_reqs[i].total_tokens;
- std_group[0] = &std_reqs[i];
- for (int k = i + 1; k < n_jobs; k++) {
- if (!done[k] && kind[k] == 1 && std_reqs[k].ready &&
- embedding_request_compatible(&std_reqs[i], &std_reqs[k]) &&
- embedding_request_fits_group(&std_reqs[k], group_inputs, group_tokens,
- s->batch_size, s->max_batch_tokens)) {
- std_group[group_n++] = &std_reqs[k];
- group_inputs += std_reqs[k].n_inputs;
- group_tokens += std_reqs[k].total_tokens;
- }
- }
- execute_embedding_request_list(std_group, group_n);
- for (int k = 0; k < group_n; k++) {
- int idx = (int)(std_group[k] - std_reqs);
- job_set_timing_header(jobs[idx]);
- enqueue_done(jobs[idx]);
- done[idx] = 1;
- }
- } else {
- int group_n = 1;
- int group_docs = ctx_reqs[i].n_docs;
- int group_tokens = ctx_reqs[i].total_tokens;
- ctx_group[0] = &ctx_reqs[i];
- for (int k = i + 1; k < n_jobs; k++) {
- if (!done[k] && kind[k] == 2 && ctx_reqs[k].ready &&
- contextual_request_compatible(&ctx_reqs[i], &ctx_reqs[k]) &&
- contextual_request_fits_group(&ctx_reqs[k], group_docs, group_tokens,
- s->batch_size, s->max_batch_tokens)) {
- ctx_group[group_n++] = &ctx_reqs[k];
- group_docs += ctx_reqs[k].n_docs;
- group_tokens += ctx_reqs[k].total_tokens;
- }
- }
- execute_contextual_request_list(ctx_group, group_n);
- for (int k = 0; k < group_n; k++) {
- int idx = (int)(ctx_group[k] - ctx_reqs);
- job_set_timing_header(jobs[idx]);
- enqueue_done(jobs[idx]);
- done[idx] = 1;
- }
- }
- }
-
- for (int i = 0; i < n_jobs; i++) {
- embedding_request_free(&std_reqs[i]);
- contextual_request_free(&ctx_reqs[i]);
- rerank_request_free(&rerank_reqs[i]);
- }
- free(ctx_group);
- free(std_group);
- free(done);
- free(kind);
- free(rerank_reqs);
- free(ctx_reqs);
- free(std_reqs);
-}
diff --git a/src/server_util.c b/src/server_util.c
deleted file mode 100644
index bc20804..0000000
--- a/src/server_util.c
+++ /dev/null
@@ -1,82 +0,0 @@
-#include "server_util.h"
-
-#include <stdarg.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-#include <sys/time.h>
-#include <time.h>
-
-void die_oom(void) {
- fprintf(stderr, "embed-server: out of memory\n");
- exit(1);
-}
-
-void *xmalloc(size_t n) {
- void *p = malloc(n ? n : 1);
- if (!p)
- die_oom();
- return p;
-}
-
-void *xcalloc(size_t n, size_t sz) {
- void *p = calloc(n ? n : 1, sz ? sz : 1);
- if (!p)
- die_oom();
- return p;
-}
-
-void *xrealloc(void *p, size_t n) {
- p = realloc(p, n ? n : 1);
- if (!p)
- die_oom();
- return p;
-}
-
-char *xstrdup(const char *s) {
- size_t n = strlen(s);
- char *p = xmalloc(n + 1);
- memcpy(p, s, n + 1);
- return p;
-}
-
-char *xstrndup(const char *s, size_t n) {
- char *p = xmalloc(n + 1);
- memcpy(p, s, n);
- p[n] = '\0';
- return p;
-}
-
-uint64_t mstime(void) {
- struct timespec ts;
- if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0)
- return (uint64_t)ts.tv_sec * 1000u + (uint64_t)ts.tv_nsec / 1000000u;
- struct timeval tv;
- gettimeofday(&tv, NULL);
- return (uint64_t)tv.tv_sec * 1000u + (uint64_t)tv.tv_usec / 1000u;
-}
-
-uint64_t nstime(void) {
- struct timespec ts;
- if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0)
- return (uint64_t)ts.tv_sec * 1000000000u + (uint64_t)ts.tv_nsec;
- struct timeval tv;
- gettimeofday(&tv, NULL);
- return (uint64_t)tv.tv_sec * 1000000000u + (uint64_t)tv.tv_usec * 1000u;
-}
-
-double ns_to_ms(uint64_t ns) { return (double)ns / 1000000.0; }
-
-void server_log(const char *fmt, ...) {
- char ts[32];
- time_t now = time(NULL);
- struct tm tmv;
- localtime_r(&now, &tmv);
- strftime(ts, sizeof(ts), "%m%d %H:%M:%S", &tmv);
- fprintf(stderr, "%s ", ts);
- va_list ap;
- va_start(ap, fmt);
- vfprintf(stderr, fmt, ap);
- va_end(ap);
- fputc('\n', stderr);
-}
diff --git a/src/server_util.h b/src/server_util.h
deleted file mode 100644
index 955903c..0000000
--- a/src/server_util.h
+++ /dev/null
@@ -1,36 +0,0 @@
-#ifndef EMBED_SERVER_UTIL_H
-#define EMBED_SERVER_UTIL_H
-
-#include <stdint.h>
-#include <stddef.h>
-
-/* Die-on-OOM allocation, monotonic clocks, and timestamped logging shared by
- * the embed-server translation units. The x* allocators abort the process on
- * failure rather than returning NULL, so callers never null-check them; the
- * attributes below carry that contract across translation-unit boundaries so
- * the static analyzer does not flag the (correctly) unchecked uses. */
-
-#if defined(__GNUC__) || defined(__clang__)
-#define SERVER_NORETURN __attribute__((noreturn))
-#define SERVER_RETURNS_NONNULL __attribute__((returns_nonnull))
-#else
-#define SERVER_NORETURN
-#define SERVER_RETURNS_NONNULL
-#endif
-
-SERVER_NORETURN void die_oom(void);
-
-SERVER_RETURNS_NONNULL void *xmalloc(size_t n);
-SERVER_RETURNS_NONNULL void *xcalloc(size_t n, size_t sz);
-SERVER_RETURNS_NONNULL void *xrealloc(void *p, size_t n);
-SERVER_RETURNS_NONNULL char *xstrdup(const char *s);
-SERVER_RETURNS_NONNULL char *xstrndup(const char *s, size_t n);
-
-uint64_t mstime(void);
-uint64_t nstime(void);
-double ns_to_ms(uint64_t ns);
-
-/* printf-style line logger: timestamp prefix, newline appended, to stderr. */
-void server_log(const char *fmt, ...) __attribute__((format(printf, 1, 2)));
-
-#endif /* EMBED_SERVER_UTIL_H */
diff --git a/src/threadpool.c b/src/threadpool.c
deleted file mode 100644
index 40d22c7..0000000
--- a/src/threadpool.c
+++ /dev/null
@@ -1,231 +0,0 @@
-#if defined(__linux__) && !defined(_GNU_SOURCE)
-#define _GNU_SOURCE
-#endif
-
-#include "threadpool.h"
-
-#include <pthread.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#ifdef __APPLE__
-#include <sys/sysctl.h>
-#else
-#ifdef __linux__
-#include <sched.h>
-#endif
-#include <unistd.h>
-#endif
-
-#if defined(USE_OPENBLAS)
-void openblas_set_num_threads(int num_threads);
-#endif
-
-/* The global verbose flag (defined in model.c); used only for optional pool
- * diagnostics below. Declared here to keep the thread pool independent of the
- * kernel/model headers. */
-extern int embed_verbose;
-
-#define EMBED_MAX_THREADS 16
-
-static struct {
- pthread_t threads[EMBED_MAX_THREADS - 1];
- int tids[EMBED_MAX_THREADS - 1];
- int n_threads;
- int shutdown;
-
- parallel_fn_t fn;
- void *arg;
- uint64_t generation;
-
- pthread_mutex_t mutex;
- pthread_cond_t cond_work;
- pthread_cond_t cond_done;
- int n_done;
-} tp = {
- .n_threads = 1,
- .shutdown = 0,
- .generation = 0,
- .mutex = PTHREAD_MUTEX_INITIALIZER,
- .cond_work = PTHREAD_COND_INITIALIZER,
- .cond_done = PTHREAD_COND_INITIALIZER,
-};
-
-static void *worker_loop(void *arg) {
- int tid = *(int *)arg;
- uint64_t my_gen = 0;
-
- for (;;) {
- pthread_mutex_lock(&tp.mutex);
- while (tp.generation == my_gen && !tp.shutdown)
- pthread_cond_wait(&tp.cond_work, &tp.mutex);
- if (tp.shutdown) {
- pthread_mutex_unlock(&tp.mutex);
- return NULL;
- }
- my_gen = tp.generation;
- parallel_fn_t fn = tp.fn;
- void *a = tp.arg;
- int nt = tp.n_threads;
- pthread_mutex_unlock(&tp.mutex);
-
- fn(tid, nt, a);
-
- pthread_mutex_lock(&tp.mutex);
- if (++tp.n_done >= tp.n_threads - 1)
- pthread_cond_signal(&tp.cond_done);
- pthread_mutex_unlock(&tp.mutex);
- }
-}
-
-void embed_set_threads(int n) {
- if (n < 1)
- n = 1;
- if (n > EMBED_MAX_THREADS)
- n = EMBED_MAX_THREADS;
-
-#if defined(USE_OPENBLAS)
- /* OpenBLAS owns the heavy F32 GEMM parallelism on Linux. Keep the
- * auxiliary pool serial to avoid nested thread contention. */
- int blas_threads = n;
- openblas_set_num_threads(blas_threads);
- n = 1;
-#endif
-
- /* Shutdown existing workers */
- if (tp.n_threads > 1) {
- pthread_mutex_lock(&tp.mutex);
- tp.shutdown = 1;
- pthread_cond_broadcast(&tp.cond_work);
- pthread_mutex_unlock(&tp.mutex);
- for (int i = 0; i < tp.n_threads - 1; i++)
- pthread_join(tp.threads[i], NULL);
- tp.shutdown = 0;
- tp.generation = 0;
- }
-
- tp.n_threads = n;
- if (n <= 1) {
- if (embed_verbose >= 2) {
-#if defined(USE_OPENBLAS)
- fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads);
-#else
- fprintf(stderr, "Thread pool: %d threads\n", n);
-#endif
- }
- return;
- }
-
- for (int i = 0; i < n - 1; i++) {
- tp.tids[i] = i + 1;
- pthread_create(&tp.threads[i], NULL, worker_loop, &tp.tids[i]);
- }
-
- if (embed_verbose >= 2) {
-#if defined(USE_OPENBLAS)
- fprintf(stderr, "Thread pool: %d threads, OpenBLAS: %d threads\n", n, blas_threads);
-#else
- fprintf(stderr, "Thread pool: %d threads\n", n);
-#endif
- }
-}
-
-#ifndef __APPLE__
-/* CPU bandwidth this process may actually use, in whole cores, or 0 if there
- * is no cgroup limit. Containers (Docker --cpus, Kubernetes, RunPod) expose
- * every host CPU through sysconf()/cpuset, but the CFS quota caps the CPU time
- * the process really gets. Sizing the BLAS/thread pool to the host count then
- * oversubscribes that quota: the threads spend the period's budget early and
- * the scheduler throttles them for the rest of each period, which adds large,
- * bursty stalls (e.g. a 16-thread default against a 5.1-core quota ran the
- * 0.6B model ~2x slower than a 5-thread pool). Floor, not ceil, so a fully
- * busy pool stays under the quota and never trips throttling. */
-static int cgroup_cpu_quota(void) {
- /* cgroup v2: "/sys/fs/cgroup/cpu.max" is "<quota> <period>", or
- * "max <period>" when unlimited. */
- FILE *f = fopen("/sys/fs/cgroup/cpu.max", "r");
- if (f) {
- char quota[32];
- long period = 0;
- int got = fscanf(f, "%31s %ld", quota, &period);
- fclose(f);
- if (got == 2 && period > 0 && strcmp(quota, "max") != 0) {
- long q = atol(quota);
- if (q > 0)
- return q / period < 1 ? 1 : (int)(q / period);
- }
- }
- /* cgroup v1: separate files; quota -1 means unlimited. */
- long q = -1, period = 0;
- f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_quota_us", "r");
- if (f) {
- if (fscanf(f, "%ld", &q) != 1)
- q = -1;
- fclose(f);
- }
- f = fopen("/sys/fs/cgroup/cpu/cpu.cfs_period_us", "r");
- if (f) {
- if (fscanf(f, "%ld", &period) != 1)
- period = 0;
- fclose(f);
- }
- if (q > 0 && period > 0)
- return q / period < 1 ? 1 : (int)(q / period);
- return 0;
-}
-#endif
-
-int embed_get_num_cpus(void) {
-#ifdef __APPLE__
- int n = 0;
- size_t len = sizeof(n);
- sysctlbyname("hw.ncpu", &n, &len, NULL, 0);
- return n > 0 ? n : 1;
-#else
- int online = (int)sysconf(_SC_NPROCESSORS_ONLN);
- if (online < 1)
- online = 1;
-#ifdef __linux__
- cpu_set_t set;
- if (sched_getaffinity(0, sizeof(set), &set) == 0) {
- int affinity = CPU_COUNT(&set);
- if (affinity >= 1 && affinity < online)
- online = affinity;
- }
-#endif
- /* Honor a cgroup CPU quota when it is tighter than the visible core count,
- * so the default thread pool does not oversubscribe a CPU-limited
- * container and get throttled. */
- int quota = cgroup_cpu_quota();
- if (quota >= 1 && quota < online)
- return quota;
- return online;
-#endif
-}
-
-int embed_num_threads(void) { return tp.n_threads; }
-
-/* Dispatch work to all threads; main thread is tid=0 */
-void parallel_for(parallel_fn_t fn, void *arg) {
- if (tp.n_threads <= 1) {
- fn(0, 1, arg);
- return;
- }
-
- pthread_mutex_lock(&tp.mutex);
- tp.fn = fn;
- tp.arg = arg;
- tp.n_done = 0;
- tp.generation++;
- pthread_cond_broadcast(&tp.cond_work);
- pthread_mutex_unlock(&tp.mutex);
-
- fn(0, tp.n_threads, arg);
-
- pthread_mutex_lock(&tp.mutex);
- while (tp.n_done < tp.n_threads - 1)
- pthread_cond_wait(&tp.cond_done, &tp.mutex);
- pthread_mutex_unlock(&tp.mutex);
-}
diff --git a/src/threadpool.h b/src/threadpool.h
deleted file mode 100644
index 1813bb0..0000000
--- a/src/threadpool.h
+++ /dev/null
@@ -1,36 +0,0 @@
-#ifndef EMBED_THREADPOOL_H
-#define EMBED_THREADPOOL_H
-
-/* Persistent worker pool for the CPU math kernels: a fixed set of threads that
- * sleep until parallel_for() hands them a callback, plus the CPU-count helper
- * that sizes the pool (cgroup-quota aware). Separated from the math so the
- * kernels depend only on the parallel_for / embed_num_threads interface, not on
- * the pool's internals. kernels.h includes this, so kernel callers need no
- * extra include. */
-
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-/* Worker callback: (thread index, total threads, user arg). */
-typedef void (*parallel_fn_t)(int tid, int n_threads, void *arg);
-
-/* Set the persistent pool size (default 1). Creates/rebuilds the pool; call
- * before inference. */
-EMBED_API void embed_set_threads(int n);
-
-/* Available CPU cores, capped to a cgroup CPU quota when one is tighter. */
-EMBED_API int embed_get_num_cpus(void);
-
-/* Current pool size (1 == serial). Kernels gate parallelism on this. */
-int embed_num_threads(void);
-
-/* Run fn across the pool; the calling thread participates as tid 0. Blocks
- * until every worker has finished. */
-void parallel_for(parallel_fn_t fn, void *arg);
-
-#endif /* EMBED_THREADPOOL_H */
diff --git a/src/tokenizer_bpe.c b/src/tokenizer_bpe.c
deleted file mode 100644
index 511b29e..0000000
--- a/src/tokenizer_bpe.c
+++ /dev/null
@@ -1,2018 +0,0 @@
-#include "tokenizer_bpe.h"
-#include "kernels.h"
-#include <limits.h>
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-/* ========================================================================
- * GPT-2 Bytes-to-Unicode Mapping
- * ======================================================================== */
-
-static int gpt2_byte_to_unicode[256];
-static int gpt2_unicode_to_byte[512]; /* codepoints up to ~384 */
-static int gpt2_mapping_initialized = 0;
-
-static void init_gpt2_mapping(void) {
- if (gpt2_mapping_initialized)
- return;
-
- memset(gpt2_unicode_to_byte, -1, sizeof(gpt2_unicode_to_byte));
-
- int n = 0;
- for (int b = 0; b < 256; b++) {
- int is_normal = 0;
- if (b >= 33 && b <= 126)
- is_normal = 1; /* '!'..'~' */
- if (b >= 161 && b <= 172)
- is_normal = 1; /* '¡'..'¬' */
- if (b >= 174 && b <= 255)
- is_normal = 1; /* '®'..'ÿ' */
-
- if (is_normal)
- gpt2_byte_to_unicode[b] = b;
- else
- gpt2_byte_to_unicode[b] = 256 + n++;
- }
-
- for (int b = 0; b < 256; b++) {
- int cp = gpt2_byte_to_unicode[b];
- if (cp < 512)
- gpt2_unicode_to_byte[cp] = b;
- }
-
- gpt2_mapping_initialized = 1;
-}
-
-/* Convert one Unicode codepoint to UTF-8 bytes. Returns bytes written. */
-static int utf8_encode_cp(int cp, char out[4]) {
- if (cp < 0x80) {
- out[0] = (char)cp;
- return 1;
- }
- if (cp < 0x800) {
- out[0] = (char)(0xC0 | (cp >> 6));
- out[1] = (char)(0x80 | (cp & 0x3F));
- return 2;
- }
- out[0] = (char)(0xE0 | (cp >> 12));
- out[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
- out[2] = (char)(0x80 | (cp & 0x3F));
- return 3;
-}
-
-/*
- * Decode a GPT-2 encoded token string (vocab key) to raw bytes/UTF-8 text.
- * Returns allocated string (caller must free).
- */
-static char *decode_gpt2_token(const char *token_str) {
- init_gpt2_mapping();
-
- size_t len = strlen(token_str);
- unsigned char *bytes = (unsigned char *)malloc(len + 1);
- if (!bytes)
- return NULL;
-
- int byte_count = 0;
- const unsigned char *p = (const unsigned char *)token_str;
- const unsigned char *end = p + len;
-
- while (p < end) {
- int cp = 0;
- int nbytes = 1;
-
- if ((*p & 0x80) == 0) {
- cp = *p;
- nbytes = 1;
- } else if ((*p & 0xE0) == 0xC0) {
- cp = (*p & 0x1F) << 6;
- if (p + 1 < end)
- cp |= (p[1] & 0x3F);
- nbytes = 2;
- } else if ((*p & 0xF0) == 0xE0) {
- cp = (*p & 0x0F) << 12;
- if (p + 1 < end)
- cp |= (p[1] & 0x3F) << 6;
- if (p + 2 < end)
- cp |= (p[2] & 0x3F);
- nbytes = 3;
- } else if ((*p & 0xF8) == 0xF0) {
- cp = (*p & 0x07) << 18;
- if (p + 1 < end)
- cp |= (p[1] & 0x3F) << 12;
- if (p + 2 < end)
- cp |= (p[2] & 0x3F) << 6;
- if (p + 3 < end)
- cp |= (p[3] & 0x3F);
- nbytes = 4;
- } else {
- cp = *p;
- nbytes = 1;
- }
- p += nbytes;
-
- if (cp < 512 && gpt2_unicode_to_byte[cp] >= 0) {
- bytes[byte_count++] = (unsigned char)gpt2_unicode_to_byte[cp];
- } else {
- bytes[byte_count++] = '?';
- }
- }
- bytes[byte_count] = '\0';
- return (char *)bytes;
-}
-
-/* ========================================================================
- * Simple JSON parser for vocab.json
- * ======================================================================== */
-
-static void skip_ws(const char **p) {
- while (**p == ' ' || **p == '\n' || **p == '\r' || **p == '\t')
- (*p)++;
-}
-
-static int parse_json_string(const char **p, char *out, size_t max_len) {
- skip_ws(p);
- if (**p != '"')
- return -1;
- (*p)++;
- size_t i = 0;
- while (**p && **p != '"' && i < max_len - 1) {
- if (**p == '\\') {
- (*p)++;
- if (**p == 'n')
- out[i++] = '\n';
- else if (**p == 't')
- out[i++] = '\t';
- else if (**p == '"')
- out[i++] = '"';
- else if (**p == '\\')
- out[i++] = '\\';
- else if (**p == '/')
- out[i++] = '/';
- else if (**p == 'u') {
- (*p)++;
- unsigned int cp = 0;
- for (int j = 0; j < 4 && **p; j++, (*p)++) {
- cp <<= 4;
- char c = **p;
- if (c >= '0' && c <= '9')
- cp |= (unsigned)(c - '0');
- else if (c >= 'a' && c <= 'f')
- cp |= (unsigned)(c - 'a' + 10);
- else if (c >= 'A' && c <= 'F')
- cp |= (unsigned)(c - 'A' + 10);
- }
- if (cp < 0x80 && i + 1 < max_len) {
- out[i++] = (char)cp;
- } else if (cp < 0x800 && i + 2 < max_len) {
- out[i++] = (char)(0xC0 | (cp >> 6));
- out[i++] = (char)(0x80 | (cp & 0x3F));
- } else if (i + 3 < max_len) {
- out[i++] = (char)(0xE0 | (cp >> 12));
- out[i++] = (char)(0x80 | ((cp >> 6) & 0x3F));
- out[i++] = (char)(0x80 | (cp & 0x3F));
- }
- continue;
- } else {
- out[i++] = **p;
- }
- } else {
- out[i++] = **p;
- }
- (*p)++;
- }
- out[i] = '\0';
- if (**p != '"')
- return -1;
- (*p)++;
- return 0;
-}
-
-static int64_t parse_json_int(const char **p) {
- skip_ws(p);
- int neg = 0;
- if (**p == '-') {
- neg = 1;
- (*p)++;
- }
- int64_t val = 0;
- while (**p >= '0' && **p <= '9') {
- val = val * 10 + (**p - '0');
- (*p)++;
- }
- return neg ? -val : val;
-}
-
-/* ========================================================================
- * String->int hash map (open addressing)
- * ======================================================================== */
-
-typedef struct {
- char *key; /* NULL means empty slot */
- int value;
-} str_int_entry_t;
-
-static uint64_t fnv1a_hash(const char *s) {
- uint64_t h = 1469598103934665603ULL;
- while (*s) {
- h ^= (unsigned char)*s++;
- h *= 1099511628211ULL;
- }
- return h;
-}
-
-static uint64_t fnv1a_hash_pair(const char *a, const char *b) {
- uint64_t h = 1469598103934665603ULL;
- while (*a) {
- h ^= (unsigned char)*a++;
- h *= 1099511628211ULL;
- }
- h ^= (unsigned char)' ';
- h *= 1099511628211ULL;
- while (*b) {
- h ^= (unsigned char)*b++;
- h *= 1099511628211ULL;
- }
- return h;
-}
-
-static int pair_key_eq(const char *key, const char *a, const char *b) {
- while (*a) {
- if (*key++ != *a++)
- return 0;
- }
- if (*key++ != ' ')
- return 0;
- while (*b) {
- if (*key++ != *b++)
- return 0;
- }
- return *key == '\0';
-}
-
-static int next_pow2(int x) {
- int p = 1;
- while (p < x)
- p <<= 1;
- return p;
-}
-
-static int map_insert(str_int_entry_t *map, int cap, char *key, int value) {
- if (!map || cap <= 0 || !key)
- return -1;
- int mask = cap - 1;
- int pos = (int)(fnv1a_hash(key) & (uint64_t)mask);
- for (int i = 0; i < cap; i++) {
- int idx = (pos + i) & mask;
- if (!map[idx].key) {
- map[idx].key = key;
- map[idx].value = value;
- return 0;
- }
- if (strcmp(map[idx].key, key) == 0) {
- map[idx].value = value;
- return 0;
- }
- }
- return -1;
-}
-
-static int map_insert_owned(str_int_entry_t *map, int cap, char **keyp, int value) {
- if (!keyp)
- return -1;
- char *key = *keyp;
- if (!map || cap <= 0 || !key)
- return -1;
- int mask = cap - 1;
- int pos = (int)(fnv1a_hash(key) & (uint64_t)mask);
- for (int i = 0; i < cap; i++) {
- int idx = (pos + i) & mask;
- if (!map[idx].key) {
- map[idx].key = key;
- map[idx].value = value;
- *keyp = NULL;
- return 0;
- }
- if (strcmp(map[idx].key, key) == 0) {
- map[idx].value = value;
- return 0;
- }
- }
- return -1;
-}
-
-static int map_get(const str_int_entry_t *map, int cap, const char *key) {
- if (!map || cap <= 0 || !key)
- return -1;
- int mask = cap - 1;
- int pos = (int)(fnv1a_hash(key) & (uint64_t)mask);
- for (int i = 0; i < cap; i++) {
- int idx = (pos + i) & mask;
- if (!map[idx].key)
- return -1;
- if (strcmp(map[idx].key, key) == 0)
- return map[idx].value;
- }
- return -1;
-}
-
-static int map_get_pair(const str_int_entry_t *map, int cap, const char *a, const char *b) {
- if (!map || cap <= 0 || !a || !b)
- return -1;
- int mask = cap - 1;
- int pos = (int)(fnv1a_hash_pair(a, b) & (uint64_t)mask);
- for (int i = 0; i < cap; i++) {
- int idx = (pos + i) & mask;
- if (!map[idx].key)
- return -1;
- if (pair_key_eq(map[idx].key, a, b))
- return map[idx].value;
- }
- return -1;
-}
-
-/* ========================================================================
- * BPE helpers
- * ======================================================================== */
-
-static int utf8_char_len(unsigned char c) {
- if ((c & 0x80) == 0)
- return 1;
- if ((c & 0xE0) == 0xC0)
- return 2;
- if ((c & 0xF0) == 0xE0)
- return 3;
- if ((c & 0xF8) == 0xF0)
- return 4;
- return 1;
-}
-
-static int split_utf8_symbols(const char *s, char ***out_syms, int *out_n) {
- *out_syms = NULL;
- *out_n = 0;
- if (!s || !*s)
- return 0;
-
- int n = 0;
- for (const unsigned char *p = (const unsigned char *)s; *p;) {
- int l = utf8_char_len(*p);
- p += l;
- n++;
- }
- if (n <= 0)
- return 0;
-
- char **syms = (char **)calloc((size_t)n, sizeof(char *));
- if (!syms)
- return -1;
-
- int i = 0;
- for (const unsigned char *p = (const unsigned char *)s; *p;) {
- int l = utf8_char_len(*p);
- syms[i] = (char *)malloc((size_t)l + 1);
- if (!syms[i]) {
- for (int j = 0; j < i; j++)
- free(syms[j]);
- free(syms);
- return -1;
- }
- memcpy(syms[i], p, (size_t)l);
- syms[i][l] = '\0';
- p += l;
- i++;
- }
-
- *out_syms = syms;
- *out_n = n;
- return 0;
-}
-
-static char *str_concat2(const char *a, const char *b) {
- size_t la = strlen(a), lb = strlen(b);
- char *r = (char *)malloc(la + lb + 1);
- if (!r)
- return NULL;
- memcpy(r, a, la);
- memcpy(r + la, b, lb);
- r[la + lb] = '\0';
- return r;
-}
-
-/* ========================================================================
- * Integer-pair BPE merges: (left_id, right_id) -> (rank, merged_id)
- * ======================================================================== */
-
-typedef struct {
- uint64_t key; /* (left_id << 32) | right_id; UINT64_MAX = empty */
- int rank;
- int merged_id;
-} pair_merge_entry_t;
-
-static uint64_t pair_hash(uint64_t k) {
- /* splitmix64 finalizer */
- k ^= k >> 30;
- k *= 0xbf58476d1ce4e5b9ULL;
- k ^= k >> 27;
- k *= 0x94d049bb133111ebULL;
- k ^= k >> 31;
- return k;
-}
-
-static void
-pair_map_insert(pair_merge_entry_t *map, int cap, uint64_t key, int rank, int merged_id) {
- int mask = cap - 1;
- int idx = (int)(pair_hash(key) & (uint64_t)mask);
- for (int probe = 0; probe < cap; probe++, idx = (idx + 1) & mask) {
- if (map[idx].key == UINT64_MAX) {
- map[idx].key = key;
- map[idx].rank = rank;
- map[idx].merged_id = merged_id;
- return;
- }
- if (map[idx].key == key)
- return; /* first (lowest) rank wins */
- }
-}
-
-/* Returns the rank and sets *merged_id, or INT_MAX when the pair never
- * merges. */
-static int pair_merge_lookup(const embed_tokenizer_t *tok, int left, int right, int *merged_id) {
- const pair_merge_entry_t *map = (const pair_merge_entry_t *)tok->int_merges;
- uint64_t key = ((uint64_t)(uint32_t)left << 32) | (uint32_t)right;
- int mask = tok->int_merges_cap - 1;
- int idx = (int)(pair_hash(key) & (uint64_t)mask);
- for (int probe = 0; probe < tok->int_merges_cap; probe++, idx = (idx + 1) & mask) {
- if (map[idx].key == UINT64_MAX)
- return INT_MAX;
- if (map[idx].key == key) {
- *merged_id = map[idx].merged_id;
- return map[idx].rank;
- }
- }
- return INT_MAX;
-}
-
-static int merge_rank(const embed_tokenizer_t *tok, const char *a, const char *b) {
- if (!tok->merge_map || tok->merge_map_cap <= 0)
- return INT_MAX;
-
- int rank = map_get_pair((const str_int_entry_t *)tok->merge_map, tok->merge_map_cap, a, b);
- return rank >= 0 ? rank : INT_MAX;
-}
-
-static int append_id(int **arr, int *n, int *cap, int id) {
- if (*n >= *cap) {
- int new_cap = (*cap == 0) ? 16 : (*cap * 2);
- int *tmp = (int *)realloc(*arr, (size_t)new_cap * sizeof(int));
- if (!tmp)
- return -1;
- *arr = tmp;
- *cap = new_cap;
- }
- (*arr)[(*n)++] = id;
- return 0;
-}
-
-/* Convert UTF-8 bytes to GPT-2 byte-level unicode string. */
-static char *text_to_bpe_unicode(const char *text) {
- init_gpt2_mapping();
- size_t len = strlen(text);
- /* Every input byte becomes one codepoint up to 3 UTF-8 bytes. */
- char *out = (char *)malloc(len * 3 + 1);
- if (!out)
- return NULL;
-
- size_t w = 0;
- for (size_t i = 0; i < len; i++) {
- int cp = gpt2_byte_to_unicode[(unsigned char)text[i]];
- char tmp[4];
- int n = utf8_encode_cp(cp, tmp);
- for (int j = 0; j < n; j++)
- out[w++] = tmp[j];
- }
- out[w] = '\0';
- return out;
-}
-
-static int
-encode_bpe_word(const embed_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n);
-
-static int
-utf8_decode_cp_len(const unsigned char *p, const unsigned char *end, int *cp_out, int *len_out) {
- if (p >= end || *p == '\0')
- return 0;
-
- unsigned char c = *p;
- if ((c & 0x80) == 0) {
- *cp_out = c;
- *len_out = 1;
- return 1;
- }
- if ((c & 0xE0) == 0xC0 && p + 1 < end && (p[1] & 0xC0) == 0x80) {
- *cp_out = ((c & 0x1F) << 6) | (p[1] & 0x3F);
- *len_out = 2;
- return 1;
- }
- if ((c & 0xF0) == 0xE0 && p + 2 < end && (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80) {
- *cp_out = ((c & 0x0F) << 12) | ((p[1] & 0x3F) << 6) | (p[2] & 0x3F);
- *len_out = 3;
- return 1;
- }
- if ((c & 0xF8) == 0xF0 && p + 3 < end && (p[1] & 0xC0) == 0x80 && (p[2] & 0xC0) == 0x80 &&
- (p[3] & 0xC0) == 0x80) {
- *cp_out = ((c & 0x07) << 18) | ((p[1] & 0x3F) << 12) | ((p[2] & 0x3F) << 6) | (p[3] & 0x3F);
- *len_out = 4;
- return 1;
- }
-
- *cp_out = c;
- *len_out = 1;
- return 1;
-}
-
-/* The pretokenizer classifies every codepoint, several times each, and the
- * input is overwhelmingly ASCII; one table lookup replaces the range chains
- * for cp < 0x80. Non-ASCII codepoints fall through to the full checks.
- * ASCII has no symbolish codepoints, so that class needs no bit. */
-#define CPC_SPACE 1
-#define CPC_NUMBER 2
-#define CPC_LETTER 4
-
-static const unsigned char ascii_cp_class[128] = {
- ['\t'] = CPC_SPACE, ['\n'] = CPC_SPACE, ['\v'] = CPC_SPACE,
- ['\f'] = CPC_SPACE, ['\r'] = CPC_SPACE, [' '] = CPC_SPACE,
- ['0' ... '9'] = CPC_NUMBER, ['A' ... 'Z'] = CPC_LETTER, ['a' ... 'z'] = CPC_LETTER,
-};
-
-static int is_number_cp(int cp) {
- if ((unsigned)cp < 0x80)
- return ascii_cp_class[cp] & CPC_NUMBER;
- if (cp >= 0x0660 && cp <= 0x0669)
- return 1;
- if (cp >= 0x06F0 && cp <= 0x06F9)
- return 1;
- if (cp >= 0x0966 && cp <= 0x096F)
- return 1;
- if (cp >= 0xFF10 && cp <= 0xFF19)
- return 1;
- return 0;
-}
-
-static int is_space_cp(int cp) {
- if ((unsigned)cp < 0x80)
- return ascii_cp_class[cp] & CPC_SPACE;
- return cp == 0x85 || cp == 0xA0 || cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) ||
- cp == 0x2028 || cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000;
-}
-
-static int is_symbolish_cp(int cp) {
- if (cp < 0x80)
- return 0;
- if (cp >= 0x0300 && cp <= 0x036F)
- return 1;
- if (cp >= 0x00A1 && cp <= 0x00BF && cp != 0x00AA && cp != 0x00B5 && cp != 0x00BA)
- return 1;
- if (cp == 0x00D7 || cp == 0x00F7)
- return 1;
- if (cp >= 0x064B && cp <= 0x065F)
- return 1;
- if ((cp >= 0x0900 && cp <= 0x0903) || (cp >= 0x093A && cp <= 0x094F) ||
- (cp >= 0x0951 && cp <= 0x0957) || (cp >= 0x0962 && cp <= 0x0963))
- return 1;
- if (cp >= 0x2000 && cp <= 0x206F)
- return 1;
- if (cp >= 0x2190 && cp <= 0x27BF)
- return 1;
- if (cp >= 0x2E00 && cp <= 0x2E7F)
- return 1;
- if (cp >= 0x3000 && cp <= 0x303F)
- return 1;
- if (cp >= 0x1F000 && cp <= 0x1FAFF)
- return 1;
- return 0;
-}
-
-static int compose_latin_mark(int base, int mark) {
- switch (mark) {
- case 0x0300:
- switch (base) {
- case 'A':
- return 0x00C0;
- case 'E':
- return 0x00C8;
- case 'I':
- return 0x00CC;
- case 'O':
- return 0x00D2;
- case 'U':
- return 0x00D9;
- case 'a':
- return 0x00E0;
- case 'e':
- return 0x00E8;
- case 'i':
- return 0x00EC;
- case 'o':
- return 0x00F2;
- case 'u':
- return 0x00F9;
- }
- break;
- case 0x0301:
- switch (base) {
- case 'A':
- return 0x00C1;
- case 'C':
- return 0x0106;
- case 'E':
- return 0x00C9;
- case 'I':
- return 0x00CD;
- case 'L':
- return 0x0139;
- case 'N':
- return 0x0143;
- case 'O':
- return 0x00D3;
- case 'R':
- return 0x0154;
- case 'S':
- return 0x015A;
- case 'U':
- return 0x00DA;
- case 'Y':
- return 0x00DD;
- case 'Z':
- return 0x0179;
- case 'a':
- return 0x00E1;
- case 'c':
- return 0x0107;
- case 'e':
- return 0x00E9;
- case 'i':
- return 0x00ED;
- case 'l':
- return 0x013A;
- case 'n':
- return 0x0144;
- case 'o':
- return 0x00F3;
- case 'r':
- return 0x0155;
- case 's':
- return 0x015B;
- case 'u':
- return 0x00FA;
- case 'y':
- return 0x00FD;
- case 'z':
- return 0x017A;
- }
- break;
- case 0x0302:
- switch (base) {
- case 'A':
- return 0x00C2;
- case 'E':
- return 0x00CA;
- case 'I':
- return 0x00CE;
- case 'O':
- return 0x00D4;
- case 'U':
- return 0x00DB;
- case 'a':
- return 0x00E2;
- case 'e':
- return 0x00EA;
- case 'i':
- return 0x00EE;
- case 'o':
- return 0x00F4;
- case 'u':
- return 0x00FB;
- }
- break;
- case 0x0303:
- switch (base) {
- case 'A':
- return 0x00C3;
- case 'N':
- return 0x00D1;
- case 'O':
- return 0x00D5;
- case 'a':
- return 0x00E3;
- case 'n':
- return 0x00F1;
- case 'o':
- return 0x00F5;
- }
- break;
- case 0x0308:
- switch (base) {
- case 'A':
- return 0x00C4;
- case 'E':
- return 0x00CB;
- case 'I':
- return 0x00CF;
- case 'O':
- return 0x00D6;
- case 'U':
- return 0x00DC;
- case 'Y':
- return 0x0178;
- case 'a':
- return 0x00E4;
- case 'e':
- return 0x00EB;
- case 'i':
- return 0x00EF;
- case 'o':
- return 0x00F6;
- case 'u':
- return 0x00FC;
- case 'y':
- return 0x00FF;
- }
- break;
- case 0x030A:
- switch (base) {
- case 'A':
- return 0x00C5;
- case 'a':
- return 0x00E5;
- }
- break;
- case 0x0327:
- switch (base) {
- case 'C':
- return 0x00C7;
- case 'c':
- return 0x00E7;
- }
- break;
- }
- return 0;
-}
-
-static char *normalize_nfc_latin(const char *text) {
- const unsigned char *p = (const unsigned char *)text;
- const unsigned char *end = p + strlen(text);
- char *out = (char *)malloc((size_t)(end - p) * 2 + 1);
- if (!out)
- return NULL;
-
- size_t w = 0;
- while (p < end) {
- const unsigned char *cur = p;
- int cp = 0, len = 0;
- if (!utf8_decode_cp_len(p, end, &cp, &len))
- break;
- p += len;
-
- if (p < end) {
- int mark = 0, mark_len = 0;
- utf8_decode_cp_len(p, end, &mark, &mark_len);
- int composed = compose_latin_mark(cp, mark);
- if (composed) {
- char tmp[4];
- int n = utf8_encode_cp(composed, tmp);
- for (int i = 0; i < n; i++)
- out[w++] = tmp[i];
- p += mark_len;
- continue;
- }
- }
-
- memcpy(out + w, cur, (size_t)len);
- w += (size_t)len;
- }
-
- out[w] = '\0';
- return out;
-}
-
-static int is_letter_cp(int cp) {
- if ((unsigned)cp < 0x80)
- return ascii_cp_class[cp] & CPC_LETTER;
- if (is_space_cp(cp) || is_number_cp(cp) || is_symbolish_cp(cp))
- return 0;
- return 1;
-}
-
-static int is_word_cp(int cp) { return is_letter_cp(cp) || is_number_cp(cp); }
-
-static int is_newline_cp(int cp) { return cp == '\n' || cp == '\r'; }
-
-static int ascii_lower(int c) { return (c >= 'A' && c <= 'Z') ? c + ('a' - 'A') : c; }
-
-static int match_contraction(const unsigned char *p, const unsigned char *end) {
- if (p >= end || *p != '\'')
- return 0;
- int remain = (int)(end - p);
- if (remain >= 2) {
- int c1 = ascii_lower(p[1]);
- if (c1 == 's' || c1 == 't' || c1 == 'm' || c1 == 'd')
- return 2;
- }
- if (remain >= 3) {
- int c1 = ascii_lower(p[1]);
- int c2 = ascii_lower(p[2]);
- if ((c1 == 'r' && c2 == 'e') || (c1 == 'v' && c2 == 'e') || (c1 == 'l' && c2 == 'l'))
- return 3;
- }
- return 0;
-}
-
-static int append_encoded_piece(const embed_tokenizer_t *tok,
- const char *piece,
- size_t piece_len,
- int **ids,
- int *n_ids,
- int *cap) {
- if (piece_len == 0)
- return 0;
-
- char *tmp = (char *)malloc(piece_len + 1);
- if (!tmp)
- return -1;
- memcpy(tmp, piece, piece_len);
- tmp[piece_len] = '\0';
-
- char *mapped = text_to_bpe_unicode(tmp);
- free(tmp);
- if (!mapped)
- return -1;
-
- int *piece_ids = NULL;
- int n_piece = 0;
- if (encode_bpe_word(tok, mapped, &piece_ids, &n_piece) != 0) {
- free(mapped);
- free(piece_ids);
- return -1;
- }
- free(mapped);
-
- for (int i = 0; i < n_piece; i++) {
- if (append_id(ids, n_ids, cap, piece_ids[i]) != 0) {
- free(piece_ids);
- return -1;
- }
- }
- free(piece_ids);
- return 0;
-}
-
-static int
-encode_pretokenized(const embed_tokenizer_t *tok, const char *text, int **out_ids, int *out_n) {
- const unsigned char *start = (const unsigned char *)text;
- const unsigned char *p = start;
- const unsigned char *end = start + strlen(text);
- int *ids = NULL;
- int n_ids = 0, cap = 0;
-
- while (p < end) {
- const unsigned char *piece = p;
- int cp = 0, len = 0;
- if (!utf8_decode_cp_len(p, end, &cp, &len))
- break;
-
- int clen = match_contraction(p, end);
- if (clen > 0) {
- p += clen;
- } else {
- const unsigned char *next = p + len;
- int next_cp = 0, next_len = 0;
- int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len);
-
- if (!is_newline_cp(cp) && !is_word_cp(cp) && has_next && is_letter_cp(next_cp)) {
- p = next + next_len;
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (!is_letter_cp(c))
- break;
- p += l;
- }
- } else if (is_letter_cp(cp)) {
- p += len;
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (!is_letter_cp(c))
- break;
- p += l;
- }
- } else if (is_number_cp(cp)) {
- p += len;
- } else if (is_space_cp(cp)) {
- const unsigned char *q = p;
- int saw_newline = 0;
- while (q < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(q, end, &c, &l);
- if (!is_space_cp(c))
- break;
- if (is_newline_cp(c))
- saw_newline = 1;
- q += l;
- if (saw_newline) {
- while (q < end) {
- int c2 = 0, l2 = 0;
- utf8_decode_cp_len(q, end, &c2, &l2);
- if (!is_newline_cp(c2))
- break;
- q += l2;
- }
- break;
- }
- }
-
- if (saw_newline) {
- p = q;
- } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && !is_word_cp(next_cp)) {
- p = next + next_len;
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (is_space_cp(c) || is_word_cp(c))
- break;
- p += l;
- }
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (!is_newline_cp(c))
- break;
- p += l;
- }
- } else {
- const unsigned char *last = p;
- const unsigned char *q2 = p;
- int count = 0;
- while (q2 < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(q2, end, &c, &l);
- if (!is_space_cp(c))
- break;
- last = q2;
- q2 += l;
- count++;
- }
- if (q2 < end && count > 1)
- p = last;
- else
- p = q2;
- }
- } else {
- p += len;
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (is_space_cp(c) || is_word_cp(c))
- break;
- p += l;
- }
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (!is_newline_cp(c))
- break;
- p += l;
- }
- }
- }
-
- if (append_encoded_piece(tok, (const char *)piece, (size_t)(p - piece), &ids, &n_ids,
- &cap) != 0) {
- free(ids);
- return -1;
- }
- }
-
- *out_ids = ids;
- *out_n = n_ids;
- return 0;
-}
-
-/* Encode one mapped BPE unicode string to token IDs. */
-static int
-encode_bpe_word(const embed_tokenizer_t *tok, const char *mapped, int **out_ids, int *out_n) {
- *out_ids = NULL;
- *out_n = 0;
- if (!mapped || !*mapped)
- return 0;
-
- char **syms = NULL;
- int n_syms = 0;
- if (split_utf8_symbols(mapped, &syms, &n_syms) != 0)
- return -1;
- if (n_syms <= 0) {
- free(syms);
- return 0;
- }
-
- while (n_syms > 1) {
- int best_rank = INT_MAX;
- int best_i = -1;
- for (int i = 0; i < n_syms - 1; i++) {
- int r = merge_rank(tok, syms[i], syms[i + 1]);
- if (r < best_rank) {
- best_rank = r;
- best_i = i;
- }
- }
- if (best_i < 0 || best_rank == INT_MAX)
- break;
-
- char *merged = str_concat2(syms[best_i], syms[best_i + 1]);
- if (!merged) {
- for (int i = 0; i < n_syms; i++)
- free(syms[i]);
- free(syms);
- return -1;
- }
- free(syms[best_i]);
- free(syms[best_i + 1]);
- syms[best_i] = merged;
- for (int j = best_i + 1; j < n_syms - 1; j++)
- syms[j] = syms[j + 1];
- n_syms--;
- }
-
- int *ids = NULL;
- int n_ids = 0, cap = 0;
- for (int i = 0; i < n_syms; i++) {
- int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, syms[i]);
- if (id < 0) {
- /* Should not happen with valid vocab + merges + byte-level mapping. */
- for (int k = 0; k < n_syms; k++)
- free(syms[k]);
- free(syms);
- free(ids);
- return -1;
- }
- if (append_id(&ids, &n_ids, &cap, id) != 0) {
- for (int k = 0; k < n_syms; k++)
- free(syms[k]);
- free(syms);
- free(ids);
- return -1;
- }
- }
-
- for (int i = 0; i < n_syms; i++)
- free(syms[i]);
- free(syms);
-
- *out_ids = ids;
- *out_n = n_ids;
- return 0;
-}
-
-typedef struct {
- size_t off;
-} bpe_sym_ref_t;
-
-struct embed_tokenizer_workspace {
- char *normalized;
- size_t normalized_cap;
- char *mapped;
- size_t mapped_cap;
- char *arena;
- size_t arena_cap;
- size_t arena_len;
- bpe_sym_ref_t *syms;
- int syms_cap;
- int *ids;
- int ids_cap;
- /* Integer-pair BPE merge scratch: token id, doubly-linked live order,
- * and the cached (rank, merged id) of the pair starting at each live
- * position. */
- int *bpe_ids;
- int *bpe_prv;
- int *bpe_nxt;
- int *bpe_rnk;
- int *bpe_mrg;
- int bpe_cap;
-};
-
-static int reserve_bpe(embed_tokenizer_workspace_t *ws, int need) {
- if (ws->bpe_cap >= need)
- return 0;
- int new_cap = ws->bpe_cap ? ws->bpe_cap : 64;
- while (new_cap < need) {
- if (new_cap > INT_MAX / 2)
- return -1;
- new_cap *= 2;
- }
- int *p;
-#define R(field) \
- p = (int *)realloc(ws->field, (size_t)new_cap * sizeof(int)); \
- if (!p) \
- return -1; \
- ws->field = p;
- R(bpe_ids)
- R(bpe_prv)
- R(bpe_nxt)
- R(bpe_rnk)
- R(bpe_mrg)
-#undef R
- ws->bpe_cap = new_cap;
- return 0;
-}
-
-static int reserve_bytes(char **buf, size_t *cap, size_t need) {
- if (*cap >= need)
- return 0;
- size_t new_cap = *cap ? *cap : 256;
- while (new_cap < need) {
- if (new_cap > SIZE_MAX / 2)
- return -1;
- new_cap *= 2;
- }
- char *p = (char *)realloc(*buf, new_cap);
- if (!p)
- return -1;
- *buf = p;
- *cap = new_cap;
- return 0;
-}
-
-static int reserve_ints(int **buf, int *cap, int need) {
- if (*cap >= need)
- return 0;
- int new_cap = *cap ? *cap : 64;
- while (new_cap < need) {
- if (new_cap > INT_MAX / 2)
- return -1;
- new_cap *= 2;
- }
- int *p = (int *)realloc(*buf, (size_t)new_cap * sizeof(int));
- if (!p)
- return -1;
- *buf = p;
- *cap = new_cap;
- return 0;
-}
-
-static int reserve_syms(embed_tokenizer_workspace_t *ws, int need) {
- if (ws->syms_cap >= need)
- return 0;
- int new_cap = ws->syms_cap ? ws->syms_cap : 64;
- while (new_cap < need) {
- if (new_cap > INT_MAX / 2)
- return -1;
- new_cap *= 2;
- }
- bpe_sym_ref_t *p = (bpe_sym_ref_t *)realloc(ws->syms, (size_t)new_cap * sizeof(*ws->syms));
- if (!p)
- return -1;
- ws->syms = p;
- ws->syms_cap = new_cap;
- return 0;
-}
-
-static int arena_append(embed_tokenizer_workspace_t *ws, const char *s, size_t len, size_t *off) {
- if (ws->arena_len > SIZE_MAX - len - 1)
- return -1;
- size_t need = ws->arena_len + len + 1;
- if (reserve_bytes(&ws->arena, &ws->arena_cap, need) != 0)
- return -1;
- *off = ws->arena_len;
- memcpy(ws->arena + ws->arena_len, s, len);
- ws->arena_len += len;
- ws->arena[ws->arena_len++] = '\0';
- return 0;
-}
-
-static const char *arena_str(const embed_tokenizer_workspace_t *ws, bpe_sym_ref_t sym) {
- return ws->arena + sym.off;
-}
-
-static int
-normalize_nfc_latin_into(embed_tokenizer_workspace_t *ws, const char *text, const char **out) {
- const unsigned char *p = (const unsigned char *)text;
- const unsigned char *end = p + strlen(text);
- size_t in_len = (size_t)(end - p);
- if (in_len > (SIZE_MAX - 1) / 2)
- return -1;
- if (reserve_bytes(&ws->normalized, &ws->normalized_cap, in_len * 2 + 1) != 0)
- return -1;
-
- size_t w = 0;
- while (p < end) {
- const unsigned char *cur = p;
- int cp = 0, len = 0;
- if (!utf8_decode_cp_len(p, end, &cp, &len))
- break;
- p += len;
-
- if (p < end) {
- int mark = 0, mark_len = 0;
- utf8_decode_cp_len(p, end, &mark, &mark_len);
- int composed = compose_latin_mark(cp, mark);
- if (composed) {
- char tmp[4];
- int n = utf8_encode_cp(composed, tmp);
- memcpy(ws->normalized + w, tmp, (size_t)n);
- w += (size_t)n;
- p += mark_len;
- continue;
- }
- }
-
- memcpy(ws->normalized + w, cur, (size_t)len);
- w += (size_t)len;
- }
-
- ws->normalized[w] = '\0';
- *out = ws->normalized;
- return 0;
-}
-
-static int text_to_bpe_unicode_into(embed_tokenizer_workspace_t *ws,
- const char *text,
- size_t len,
- const char **out) {
- init_gpt2_mapping();
- if (len > (SIZE_MAX - 1) / 3)
- return -1;
- if (reserve_bytes(&ws->mapped, &ws->mapped_cap, len * 3 + 1) != 0)
- return -1;
-
- size_t w = 0;
- for (size_t i = 0; i < len; i++) {
- int cp = gpt2_byte_to_unicode[(unsigned char)text[i]];
- char tmp[4];
- int n = utf8_encode_cp(cp, tmp);
- memcpy(ws->mapped + w, tmp, (size_t)n);
- w += (size_t)n;
- }
- ws->mapped[w] = '\0';
- *out = ws->mapped;
- return 0;
-}
-
-static int output_append_id(int *out_ids, int out_cap, int *n_ids, int *overflow, int id) {
- if (*n_ids < out_cap && out_ids)
- out_ids[*n_ids] = id;
- else
- *overflow = 1;
- (*n_ids)++;
- return 0;
-}
-
-static int encode_bpe_word_into_slow(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *mapped,
- int *out_ids,
- int out_cap,
- int *n_ids,
- int *overflow) {
- if (!mapped || !*mapped)
- return 0;
-
- int n_syms = 0;
- for (const unsigned char *p = (const unsigned char *)mapped; *p;) {
- int l = utf8_char_len(*p);
- p += l;
- n_syms++;
- }
- if (n_syms <= 0)
- return 0;
- if (reserve_syms(ws, n_syms) != 0)
- return -1;
-
- ws->arena_len = 0;
- int i = 0;
- for (const unsigned char *p = (const unsigned char *)mapped; *p;) {
- int l = utf8_char_len(*p);
- size_t off = 0;
- if (arena_append(ws, (const char *)p, (size_t)l, &off) != 0)
- return -1;
- ws->syms[i++].off = off;
- p += l;
- }
-
- while (n_syms > 1) {
- int best_rank = INT_MAX;
- int best_i = -1;
- for (int j = 0; j < n_syms - 1; j++) {
- int r = merge_rank(tok, arena_str(ws, ws->syms[j]), arena_str(ws, ws->syms[j + 1]));
- if (r < best_rank) {
- best_rank = r;
- best_i = j;
- }
- }
- if (best_i < 0 || best_rank == INT_MAX)
- break;
-
- const char *a = arena_str(ws, ws->syms[best_i]);
- const char *b = arena_str(ws, ws->syms[best_i + 1]);
- size_t la = strlen(a), lb = strlen(b);
- if (la > SIZE_MAX - lb)
- return -1;
- if (reserve_bytes(&ws->arena, &ws->arena_cap, ws->arena_len + la + lb + 1) != 0)
- return -1;
-
- a = arena_str(ws, ws->syms[best_i]);
- b = arena_str(ws, ws->syms[best_i + 1]);
- size_t off = ws->arena_len;
- memcpy(ws->arena + ws->arena_len, a, la);
- ws->arena_len += la;
- memcpy(ws->arena + ws->arena_len, b, lb);
- ws->arena_len += lb;
- ws->arena[ws->arena_len++] = '\0';
- ws->syms[best_i].off = off;
- for (int j = best_i + 1; j < n_syms - 1; j++)
- ws->syms[j] = ws->syms[j + 1];
- n_syms--;
- }
-
- for (int j = 0; j < n_syms; j++) {
- int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap,
- arena_str(ws, ws->syms[j]));
- if (id < 0)
- return -1;
- output_append_id(out_ids, out_cap, n_ids, overflow, id);
- }
- return 0;
-}
-
-/* Integer-pair BPE merge: token ids in a doubly-linked array with cached
- * pair ranks; each merge updates only the two adjacent pairs. Falls back to
- * the string-based loop when the int tables are unavailable. Identical
- * output: initial ids are the byte-level single-char token ids and every
- * merge result is the vocab id of the concatenated pair. */
-static int encode_bpe_word_into(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *mapped,
- int *out_ids,
- int out_cap,
- int *n_ids,
- int *overflow) {
- if (!mapped || !*mapped)
- return 0;
- if (!tok->int_merges || !tok->cp_to_id)
- return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, n_ids, overflow);
-
- int n = 0;
- for (const unsigned char *p = (const unsigned char *)mapped; *p;) {
- p += utf8_char_len(*p);
- n++;
- }
- if (n <= 0)
- return 0;
- if (reserve_bpe(ws, n) != 0)
- return -1;
-
- int *ids = ws->bpe_ids, *prv = ws->bpe_prv, *nxt = ws->bpe_nxt;
- int *rnk = ws->bpe_rnk, *mrg = ws->bpe_mrg;
-
- int i = 0;
- for (const unsigned char *p = (const unsigned char *)mapped; *p;) {
- int len = utf8_char_len(*p);
- int cp;
- if (len == 1)
- cp = *p;
- else if (len == 2)
- cp = ((*p & 0x1F) << 6) | (p[1] & 0x3F);
- else
- cp = 512; /* byte-level alphabet never exceeds two bytes */
- int id = cp < 512 ? tok->cp_to_id[cp] : -1;
- if (id < 0)
- return encode_bpe_word_into_slow(tok, ws, mapped, out_ids, out_cap, n_ids, overflow);
- ids[i] = id;
- prv[i] = i - 1;
- nxt[i] = i + 1 < n ? i + 1 : -1;
- i++;
- p += len;
- }
-
- for (int j = 0; j < n - 1; j++)
- rnk[j] = pair_merge_lookup(tok, ids[j], ids[j + 1], &mrg[j]);
- if (n > 0)
- rnk[n - 1] = INT_MAX;
-
- for (;;) {
- int best = -1, best_rank = INT_MAX;
- for (int j = 0; j != -1 && nxt[j] != -1; j = nxt[j]) {
- if (rnk[j] < best_rank) {
- best_rank = rnk[j];
- best = j;
- }
- }
- if (best < 0)
- break;
-
- ids[best] = mrg[best];
- int dead = nxt[best];
- nxt[best] = nxt[dead];
- if (nxt[dead] != -1)
- prv[nxt[dead]] = best;
-
- if (prv[best] != -1)
- rnk[prv[best]] = pair_merge_lookup(tok, ids[prv[best]], ids[best], &mrg[prv[best]]);
- rnk[best] = nxt[best] != -1 ? pair_merge_lookup(tok, ids[best], ids[nxt[best]], &mrg[best])
- : INT_MAX;
- }
-
- for (int j = 0; j != -1; j = nxt[j])
- output_append_id(out_ids, out_cap, n_ids, overflow, ids[j]);
- return 0;
-}
-
-static int append_encoded_piece_into(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *piece,
- size_t piece_len,
- int *out_ids,
- int out_cap,
- int *n_ids,
- int *overflow) {
- if (piece_len == 0)
- return 0;
-
- const char *mapped = NULL;
- if (text_to_bpe_unicode_into(ws, piece, piece_len, &mapped) != 0)
- return -1;
- return encode_bpe_word_into(tok, ws, mapped, out_ids, out_cap, n_ids, overflow);
-}
-
-static int encode_pretokenized_into(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n) {
- const unsigned char *start = (const unsigned char *)text;
- const unsigned char *p = start;
- const unsigned char *end = start + strlen(text);
- int n_ids = 0;
- int overflow = 0;
-
- while (p < end) {
- const unsigned char *piece = p;
- int cp = 0, len = 0;
- if (!utf8_decode_cp_len(p, end, &cp, &len))
- break;
-
- int clen = match_contraction(p, end);
- if (clen > 0) {
- p += clen;
- } else {
- const unsigned char *next = p + len;
- int next_cp = 0, next_len = 0;
- int has_next = utf8_decode_cp_len(next, end, &next_cp, &next_len);
-
- if (!is_newline_cp(cp) && !is_word_cp(cp) && has_next && is_letter_cp(next_cp)) {
- p = next + next_len;
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (!is_letter_cp(c))
- break;
- p += l;
- }
- } else if (is_letter_cp(cp)) {
- p += len;
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (!is_letter_cp(c))
- break;
- p += l;
- }
- } else if (is_number_cp(cp)) {
- p += len;
- } else if (is_space_cp(cp)) {
- const unsigned char *q = p;
- int saw_newline = 0;
- while (q < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(q, end, &c, &l);
- if (!is_space_cp(c))
- break;
- if (is_newline_cp(c))
- saw_newline = 1;
- q += l;
- if (saw_newline) {
- while (q < end) {
- int c2 = 0, l2 = 0;
- utf8_decode_cp_len(q, end, &c2, &l2);
- if (!is_newline_cp(c2))
- break;
- q += l2;
- }
- break;
- }
- }
-
- if (saw_newline) {
- p = q;
- } else if (cp == ' ' && has_next && !is_space_cp(next_cp) && !is_word_cp(next_cp)) {
- p = next + next_len;
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (is_space_cp(c) || is_word_cp(c))
- break;
- p += l;
- }
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (!is_newline_cp(c))
- break;
- p += l;
- }
- } else {
- const unsigned char *last = p;
- const unsigned char *q2 = p;
- int count = 0;
- while (q2 < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(q2, end, &c, &l);
- if (!is_space_cp(c))
- break;
- last = q2;
- q2 += l;
- count++;
- }
- if (q2 < end && count > 1)
- p = last;
- else
- p = q2;
- }
- } else {
- p += len;
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (is_space_cp(c) || is_word_cp(c))
- break;
- p += l;
- }
- while (p < end) {
- int c = 0, l = 0;
- utf8_decode_cp_len(p, end, &c, &l);
- if (!is_newline_cp(c))
- break;
- p += l;
- }
- }
- }
-
- if (append_encoded_piece_into(tok, ws, (const char *)piece, (size_t)(p - piece), out_ids,
- out_cap, &n_ids, &overflow) != 0)
- return -1;
- }
-
- *out_n = n_ids;
- return overflow ? -2 : 0;
-}
-
-static int derive_merges_path(const char *vocab_path, char *out_path, size_t out_cap) {
- const char *slash = strrchr(vocab_path, '/');
- if (!slash) {
- if (snprintf(out_path, out_cap, "merges.txt") >= (int)out_cap)
- return -1;
- return 0;
- }
- size_t dir_len = (size_t)(slash - vocab_path);
- if (dir_len + 12 >= out_cap)
- return -1;
- memcpy(out_path, vocab_path, dir_len);
- out_path[dir_len] = '\0';
- snprintf(out_path + dir_len, out_cap - dir_len, "/merges.txt");
- return 0;
-}
-
-static void trim_newline(char *line) {
- size_t n = strlen(line);
- while (n > 0 && (line[n - 1] == '\n' || line[n - 1] == '\r'))
- line[--n] = '\0';
-}
-
-static int parse_merge_pair(char *line, char **a, char **b) {
- *a = NULL;
- *b = NULL;
- trim_newline(line);
- if (line[0] == '\0')
- return -1;
- if (line[0] == '#')
- return -1;
-
- char *sp = strchr(line, ' ');
- if (!sp)
- return -1;
- *sp = '\0';
- char *p2 = sp + 1;
- while (*p2 == ' ')
- p2++;
- if (*line == '\0' || *p2 == '\0')
- return -1;
- *a = line;
- *b = p2;
- return 0;
-}
-
-static int load_merges_map(embed_tokenizer_t *tok, const char *merges_path) {
- FILE *f = fopen(merges_path, "rb");
- if (!f)
- return -1;
-
- int n_pairs = 0;
- char line[8192];
- while (fgets(line, sizeof(line), f)) {
- char *a = NULL, *b = NULL;
- if (parse_merge_pair(line, &a, &b) == 0)
- n_pairs++;
- }
- if (n_pairs <= 0) {
- fclose(f);
- return -1;
- }
-
- tok->merge_map_cap = next_pow2(n_pairs * 2);
- tok->merge_map = calloc((size_t)tok->merge_map_cap, sizeof(str_int_entry_t));
- if (!tok->merge_map) {
- fclose(f);
- tok->merge_map_cap = 0;
- return -1;
- }
-
- if (fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return -1;
- }
- tok->int_merges_cap = next_pow2(n_pairs * 2);
- pair_merge_entry_t *pm =
- (pair_merge_entry_t *)calloc((size_t)tok->int_merges_cap, sizeof(pair_merge_entry_t));
- if (pm) {
- for (int i = 0; i < tok->int_merges_cap; i++) {
- pm[i].key = UINT64_MAX;
- pm[i].rank = INT_MAX;
- pm[i].merged_id = -1;
- }
- }
- tok->int_merges = pm;
-
- int rank = 0;
- while (fgets(line, sizeof(line), f)) {
- char *a = NULL, *b = NULL;
- if (parse_merge_pair(line, &a, &b) != 0)
- continue;
-
- size_t la = strlen(a), lb = strlen(b);
- char *key = (char *)malloc(la + 1 + lb + 1);
- if (!key)
- continue;
- memcpy(key, a, la);
- key[la] = ' ';
- memcpy(key + la + 1, b, lb);
- key[la + 1 + lb] = '\0';
-
- if (tok->int_merges) {
- /* Resolve (a, b) and the merged token a+b to vocab ids; if any
- * merge is unresolvable, drop the whole int table and keep the
- * string-based loop as the only path. */
- const str_int_entry_t *vm = (const str_int_entry_t *)tok->vocab_map;
- int left = map_get(vm, tok->vocab_map_cap, a);
- int right = map_get(vm, tok->vocab_map_cap, b);
- char ab[8192];
- int merged = -1;
- if (la + lb < sizeof(ab)) {
- memcpy(ab, a, la);
- memcpy(ab + la, b, lb);
- ab[la + lb] = '\0';
- merged = map_get(vm, tok->vocab_map_cap, ab);
- }
- if (left < 0 || right < 0 || merged < 0) {
- free(tok->int_merges);
- tok->int_merges = NULL;
- tok->int_merges_cap = 0;
- } else {
- pair_map_insert((pair_merge_entry_t *)tok->int_merges, tok->int_merges_cap,
- ((uint64_t)(uint32_t)left << 32) | (uint32_t)right, rank, merged);
- }
- }
- (void)map_insert_owned((str_int_entry_t *)tok->merge_map, tok->merge_map_cap, &key, rank);
- free(key);
- rank++;
- }
- fclose(f);
-
- /* Byte-level unicode codepoint -> single-char token id table. */
- if (tok->int_merges) {
- init_gpt2_mapping();
- tok->cp_to_id = (int *)malloc(512 * sizeof(int));
- if (tok->cp_to_id) {
- for (int i = 0; i < 512; i++)
- tok->cp_to_id[i] = -1;
- for (int byte = 0; byte < 256 && tok->cp_to_id; byte++) {
- int cp = gpt2_byte_to_unicode[byte];
- char buf[4];
- int n;
- if (cp < 0x80) {
- buf[0] = (char)cp;
- n = 1;
- } else {
- buf[0] = (char)(0xC0 | (cp >> 6));
- buf[1] = (char)(0x80 | (cp & 0x3F));
- n = 2;
- }
- buf[n] = '\0';
- int id = map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, buf);
- if (id < 0) {
- free(tok->cp_to_id);
- tok->cp_to_id = NULL;
- } else {
- tok->cp_to_id[cp] = id;
- }
- }
- }
- }
- return 0;
-}
-
-/* ========================================================================
- * Public API
- * ======================================================================== */
-
-embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path) {
- FILE *f = fopen(vocab_json_path, "rb");
- if (!f) {
- fprintf(stderr, "embed_tokenizer_load: cannot open %s\n", vocab_json_path);
- return NULL;
- }
-
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return NULL;
- }
- long file_size = ftell(f);
- if (file_size < 0 || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return NULL;
- }
-
- char *json = (char *)malloc((size_t)file_size + 1);
- if (!json || fread(json, 1, (size_t)file_size, f) != (size_t)file_size) {
- fclose(f);
- free(json);
- return NULL;
- }
- fclose(f);
- json[file_size] = '\0';
-
- int max_id = 0;
- const char *p = json;
- skip_ws(&p);
- if (*p != '{') {
- free(json);
- return NULL;
- }
- p++;
-
- const char *saved = p;
- while (*p && *p != '}') {
- skip_ws(&p);
- if (*p == ',') {
- p++;
- continue;
- }
- if (*p == '}')
- break;
-
- char key[4096] = {0};
- if (parse_json_string(&p, key, sizeof(key)) != 0)
- break;
- skip_ws(&p);
- if (*p != ':')
- break;
- p++;
- int64_t id = parse_json_int(&p);
- if (id > max_id)
- max_id = (int)id;
- }
-
- int vocab_size = max_id + 1;
- embed_tokenizer_t *tok = (embed_tokenizer_t *)calloc(1, sizeof(embed_tokenizer_t));
- if (!tok) {
- free(json);
- return NULL;
- }
- tok->vocab_size = vocab_size;
- tok->id_to_text = (char **)calloc((size_t)vocab_size, sizeof(char *));
- tok->id_to_bpe = (char **)calloc((size_t)vocab_size, sizeof(char *));
- if (!tok->id_to_text || !tok->id_to_bpe) {
- embed_tokenizer_free(tok);
- free(json);
- return NULL;
- }
-
- p = saved;
- while (*p && *p != '}') {
- skip_ws(&p);
- if (*p == ',') {
- p++;
- continue;
- }
- if (*p == '}')
- break;
-
- char key[4096] = {0};
- if (parse_json_string(&p, key, sizeof(key)) != 0)
- break;
- skip_ws(&p);
- if (*p != ':')
- break;
- p++;
- int64_t id = parse_json_int(&p);
-
- if (id >= 0 && id < vocab_size) {
- free(tok->id_to_bpe[id]);
- free(tok->id_to_text[id]);
- tok->id_to_bpe[id] = strdup(key);
- tok->id_to_text[id] = decode_gpt2_token(key);
- }
- }
- free(json);
-
- int n_vocab_entries = 0;
- for (int i = 0; i < vocab_size; i++)
- if (tok->id_to_bpe[i])
- n_vocab_entries++;
- tok->vocab_map_cap = next_pow2(n_vocab_entries * 2 + 1);
- tok->vocab_map = calloc((size_t)tok->vocab_map_cap, sizeof(str_int_entry_t));
- if (!tok->vocab_map) {
- embed_tokenizer_free(tok);
- return NULL;
- }
- for (int i = 0; i < vocab_size; i++) {
- if (tok->id_to_bpe[i]) {
- map_insert((str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, tok->id_to_bpe[i], i);
- }
- }
-
- char merges_path[1024];
- if (derive_merges_path(vocab_json_path, merges_path, sizeof(merges_path)) == 0) {
- if (load_merges_map(tok, merges_path) != 0 && embed_verbose >= 2) {
- fprintf(stderr,
- "Tokenizer: merges not loaded from %s (encoding falls back to byte-level)\n",
- merges_path);
- }
- }
-
- return tok;
-}
-
-const char *embed_tokenizer_decode(const embed_tokenizer_t *tok, int token_id) {
- if (!tok || token_id < 0 || token_id >= tok->vocab_size)
- return "";
- return tok->id_to_text[token_id] ? tok->id_to_text[token_id] : "";
-}
-
-int embed_tokenizer_token_id(const embed_tokenizer_t *tok, const char *token) {
- if (!tok || !token || !tok->vocab_map)
- return -1;
- return map_get((const str_int_entry_t *)tok->vocab_map, tok->vocab_map_cap, token);
-}
-
-int *embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int *out_n_tokens) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- if (!tok || !text || text[0] == '\0')
- return NULL;
-
- char *normalized = normalize_nfc_latin(text);
- if (!normalized)
- return NULL;
-
- int *ids = NULL;
- int n_ids = 0;
- if (encode_pretokenized(tok, normalized, &ids, &n_ids) != 0) {
- free(normalized);
- free(ids);
- return NULL;
- }
- free(normalized);
-
- if (out_n_tokens)
- *out_n_tokens = n_ids;
- return ids;
-}
-
-embed_tokenizer_workspace_t *embed_tokenizer_workspace_new(void) {
- return (embed_tokenizer_workspace_t *)calloc(1, sizeof(embed_tokenizer_workspace_t));
-}
-
-void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws) {
- if (!ws)
- return;
- free(ws->normalized);
- free(ws->mapped);
- free(ws->arena);
- free(ws->syms);
- free(ws->ids);
- free(ws->bpe_ids);
- free(ws->bpe_prv);
- free(ws->bpe_nxt);
- free(ws->bpe_rnk);
- free(ws->bpe_mrg);
- free(ws);
-}
-
-int embed_tokenizer_encode_into(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- if (!tok || !ws || !text || out_cap < 0)
- return -1;
- if (text[0] == '\0')
- return 0;
-
- const char *normalized = NULL;
- if (normalize_nfc_latin_into(ws, text, &normalized) != 0)
- return -1;
-
- int n_ids = 0;
- int rc = encode_pretokenized_into(tok, ws, normalized, out_ids, out_cap, &n_ids);
- if (out_n_tokens)
- *out_n_tokens = n_ids;
- return rc;
-}
-
-int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *text,
- int *out_n_tokens) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- if (!tok || !ws || !text || text[0] == '\0')
- return NULL;
-
- size_t len = strlen(text);
- size_t cap_sz = len > (SIZE_MAX - 8) / 2 ? SIZE_MAX : len * 2 + 8;
- if (cap_sz > (size_t)INT_MAX)
- return NULL;
- int cap = (int)cap_sz;
-
- if (reserve_ints(&ws->ids, &ws->ids_cap, cap) != 0)
- return NULL;
-
- int n_ids = 0;
- int rc = embed_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids);
- if (rc == -2) {
- if (reserve_ints(&ws->ids, &ws->ids_cap, n_ids) != 0)
- return NULL;
- rc = embed_tokenizer_encode_into(tok, ws, text, ws->ids, ws->ids_cap, &n_ids);
- }
- if (rc != 0 || n_ids <= 0)
- return NULL;
-
- int *ids = (int *)malloc((size_t)n_ids * sizeof(int));
- if (!ids)
- return NULL;
- memcpy(ids, ws->ids, (size_t)n_ids * sizeof(int));
- if (out_n_tokens)
- *out_n_tokens = n_ids;
- return ids;
-}
-
-void embed_tokenizer_free(embed_tokenizer_t *tok) {
- if (!tok)
- return;
-
- if (tok->id_to_text) {
- for (int i = 0; i < tok->vocab_size; i++)
- free(tok->id_to_text[i]);
- free(tok->id_to_text);
- }
- if (tok->id_to_bpe) {
- for (int i = 0; i < tok->vocab_size; i++)
- free(tok->id_to_bpe[i]);
- free(tok->id_to_bpe);
- }
-
- if (tok->merge_map) {
- str_int_entry_t *m = (str_int_entry_t *)tok->merge_map;
- for (int i = 0; i < tok->merge_map_cap; i++)
- free(m[i].key);
- free(tok->merge_map);
- }
- free(tok->vocab_map);
- free(tok->int_merges);
- free(tok->cp_to_id);
- free(tok);
-}
diff --git a/src/tokenizer_bpe.h b/src/tokenizer_bpe.h
deleted file mode 100644
index 8d13d47..0000000
--- a/src/tokenizer_bpe.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * embed_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level)
- *
- * Supports:
- * - decode token ID -> UTF-8 text
- * - encode UTF-8 text -> token IDs (vocab.json + merges.txt)
- */
-
-#ifndef EMBED_TOKENIZER_H
-#define EMBED_TOKENIZER_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-typedef struct {
- char **id_to_text; /* [vocab_size] decoded text strings */
- char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */
- int vocab_size;
-
- /* Internal hash maps (opaque to callers) */
- void *vocab_map;
- int vocab_map_cap;
- void *merge_map;
- int merge_map_cap;
- /* Integer-pair BPE merge table: (left_id, right_id) -> (rank, merged_id).
- * NULL when the vocab could not resolve every merge; encoding then falls
- * back to the string-based merge loop. */
- void *int_merges;
- int int_merges_cap;
- /* Byte-level unicode codepoint -> single-character token id (512 slots),
- * or NULL when any byte token is missing from the vocab. */
- int *cp_to_id;
-} embed_tokenizer_t;
-
-typedef struct embed_tokenizer_workspace embed_tokenizer_workspace_t;
-
-/* Load tokenizer from vocab.json in model directory */
-EMBED_API embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path);
-
-/* Decode a single token ID to text. Returns pointer to internal string. */
-EMBED_API const char *embed_tokenizer_decode(const embed_tokenizer_t *tok, int token_id);
-
-/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>").
- * Returns -1 when the token is not in the vocab. */
-EMBED_API int embed_tokenizer_token_id(const embed_tokenizer_t *tok, const char *token);
-
-/* Encode UTF-8 text into token IDs using BPE.
- * Returns malloc'd array of token IDs and sets *out_n_tokens.
- * Returns NULL on error (and sets *out_n_tokens to 0). */
-EMBED_API int *
-embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int *out_n_tokens);
-
-/* Reusable scratch for allocation-light repeated encoding. */
-EMBED_API embed_tokenizer_workspace_t *embed_tokenizer_workspace_new(void);
-EMBED_API void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws);
-
-/* Encode into caller-provided out_ids[0..out_cap-1].
- * Returns 0 on success, -1 on error, -2 if out_cap is too small.
- * In all cases *out_n_tokens is set to the number of tokens needed/emitted
- * when out_n_tokens is non-NULL. */
-EMBED_API int embed_tokenizer_encode_into(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens);
-
-/* Same tokenization as embed_tokenizer_encode(), but reuses ws internally and
- * performs only the final returned int[] allocation. */
-EMBED_API int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
-
-/* Free tokenizer */
-EMBED_API void embed_tokenizer_free(embed_tokenizer_t *tok);
-
-#endif /* EMBED_TOKENIZER_H */
diff --git a/src/tokenizer_sentencepiece.c b/src/tokenizer_sentencepiece.c
deleted file mode 100644
index 657e8c9..0000000
--- a/src/tokenizer_sentencepiece.c
+++ /dev/null
@@ -1,963 +0,0 @@
-/*
- * tokenizer_sentencepiece.c - SentencePiece Unigram tokenizer.
- *
- * The implementation reads the small subset of ModelProto needed at runtime:
- * pieces, scores, piece types, trainer model_type, and normalizer options. The
- * normalizer supports SentencePiece's precompiled charsmap format used by
- * nmt_nfkc, then the encoder runs the standard Unigram Viterbi dynamic program
- * over UTF-8 byte offsets.
- */
-
-#include "tokenizer_sentencepiece.h"
-
-#include <errno.h>
-#include <float.h>
-#include <limits.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-enum {
- SP_MODEL_UNIGRAM = 1,
- SP_PIECE_NORMAL = 1,
- SP_PIECE_UNKNOWN = 2,
- SP_PIECE_CONTROL = 3,
- SP_PIECE_USER_DEFINED = 4,
- SP_PIECE_UNUSED = 5,
- SP_PIECE_BYTE = 6,
-};
-
-typedef struct {
- const char *key;
- uint16_t len;
- int val;
-} sp_entry_t;
-
-typedef struct {
- int id;
- int prev;
- double score;
-} sp_best_t;
-
-struct sentencepiece_workspace {
- char *norm;
- size_t norm_len;
- size_t norm_cap;
- sp_best_t *best;
- int best_cap;
- int *ids;
- int ids_cap;
-};
-
-static uint64_t fnv1a_bytes(const char *s, size_t n) {
- uint64_t h = 1469598103934665603ULL;
- for (size_t i = 0; i < n; i++) {
- h ^= (unsigned char)s[i];
- h *= 1099511628211ULL;
- }
- return h;
-}
-
-static int next_pow2(int x) {
- int p = 1;
- while (p < x && p <= (1 << 29))
- p <<= 1;
- return p;
-}
-
-static int map_put(sp_entry_t *map, int cap, const char *key, int len, int val) {
- if (len <= 0 || len > UINT16_MAX)
- return -1;
- uint64_t mask = (uint64_t)cap - 1;
- int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask);
- while (map[pos].key) {
- if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0) {
- map[pos].val = val;
- return 0;
- }
- pos = (int)((pos + 1) & (int)mask);
- }
- map[pos].key = key;
- map[pos].len = (uint16_t)len;
- map[pos].val = val;
- return 0;
-}
-
-static int map_get(const sp_entry_t *map, int cap, const char *key, int len) {
- if (!map || len <= 0)
- return -1;
- uint64_t mask = (uint64_t)cap - 1;
- int pos = (int)(fnv1a_bytes(key, (size_t)len) & mask);
- while (map[pos].key) {
- if (map[pos].len == (uint16_t)len && memcmp(map[pos].key, key, (size_t)len) == 0)
- return map[pos].val;
- pos = (int)((pos + 1) & (int)mask);
- }
- return -1;
-}
-
-static int is_tokenizable_type(int type) {
- return type == SP_PIECE_NORMAL || type == SP_PIECE_USER_DEFINED || type == SP_PIECE_UNUSED ||
- type == SP_PIECE_BYTE;
-}
-
-static int reserve_chars(char **buf, size_t *cap, size_t need) {
- if (need <= *cap)
- return 0;
- size_t nc = *cap ? *cap : 128;
- while (nc < need) {
- if (nc > SIZE_MAX / 2)
- return -1;
- nc *= 2;
- }
- char *p = (char *)realloc(*buf, nc);
- if (!p)
- return -1;
- *buf = p;
- *cap = nc;
- return 0;
-}
-
-static int append_bytes(char **buf, size_t *len, size_t *cap, const char *s, size_t n) {
- if (n > SIZE_MAX - *len - 1 || reserve_chars(buf, cap, *len + n + 1) != 0)
- return -1;
- memcpy(*buf + *len, s, n);
- *len += n;
- (*buf)[*len] = '\0';
- return 0;
-}
-
-static int reserve_best(sentencepiece_workspace_t *ws, int need) {
- if (need <= ws->best_cap)
- return 0;
- int nc = ws->best_cap ? ws->best_cap : 128;
- while (nc < need) {
- if (nc > INT_MAX / 2)
- return -1;
- nc *= 2;
- }
- sp_best_t *p = (sp_best_t *)realloc(ws->best, (size_t)nc * sizeof(*p));
- if (!p)
- return -1;
- ws->best = p;
- ws->best_cap = nc;
- return 0;
-}
-
-static int reserve_ids(sentencepiece_workspace_t *ws, int need) {
- if (need <= ws->ids_cap)
- return 0;
- int nc = ws->ids_cap ? ws->ids_cap : 64;
- while (nc < need) {
- if (nc > INT_MAX / 2)
- return -1;
- nc *= 2;
- }
- int *p = (int *)realloc(ws->ids, (size_t)nc * sizeof(*p));
- if (!p)
- return -1;
- ws->ids = p;
- ws->ids_cap = nc;
- return 0;
-}
-
-static int utf8_len(unsigned char c) {
- if (c < 0x80)
- return 1;
- if ((c & 0xE0) == 0xC0)
- return 2;
- if ((c & 0xF0) == 0xE0)
- return 3;
- if ((c & 0xF8) == 0xF0)
- return 4;
- return 1;
-}
-
-static int valid_utf8_one(const char *s, size_t n, size_t off, size_t *len_out) {
- if (off >= n)
- return 0;
- const unsigned char *p = (const unsigned char *)s + off;
- int len = utf8_len(p[0]);
- if (off + (size_t)len > n) {
- *len_out = 1;
- return 0;
- }
- if (len > 1) {
- for (int i = 1; i < len; i++) {
- if ((p[i] & 0xC0) != 0x80) {
- *len_out = 1;
- return 0;
- }
- }
- }
- *len_out = (size_t)len;
- return 1;
-}
-
-static uint32_t read_le32(const unsigned char *p) {
- return (uint32_t)p[0] | ((uint32_t)p[1] << 8) | ((uint32_t)p[2] << 16) | ((uint32_t)p[3] << 24);
-}
-
-static int pb_read_varint(const unsigned char **p, const unsigned char *end, uint64_t *out) {
- uint64_t v = 0;
- int shift = 0;
- while (*p < end && shift <= 63) {
- unsigned char c = *(*p)++;
- v |= (uint64_t)(c & 0x7f) << shift;
- if (!(c & 0x80)) {
- *out = v;
- return 0;
- }
- shift += 7;
- }
- return -1;
-}
-
-static int pb_read_key(const unsigned char **p, const unsigned char *end, int *field, int *wire) {
- uint64_t key = 0;
- if (pb_read_varint(p, end, &key) != 0)
- return -1;
- *field = (int)(key >> 3);
- *wire = (int)(key & 7);
- return *field > 0 ? 0 : -1;
-}
-
-static int pb_read_len(const unsigned char **p,
- const unsigned char *end,
- const unsigned char **data,
- size_t *len) {
- uint64_t n = 0;
- if (pb_read_varint(p, end, &n) != 0 || n > (uint64_t)(end - *p))
- return -1;
- *data = *p;
- *len = (size_t)n;
- *p += n;
- return 0;
-}
-
-static int pb_skip(const unsigned char **p, const unsigned char *end, int wire) {
- uint64_t v = 0;
- const unsigned char *data = NULL;
- size_t len = 0;
- switch (wire) {
- case 0:
- return pb_read_varint(p, end, &v);
- case 1:
- if ((size_t)(end - *p) < 8)
- return -1;
- *p += 8;
- return 0;
- case 2:
- return pb_read_len(p, end, &data, &len);
- case 5:
- if ((size_t)(end - *p) < 4)
- return -1;
- *p += 4;
- return 0;
- default:
- return -1;
- }
-}
-
-static char *memdup0(const unsigned char *data, size_t len) {
- char *s = (char *)malloc(len + 1);
- if (!s)
- return NULL;
- memcpy(s, data, len);
- s[len] = '\0';
- return s;
-}
-
-static int
-parse_piece(sentencepiece_tokenizer_t *tok, int idx, const unsigned char *data, size_t len) {
- const unsigned char *p = data, *end = data + len;
- char *piece = NULL;
- float score = 0.0f;
- int type = SP_PIECE_NORMAL;
- while (p < end) {
- int field = 0, wire = 0;
- if (pb_read_key(&p, end, &field, &wire) != 0)
- goto fail;
- if (field == 1 && wire == 2) {
- const unsigned char *s = NULL;
- size_t n = 0;
- if (pb_read_len(&p, end, &s, &n) != 0 || n > UINT16_MAX)
- goto fail;
- free(piece);
- piece = memdup0(s, n);
- if (!piece)
- goto fail;
- tok->piece_lens[idx] = (uint16_t)n;
- } else if (field == 2 && wire == 5) {
- if ((size_t)(end - p) < 4)
- goto fail;
- uint32_t bits = read_le32(p);
- memcpy(&score, &bits, sizeof(score));
- p += 4;
- } else if (field == 3 && wire == 0) {
- uint64_t v = 0;
- if (pb_read_varint(&p, end, &v) != 0 || v > UINT8_MAX)
- goto fail;
- type = (int)v;
- } else if (pb_skip(&p, end, wire) != 0) {
- goto fail;
- }
- }
- if (!piece)
- goto fail;
- tok->id_to_piece[idx] = piece;
- tok->scores[idx] = score;
- tok->types[idx] = (uint8_t)type;
- if (is_tokenizable_type(type) && tok->piece_lens[idx] > tok->max_piece_len)
- tok->max_piece_len = tok->piece_lens[idx];
- return 0;
-fail:
- free(piece);
- return -1;
-}
-
-static int parse_trainer(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) {
- const unsigned char *p = data, *end = data + len;
- int model_type = SP_MODEL_UNIGRAM;
- while (p < end) {
- int field = 0, wire = 0;
- if (pb_read_key(&p, end, &field, &wire) != 0)
- return -1;
- if (field == 3 && wire == 0) {
- uint64_t v = 0;
- if (pb_read_varint(&p, end, &v) != 0 || v > INT_MAX)
- return -1;
- model_type = (int)v;
- } else if (pb_skip(&p, end, wire) != 0) {
- return -1;
- }
- }
- if (model_type != SP_MODEL_UNIGRAM) {
- fprintf(stderr, "sentencepiece: unsupported model_type %d (only Unigram is implemented)\n",
- model_type);
- return -1;
- }
- (void)tok;
- return 0;
-}
-
-static int parse_normalizer(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) {
- const unsigned char *p = data, *end = data + len;
- while (p < end) {
- int field = 0, wire = 0;
- if (pb_read_key(&p, end, &field, &wire) != 0)
- return -1;
- if ((field == 3 || field == 4 || field == 5) && wire == 0) {
- uint64_t v = 0;
- if (pb_read_varint(&p, end, &v) != 0)
- return -1;
- if (field == 3)
- tok->add_dummy_prefix = v != 0;
- else if (field == 4)
- tok->remove_extra_whitespaces = v != 0;
- else
- tok->escape_whitespaces = v != 0;
- } else if (field == 2 && wire == 2) {
- const unsigned char *blob = NULL;
- size_t n = 0;
- if (pb_read_len(&p, end, &blob, &n) != 0)
- return -1;
- if (n >= 4) {
- uint32_t xcda_bytes = read_le32(blob);
- if ((size_t)xcda_bytes + 4 <= n && (xcda_bytes % 4) == 0) {
- tok->xcda_count = xcda_bytes / 4;
- tok->xcda = (uint32_t *)malloc(tok->xcda_count * sizeof(*tok->xcda));
- if (!tok->xcda)
- return -1;
- for (size_t i = 0; i < tok->xcda_count; i++)
- tok->xcda[i] = read_le32(blob + 4 + i * 4);
- tok->prefix_replacements_len = n - 4 - xcda_bytes;
- tok->prefix_replacements = (char *)malloc(tok->prefix_replacements_len + 1);
- if (!tok->prefix_replacements)
- return -1;
- memcpy(tok->prefix_replacements, blob + 4 + xcda_bytes,
- tok->prefix_replacements_len);
- tok->prefix_replacements[tok->prefix_replacements_len] = '\0';
- }
- }
- } else if (pb_skip(&p, end, wire) != 0) {
- return -1;
- }
- }
- return 0;
-}
-
-static int count_pieces(const unsigned char *data, size_t len) {
- const unsigned char *p = data, *end = data + len;
- int n = 0;
- while (p < end) {
- int field = 0, wire = 0;
- if (pb_read_key(&p, end, &field, &wire) != 0)
- return -1;
- if (field == 1 && wire == 2) {
- const unsigned char *msg = NULL;
- size_t msg_len = 0;
- if (pb_read_len(&p, end, &msg, &msg_len) != 0)
- return -1;
- n++;
- } else if (pb_skip(&p, end, wire) != 0) {
- return -1;
- }
- }
- return n;
-}
-
-static int parse_model(sentencepiece_tokenizer_t *tok, const unsigned char *data, size_t len) {
- int n_pieces = count_pieces(data, len);
- if (n_pieces <= 0)
- return -1;
- tok->piece_count = n_pieces;
- tok->id_to_piece = (char **)calloc((size_t)n_pieces, sizeof(*tok->id_to_piece));
- tok->piece_lens = (uint16_t *)calloc((size_t)n_pieces, sizeof(*tok->piece_lens));
- tok->scores = (float *)calloc((size_t)n_pieces, sizeof(*tok->scores));
- tok->types = (uint8_t *)calloc((size_t)n_pieces, sizeof(*tok->types));
- if (!tok->id_to_piece || !tok->piece_lens || !tok->scores || !tok->types)
- return -1;
-
- const unsigned char *p = data, *end = data + len;
- int piece_idx = 0;
- int saw_trainer = 0;
- while (p < end) {
- int field = 0, wire = 0;
- if (pb_read_key(&p, end, &field, &wire) != 0)
- return -1;
- if (wire == 2 && field == 1) {
- const unsigned char *msg = NULL;
- size_t msg_len = 0;
- if (pb_read_len(&p, end, &msg, &msg_len) != 0 || piece_idx >= n_pieces ||
- parse_piece(tok, piece_idx++, msg, msg_len) != 0)
- return -1;
- } else if (wire == 2 && field == 2) {
- const unsigned char *msg = NULL;
- size_t msg_len = 0;
- if (pb_read_len(&p, end, &msg, &msg_len) != 0 || parse_trainer(tok, msg, msg_len) != 0)
- return -1;
- saw_trainer = 1;
- } else if (wire == 2 && field == 3) {
- const unsigned char *msg = NULL;
- size_t msg_len = 0;
- if (pb_read_len(&p, end, &msg, &msg_len) != 0 ||
- parse_normalizer(tok, msg, msg_len) != 0)
- return -1;
- } else if (pb_skip(&p, end, wire) != 0) {
- return -1;
- }
- }
- if (piece_idx != n_pieces || !saw_trainer)
- return -1;
- return 0;
-}
-
-static int raw_piece_id(const sentencepiece_tokenizer_t *tok, const char *piece) {
- if (!tok || !piece)
- return -1;
- return map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap, piece,
- (int)strlen(piece));
-}
-
-static int raw_to_model_id(const sentencepiece_tokenizer_t *tok, int raw_id) {
- if (!tok || raw_id < 0 || raw_id >= tok->piece_count)
- return -1;
- if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
- if (raw_id == 1)
- return 0; /* <s> */
- if (raw_id == 2)
- return 2; /* </s> */
- if (raw_id == 0)
- return 3; /* <unk> */
- return raw_id + 1;
- }
- return raw_id;
-}
-
-static int model_to_raw_id(const sentencepiece_tokenizer_t *tok, int id) {
- if (!tok || id < 0)
- return -1;
- if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
- if (id == 0)
- return 1;
- if (id == 2)
- return 2;
- if (id == 3)
- return 0;
- if (id >= 4 && id <= tok->piece_count)
- return id - 1;
- return -1;
- }
- return id < tok->piece_count ? id : -1;
-}
-
-static int file_contains_xlm_roberta(const char *model_dir, const char *name) {
- char path[1024];
- int n = snprintf(path, sizeof(path), "%s/%s", model_dir, name);
- if (n < 0 || (size_t)n >= sizeof(path))
- return 0;
- FILE *f = fopen(path, "rb");
- if (!f)
- return 0;
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return 0;
- }
- long sz = ftell(f);
- if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return 0;
- }
- char *buf = (char *)malloc((size_t)sz + 1);
- if (!buf) {
- fclose(f);
- return 0;
- }
- size_t got = fread(buf, 1, (size_t)sz, f);
- fclose(f);
- buf[got] = '\0';
- int yes = strstr(buf, "XLMRobertaTokenizer") != NULL ||
- strstr(buf, "XLMRobertaTokenizerFast") != NULL ||
- strstr(buf, "XLMRobertaModel") != NULL || strstr(buf, "xlm-roberta") != NULL;
- free(buf);
- return yes;
-}
-
-static int detect_xlm_roberta(const char *model_dir) {
- return file_contains_xlm_roberta(model_dir, "tokenizer_config.json") ||
- file_contains_xlm_roberta(model_dir, "config.json");
-}
-
-static int find_model_file(const char *model_dir, char path[1024]) {
- static const char *names[] = {
- "sentencepiece.bpe.model",
- "tokenizer.model",
- "spiece.model",
- };
- for (int i = 0; i < (int)(sizeof(names) / sizeof(names[0])); i++) {
- int n = snprintf(path, 1024, "%s/%s", model_dir, names[i]);
- if (n < 0 || n >= 1024)
- return -1;
- FILE *f = fopen(path, "rb");
- if (f) {
- fclose(f);
- return 0;
- }
- if (errno != ENOENT)
- return -1;
- }
- return -1;
-}
-
-static int load_file(const char *path, unsigned char **out, size_t *out_len) {
- FILE *f = fopen(path, "rb");
- if (!f)
- return -1;
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return -1;
- }
- long sz = ftell(f);
- if (sz < 0 || (size_t)sz == SIZE_MAX || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return -1;
- }
- unsigned char *buf = (unsigned char *)malloc((size_t)sz);
- if (!buf) {
- fclose(f);
- return -1;
- }
- if (fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
- fclose(f);
- free(buf);
- return -1;
- }
- fclose(f);
- *out = buf;
- *out_len = (size_t)sz;
- return 0;
-}
-
-static uint32_t xcda_base(const sentencepiece_tokenizer_t *tok, size_t idx) {
- uint32_t v = tok->xcda[idx];
- return (v >> 10) << ((v & (1u << 9)) >> 6);
-}
-
-static uint32_t xcda_lcheck(const sentencepiece_tokenizer_t *tok, size_t idx) {
- return tok->xcda[idx] & ((1u << 31) | 0xffu);
-}
-
-static int xcda_leaf(const sentencepiece_tokenizer_t *tok, size_t idx) {
- return (tok->xcda[idx] >> 8) & 1u;
-}
-
-static uint32_t xcda_value(const sentencepiece_tokenizer_t *tok, size_t idx) {
- return tok->xcda[idx] & ((1u << 31) - 1u);
-}
-
-typedef struct {
- const char *data;
- size_t len;
- size_t consumed;
-} norm_prefix_t;
-
-static norm_prefix_t normalize_prefix(const sentencepiece_tokenizer_t *tok,
- const char *input,
- size_t input_len,
- size_t off) {
- static const char repl[] = "\xef\xbf\xbd";
- if (off >= input_len)
- return (norm_prefix_t){input + off, 0, 0};
-
- size_t best_len = 0;
- size_t best_repl = 0;
- if (tok->xcda && tok->xcda_count > 0) {
- uint32_t node = xcda_base(tok, 0);
- for (size_t p = off; p < input_len; p++) {
- unsigned char c = (unsigned char)input[p];
- if (c == 0)
- break;
- node ^= c;
- if (node >= tok->xcda_count || xcda_lcheck(tok, node) != c)
- break;
- int leaf = xcda_leaf(tok, node);
- node ^= xcda_base(tok, node);
- if (leaf) {
- best_len = p - off + 1;
- best_repl = xcda_value(tok, node);
- }
- }
- }
- if (best_len > 0 && best_repl < tok->prefix_replacements_len) {
- const char *r = tok->prefix_replacements + best_repl;
- size_t max = tok->prefix_replacements_len - best_repl;
- size_t n = strnlen(r, max);
- return (norm_prefix_t){r, n, best_len};
- }
-
- size_t clen = 1;
- if (valid_utf8_one(input, input_len, off, &clen))
- return (norm_prefix_t){input + off, clen, clen};
- return (norm_prefix_t){repl, 3, 1};
-}
-
-static int normalize_into(const sentencepiece_tokenizer_t *tok,
- sentencepiece_workspace_t *ws,
- const char *text) {
- ws->norm_len = 0;
- if (!text)
- return -1;
- size_t input_len = strlen(text);
- const char space[] = "\xe2\x96\x81"; /* U+2581 */
- const char *space_s = tok->escape_whitespaces ? space : " ";
- size_t space_n = tok->escape_whitespaces ? 3 : 1;
- int processing_non_ws = 0;
- int space_prepended = 0;
-
- for (size_t off = 0; off < input_len;) {
- norm_prefix_t r = normalize_prefix(tok, text, input_len, off);
- for (size_t i = 0; i < r.len; i++) {
- char c = r.data[i];
- if (c != ' ') {
- if (!processing_non_ws) {
- processing_non_ws = 1;
- if ((tok->add_dummy_prefix && !space_prepended) ||
- tok->remove_extra_whitespaces) {
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s,
- space_n) != 0)
- return -1;
- space_prepended = 1;
- }
- }
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, &c, 1) != 0)
- return -1;
- } else {
- processing_non_ws = 0;
- if (!tok->remove_extra_whitespaces) {
- if (append_bytes(&ws->norm, &ws->norm_len, &ws->norm_cap, space_s, space_n) !=
- 0)
- return -1;
- }
- }
- }
- off += r.consumed ? r.consumed : 1;
- }
- if (ws->norm)
- ws->norm[ws->norm_len] = '\0';
- return 0;
-}
-
-static double compute_unknown_score(const sentencepiece_tokenizer_t *tok) {
- float min_score = FLT_MAX;
- for (int i = 0; i < tok->piece_count; i++) {
- if (tok->types[i] == SP_PIECE_NORMAL && tok->scores[i] < min_score)
- min_score = tok->scores[i];
- }
- if (min_score == FLT_MAX)
- min_score = -10.0f;
- return (double)min_score - 10.0;
-}
-
-static int segment(sentencepiece_tokenizer_t const *tok, sentencepiece_workspace_t *ws) {
- size_t n = ws->norm_len;
- if (n == 0)
- return 0;
- if (n > (size_t)INT_MAX - 1 || reserve_best(ws, (int)n + 1) != 0 ||
- reserve_ids(ws, (int)n + 1) != 0)
- return -1;
- for (size_t i = 0; i <= n; i++)
- ws->best[i] = (sp_best_t){tok->unk_id, -1, -DBL_MAX};
- ws->best[0] = (sp_best_t){tok->unk_id, 0, 0.0};
- double unk_score = tok->unknown_score;
-
- for (size_t off = 0; off < n;) {
- size_t cp_len = 1;
- valid_utf8_one(ws->norm, n, off, &cp_len);
- sp_best_t cur = ws->best[off];
- if (cur.score <= -DBL_MAX / 2.0) {
- off += cp_len;
- continue;
- }
-
- int single_cp_found = 0;
- for (size_t end = off + cp_len; end <= n && end - off <= (size_t)tok->max_piece_len;) {
- int raw = map_get((const sp_entry_t *)tok->piece_map, tok->piece_map_cap,
- ws->norm + off, (int)(end - off));
- if (raw >= 0 && is_tokenizable_type(tok->types[raw])) {
- if (end - off == cp_len)
- single_cp_found = 1;
- double sc =
- cur.score + (tok->types[raw] == SP_PIECE_USER_DEFINED ? 0.0 : tok->scores[raw]);
- if (sc > ws->best[end].score)
- ws->best[end] = (sp_best_t){raw_to_model_id(tok, raw), (int)off, sc};
- }
- if (end == n)
- break;
- size_t next_len = 1;
- valid_utf8_one(ws->norm, n, end, &next_len);
- end += next_len;
- }
- if (!single_cp_found) {
- size_t end = off + cp_len;
- double sc = cur.score + unk_score;
- if (sc > ws->best[end].score)
- ws->best[end] = (sp_best_t){tok->unk_id, (int)off, sc};
- }
- off += cp_len;
- }
-
- int out_n = 0;
- int prev_unk = 0;
- for (size_t off = n; off > 0;) {
- sp_best_t b = ws->best[off];
- if (b.prev < 0 || b.prev >= (int)off)
- return -1;
- int is_unk = b.id == tok->unk_id;
- if (!(prev_unk && is_unk)) {
- if (out_n >= ws->ids_cap)
- return -1;
- ws->ids[out_n++] = b.id;
- }
- prev_unk = is_unk;
- off = (size_t)b.prev;
- }
- for (int i = 0; i < out_n / 2; i++) {
- int t = ws->ids[i];
- ws->ids[i] = ws->ids[out_n - 1 - i];
- ws->ids[out_n - 1 - i] = t;
- }
- return out_n;
-}
-
-sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir) {
- if (!model_dir)
- return NULL;
- char path[1024];
- if (find_model_file(model_dir, path) != 0) {
- fprintf(stderr, "sentencepiece: model file not found under %s\n", model_dir);
- return NULL;
- }
- unsigned char *data = NULL;
- size_t len = 0;
- if (load_file(path, &data, &len) != 0) {
- fprintf(stderr, "sentencepiece: cannot read %s\n", path);
- return NULL;
- }
-
- sentencepiece_tokenizer_t *tok =
- (sentencepiece_tokenizer_t *)calloc(1, sizeof(sentencepiece_tokenizer_t));
- if (!tok) {
- free(data);
- return NULL;
- }
- tok->add_dummy_prefix = 1;
- tok->remove_extra_whitespaces = 1;
- tok->escape_whitespaces = 1;
- tok->id_map =
- detect_xlm_roberta(model_dir) ? SENTENCEPIECE_XLM_ROBERTA : SENTENCEPIECE_IDENTITY;
- if (parse_model(tok, data, len) != 0) {
- fprintf(stderr, "sentencepiece: malformed or unsupported model: %s\n", path);
- free(data);
- sentencepiece_tokenizer_free(tok);
- return NULL;
- }
- free(data);
-
- int cap = next_pow2(tok->piece_count * 2 + 1);
- tok->piece_map = calloc((size_t)cap, sizeof(sp_entry_t));
- if (!tok->piece_map) {
- sentencepiece_tokenizer_free(tok);
- return NULL;
- }
- tok->piece_map_cap = cap;
- for (int i = 0; i < tok->piece_count; i++) {
- if (!tok->id_to_piece[i] || map_put((sp_entry_t *)tok->piece_map, cap, tok->id_to_piece[i],
- tok->piece_lens[i], i) != 0) {
- sentencepiece_tokenizer_free(tok);
- return NULL;
- }
- }
-
- tok->unk_id = raw_to_model_id(tok, raw_piece_id(tok, "<unk>"));
- tok->bos_id = raw_to_model_id(tok, raw_piece_id(tok, "<s>"));
- tok->eos_id = raw_to_model_id(tok, raw_piece_id(tok, "</s>"));
- tok->pad_id = tok->id_map == SENTENCEPIECE_XLM_ROBERTA
- ? 1
- : raw_to_model_id(tok, raw_piece_id(tok, "<pad>"));
- tok->mask_id = tok->id_map == SENTENCEPIECE_XLM_ROBERTA
- ? tok->piece_count + 1
- : raw_to_model_id(tok, raw_piece_id(tok, "<mask>"));
- tok->vocab_size =
- tok->id_map == SENTENCEPIECE_XLM_ROBERTA ? tok->piece_count + 2 : tok->piece_count;
- tok->unknown_score = compute_unknown_score(tok);
- if (tok->unk_id < 0 || tok->max_piece_len <= 0) {
- fprintf(stderr, "sentencepiece: missing <unk> or normal pieces in %s\n", path);
- sentencepiece_tokenizer_free(tok);
- return NULL;
- }
- return tok;
-}
-
-const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id) {
- if (!tok || id < 0)
- return NULL;
- if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
- if (id == 1)
- return "<pad>";
- if (id == tok->piece_count + 1)
- return "<mask>";
- }
- int raw = model_to_raw_id(tok, id);
- return raw >= 0 ? tok->id_to_piece[raw] : NULL;
-}
-
-int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok, const char *token) {
- if (!tok || !token)
- return -1;
- if (tok->id_map == SENTENCEPIECE_XLM_ROBERTA) {
- if (!strcmp(token, "<s>"))
- return 0;
- if (!strcmp(token, "<pad>"))
- return 1;
- if (!strcmp(token, "</s>"))
- return 2;
- if (!strcmp(token, "<unk>"))
- return 3;
- if (!strcmp(token, "<mask>"))
- return tok->piece_count + 1;
- }
- return raw_to_model_id(tok, raw_piece_id(tok, token));
-}
-
-int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok,
- sentencepiece_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- if (!tok || !ws || !text)
- return -1;
- if (normalize_into(tok, ws, text) != 0)
- return -1;
- int n = segment(tok, ws);
- if (n < 0)
- return -1;
- if (out_n_tokens)
- *out_n_tokens = n;
- if (n == 0)
- return 0;
- if (!out_ids || out_cap < n)
- return -2;
- memcpy(out_ids, ws->ids, (size_t)n * sizeof(*out_ids));
- return 0;
-}
-
-int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok,
- sentencepiece_workspace_t *ws,
- const char *text,
- int *out_n_tokens) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- if (!tok || !ws || !text)
- return NULL;
- int n = 0;
- int rc = sentencepiece_tokenizer_encode_into(tok, ws, text, NULL, 0, &n);
- if (rc < 0 && rc != -2)
- return NULL;
- if (out_n_tokens)
- *out_n_tokens = n;
- if (n == 0)
- return NULL;
- int *ids = (int *)malloc((size_t)n * sizeof(*ids));
- if (!ids)
- return NULL;
- memcpy(ids, ws->ids, (size_t)n * sizeof(*ids));
- return ids;
-}
-
-int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok,
- const char *text,
- int *out_n_tokens) {
- sentencepiece_workspace_t *ws = sentencepiece_workspace_new();
- if (!ws) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- return NULL;
- }
- int *ids = sentencepiece_tokenizer_encode_with_workspace(tok, ws, text, out_n_tokens);
- sentencepiece_workspace_free(ws);
- return ids;
-}
-
-sentencepiece_workspace_t *sentencepiece_workspace_new(void) {
- return (sentencepiece_workspace_t *)calloc(1, sizeof(sentencepiece_workspace_t));
-}
-
-void sentencepiece_workspace_free(sentencepiece_workspace_t *ws) {
- if (!ws)
- return;
- free(ws->norm);
- free(ws->best);
- free(ws->ids);
- free(ws);
-}
-
-void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok) {
- if (!tok)
- return;
- if (tok->id_to_piece) {
- for (int i = 0; i < tok->piece_count; i++)
- free(tok->id_to_piece[i]);
- }
- free(tok->id_to_piece);
- free(tok->piece_lens);
- free(tok->scores);
- free(tok->types);
- free(tok->piece_map);
- free(tok->xcda);
- free(tok->prefix_replacements);
- free(tok);
-}
diff --git a/src/tokenizer_sentencepiece.h b/src/tokenizer_sentencepiece.h
deleted file mode 100644
index b4ce48c..0000000
--- a/src/tokenizer_sentencepiece.h
+++ /dev/null
@@ -1,99 +0,0 @@
-/*
- * tokenizer_sentencepiece.h - SentencePiece tokenizer for XLM-R style models.
- *
- * Loads a SentencePiece ModelProto (`sentencepiece.bpe.model`, `tokenizer.model`,
- * or `spiece.model`) and implements the Unigram Viterbi encoder used by XLM-R
- * and multilingual E5. It encodes text only; frontends add the model's special
- * token layout, e.g. <s> ... </s> for XLM-R.
- */
-
-#ifndef SENTENCEPIECE_TOKENIZER_H
-#define SENTENCEPIECE_TOKENIZER_H
-
-#include <stddef.h>
-#include <stdint.h>
-
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-typedef enum {
- SENTENCEPIECE_IDENTITY = 0,
- SENTENCEPIECE_XLM_ROBERTA = 1,
-} sentencepiece_id_map_t;
-
-typedef struct {
- char **id_to_piece; /* [piece_count] raw SentencePiece ids */
- uint16_t *piece_lens; /* byte length for id_to_piece entries */
- float *scores; /* SentencePiece score per raw id */
- uint8_t *types; /* SentencePiece piece type per raw id */
- int piece_count; /* raw SentencePiece vocabulary size */
- int vocab_size; /* model embedding vocab size after id mapping */
- int unk_id; /* model-space unknown-token id */
- int bos_id; /* model-space beginning-token id, or -1 */
- int eos_id; /* model-space end-token id, or -1 */
- int pad_id; /* model-space pad-token id, or -1 */
- int mask_id; /* model-space mask-token id, or -1 */
- int max_piece_len; /* max normal/user/unused piece byte length */
- double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */
- int add_dummy_prefix;
- int remove_extra_whitespaces;
- int escape_whitespaces;
- sentencepiece_id_map_t id_map;
-
- void *piece_map; /* piece bytes -> raw SentencePiece id */
- int piece_map_cap;
-
- uint32_t *xcda;
- size_t xcda_count;
- char *prefix_replacements;
- size_t prefix_replacements_len;
-} sentencepiece_tokenizer_t;
-
-typedef struct sentencepiece_workspace sentencepiece_workspace_t;
-
-/* Load the first supported SentencePiece model found in model_dir. Returns NULL
- * on missing model file, unsupported model type, malformed protobuf, or OOM. */
-EMBED_API sentencepiece_tokenizer_t *sentencepiece_tokenizer_load(const char *model_dir);
-
-/* Decode one model-space id to an internal token string, or NULL when invalid. */
-EMBED_API const char *sentencepiece_tokenizer_decode(const sentencepiece_tokenizer_t *tok, int id);
-
-/* Resolve a named token (e.g. "<s>", "</s>", "<pad>", "<mask>") to a
- * model-space id, or -1 when absent. */
-EMBED_API int sentencepiece_tokenizer_token_id(const sentencepiece_tokenizer_t *tok,
- const char *token);
-
-/* Encode UTF-8 text into malloc'd model-space token ids. Empty text returns
- * NULL with *out_n_tokens = 0. NULL with a negative/unspecified count is an
- * error. */
-EMBED_API int *sentencepiece_tokenizer_encode(const sentencepiece_tokenizer_t *tok,
- const char *text,
- int *out_n_tokens);
-
-EMBED_API sentencepiece_workspace_t *sentencepiece_workspace_new(void);
-EMBED_API void sentencepiece_workspace_free(sentencepiece_workspace_t *ws);
-
-/* Encode into caller-provided out_ids. Returns 0 on success, -1 on error, -2 if
- * out_cap is too small. *out_n_tokens receives the emitted/needed token count. */
-EMBED_API int sentencepiece_tokenizer_encode_into(const sentencepiece_tokenizer_t *tok,
- sentencepiece_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens);
-
-/* Same tokenization as sentencepiece_tokenizer_encode(), reusing ws internally
- * and performing only the final returned int[] allocation. */
-EMBED_API int *sentencepiece_tokenizer_encode_with_workspace(const sentencepiece_tokenizer_t *tok,
- sentencepiece_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
-
-EMBED_API void sentencepiece_tokenizer_free(sentencepiece_tokenizer_t *tok);
-
-#endif /* SENTENCEPIECE_TOKENIZER_H */
diff --git a/src/tokenizer_unicode.h b/src/tokenizer_unicode.h
deleted file mode 100644
index 8ca2458..0000000
--- a/src/tokenizer_unicode.h
+++ /dev/null
@@ -1,4985 +0,0 @@
-/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata
- * (Unicode 13.0.0). Do not edit by hand.
- *
- * Tables for the BasicTokenizer, matching transformers' BertTokenizer.
- *
- * WP_PUNCT_RANGES / WP_CONTROL_RANGES: _is_punctuation and _is_control as sorted
- * [lo, hi] ranges. Punctuation is the ASCII set (33-47/58-64/91-96/123-126) plus
- * every category P* codepoint. Control is categories Cc, Cf, Co minus
- * tab/newline/carriage-return (those route to whitespace). Unassigned (Cn) is
- * intentionally not control: it never occurs in real text and would bind the
- * table to one Unicode version.
- *
- * WP_FOLD: the do_lower_case fold (Python str.lower then NFD then drop Mn marks),
- * sorted by codepoint. n is the output length (0 drops a combining mark; up to
- * 3). Hangul syllables (U+AC00..U+D7A3) decompose by formula on the C side and
- * are excluded here. */
-#ifndef WORDPIECE_UNICODE_H
-#define WORDPIECE_UNICODE_H
-
-#include <stdint.h>
-
-typedef struct {
- uint32_t lo;
- uint32_t hi;
-} wp_range_t;
-
-typedef struct {
- uint32_t cp;
- uint32_t out[3];
- uint8_t n;
-} wp_fold_t;
-
-static const wp_range_t WP_PUNCT_RANGES[] = {
- {0x21, 0x2F}, {0x3A, 0x40}, {0x5B, 0x60}, {0x7B, 0x7E}, {0xA1, 0xA1}, {0xA7, 0xA7},
- {0xAB, 0xAB}, {0xB6, 0xB7}, {0xBB, 0xBB}, {0xBF, 0xBF}, {0x37E, 0x37E}, {0x387, 0x387},
- {0x55A, 0x55F}, {0x589, 0x58A}, {0x5BE, 0x5BE}, {0x5C0, 0x5C0}, {0x5C3, 0x5C3},
- {0x5C6, 0x5C6}, {0x5F3, 0x5F4}, {0x609, 0x60A}, {0x60C, 0x60D}, {0x61B, 0x61B},
- {0x61E, 0x61F}, {0x66A, 0x66D}, {0x6D4, 0x6D4}, {0x700, 0x70D}, {0x7F7, 0x7F9},
- {0x830, 0x83E}, {0x85E, 0x85E}, {0x964, 0x965}, {0x970, 0x970}, {0x9FD, 0x9FD},
- {0xA76, 0xA76}, {0xAF0, 0xAF0}, {0xC77, 0xC77}, {0xC84, 0xC84}, {0xDF4, 0xDF4},
- {0xE4F, 0xE4F}, {0xE5A, 0xE5B}, {0xF04, 0xF12}, {0xF14, 0xF14}, {0xF3A, 0xF3D},
- {0xF85, 0xF85}, {0xFD0, 0xFD4}, {0xFD9, 0xFDA}, {0x104A, 0x104F}, {0x10FB, 0x10FB},
- {0x1360, 0x1368}, {0x1400, 0x1400}, {0x166E, 0x166E}, {0x169B, 0x169C}, {0x16EB, 0x16ED},
- {0x1735, 0x1736}, {0x17D4, 0x17D6}, {0x17D8, 0x17DA}, {0x1800, 0x180A}, {0x1944, 0x1945},
- {0x1A1E, 0x1A1F}, {0x1AA0, 0x1AA6}, {0x1AA8, 0x1AAD}, {0x1B5A, 0x1B60}, {0x1BFC, 0x1BFF},
- {0x1C3B, 0x1C3F}, {0x1C7E, 0x1C7F}, {0x1CC0, 0x1CC7}, {0x1CD3, 0x1CD3}, {0x2010, 0x2027},
- {0x2030, 0x2043}, {0x2045, 0x2051}, {0x2053, 0x205E}, {0x207D, 0x207E}, {0x208D, 0x208E},
- {0x2308, 0x230B}, {0x2329, 0x232A}, {0x2768, 0x2775}, {0x27C5, 0x27C6}, {0x27E6, 0x27EF},
- {0x2983, 0x2998}, {0x29D8, 0x29DB}, {0x29FC, 0x29FD}, {0x2CF9, 0x2CFC}, {0x2CFE, 0x2CFF},
- {0x2D70, 0x2D70}, {0x2E00, 0x2E2E}, {0x2E30, 0x2E4F}, {0x2E52, 0x2E52}, {0x3001, 0x3003},
- {0x3008, 0x3011}, {0x3014, 0x301F}, {0x3030, 0x3030}, {0x303D, 0x303D}, {0x30A0, 0x30A0},
- {0x30FB, 0x30FB}, {0xA4FE, 0xA4FF}, {0xA60D, 0xA60F}, {0xA673, 0xA673}, {0xA67E, 0xA67E},
- {0xA6F2, 0xA6F7}, {0xA874, 0xA877}, {0xA8CE, 0xA8CF}, {0xA8F8, 0xA8FA}, {0xA8FC, 0xA8FC},
- {0xA92E, 0xA92F}, {0xA95F, 0xA95F}, {0xA9C1, 0xA9CD}, {0xA9DE, 0xA9DF}, {0xAA5C, 0xAA5F},
- {0xAADE, 0xAADF}, {0xAAF0, 0xAAF1}, {0xABEB, 0xABEB}, {0xFD3E, 0xFD3F}, {0xFE10, 0xFE19},
- {0xFE30, 0xFE52}, {0xFE54, 0xFE61}, {0xFE63, 0xFE63}, {0xFE68, 0xFE68}, {0xFE6A, 0xFE6B},
- {0xFF01, 0xFF03}, {0xFF05, 0xFF0A}, {0xFF0C, 0xFF0F}, {0xFF1A, 0xFF1B}, {0xFF1F, 0xFF20},
- {0xFF3B, 0xFF3D}, {0xFF3F, 0xFF3F}, {0xFF5B, 0xFF5B}, {0xFF5D, 0xFF5D}, {0xFF5F, 0xFF65},
- {0x10100, 0x10102}, {0x1039F, 0x1039F}, {0x103D0, 0x103D0}, {0x1056F, 0x1056F},
- {0x10857, 0x10857}, {0x1091F, 0x1091F}, {0x1093F, 0x1093F}, {0x10A50, 0x10A58},
- {0x10A7F, 0x10A7F}, {0x10AF0, 0x10AF6}, {0x10B39, 0x10B3F}, {0x10B99, 0x10B9C},
- {0x10EAD, 0x10EAD}, {0x10F55, 0x10F59}, {0x11047, 0x1104D}, {0x110BB, 0x110BC},
- {0x110BE, 0x110C1}, {0x11140, 0x11143}, {0x11174, 0x11175}, {0x111C5, 0x111C8},
- {0x111CD, 0x111CD}, {0x111DB, 0x111DB}, {0x111DD, 0x111DF}, {0x11238, 0x1123D},
- {0x112A9, 0x112A9}, {0x1144B, 0x1144F}, {0x1145A, 0x1145B}, {0x1145D, 0x1145D},
- {0x114C6, 0x114C6}, {0x115C1, 0x115D7}, {0x11641, 0x11643}, {0x11660, 0x1166C},
- {0x1173C, 0x1173E}, {0x1183B, 0x1183B}, {0x11944, 0x11946}, {0x119E2, 0x119E2},
- {0x11A3F, 0x11A46}, {0x11A9A, 0x11A9C}, {0x11A9E, 0x11AA2}, {0x11C41, 0x11C45},
- {0x11C70, 0x11C71}, {0x11EF7, 0x11EF8}, {0x11FFF, 0x11FFF}, {0x12470, 0x12474},
- {0x16A6E, 0x16A6F}, {0x16AF5, 0x16AF5}, {0x16B37, 0x16B3B}, {0x16B44, 0x16B44},
- {0x16E97, 0x16E9A}, {0x16FE2, 0x16FE2}, {0x1BC9F, 0x1BC9F}, {0x1DA87, 0x1DA8B},
- {0x1E95E, 0x1E95F},
-};
-enum { WP_PUNCT_RANGES_N = (int)(sizeof(WP_PUNCT_RANGES) / sizeof(WP_PUNCT_RANGES[0])) };
-
-static const wp_range_t WP_CONTROL_RANGES[] = {
- {0x0, 0x8}, {0xB, 0xC}, {0xE, 0x1F}, {0x7F, 0x9F}, {0xAD, 0xAD}, {0x600, 0x605},
- {0x61C, 0x61C}, {0x6DD, 0x6DD}, {0x70F, 0x70F}, {0x8E2, 0x8E2}, {0x180E, 0x180E},
- {0x200B, 0x200F}, {0x202A, 0x202E}, {0x2060, 0x2064}, {0x2066, 0x206F}, {0xE000, 0xF8FF},
- {0xFEFF, 0xFEFF}, {0xFFF9, 0xFFFB}, {0x110BD, 0x110BD}, {0x110CD, 0x110CD},
- {0x13430, 0x13438}, {0x1BCA0, 0x1BCA3}, {0x1D173, 0x1D17A}, {0xE0001, 0xE0001},
- {0xE0020, 0xE007F}, {0xF0000, 0xFFFFD}, {0x100000, 0x10FFFD},
-};
-enum { WP_CONTROL_RANGES_N = (int)(sizeof(WP_CONTROL_RANGES) / sizeof(WP_CONTROL_RANGES[0])) };
-
-static const wp_fold_t WP_FOLD[] = {
- {0x41, {0x61, 0x0, 0x0}, 1},
- {0x42, {0x62, 0x0, 0x0}, 1},
- {0x43, {0x63, 0x0, 0x0}, 1},
- {0x44, {0x64, 0x0, 0x0}, 1},
- {0x45, {0x65, 0x0, 0x0}, 1},
- {0x46, {0x66, 0x0, 0x0}, 1},
- {0x47, {0x67, 0x0, 0x0}, 1},
- {0x48, {0x68, 0x0, 0x0}, 1},
- {0x49, {0x69, 0x0, 0x0}, 1},
- {0x4A, {0x6A, 0x0, 0x0}, 1},
- {0x4B, {0x6B, 0x0, 0x0}, 1},
- {0x4C, {0x6C, 0x0, 0x0}, 1},
- {0x4D, {0x6D, 0x0, 0x0}, 1},
- {0x4E, {0x6E, 0x0, 0x0}, 1},
- {0x4F, {0x6F, 0x0, 0x0}, 1},
- {0x50, {0x70, 0x0, 0x0}, 1},
- {0x51, {0x71, 0x0, 0x0}, 1},
- {0x52, {0x72, 0x0, 0x0}, 1},
- {0x53, {0x73, 0x0, 0x0}, 1},
- {0x54, {0x74, 0x0, 0x0}, 1},
- {0x55, {0x75, 0x0, 0x0}, 1},
- {0x56, {0x76, 0x0, 0x0}, 1},
- {0x57, {0x77, 0x0, 0x0}, 1},
- {0x58, {0x78, 0x0, 0x0}, 1},
- {0x59, {0x79, 0x0, 0x0}, 1},
- {0x5A, {0x7A, 0x0, 0x0}, 1},
- {0xC0, {0x61, 0x0, 0x0}, 1},
- {0xC1, {0x61, 0x0, 0x0}, 1},
- {0xC2, {0x61, 0x0, 0x0}, 1},
- {0xC3, {0x61, 0x0, 0x0}, 1},
- {0xC4, {0x61, 0x0, 0x0}, 1},
- {0xC5, {0x61, 0x0, 0x0}, 1},
- {0xC6, {0xE6, 0x0, 0x0}, 1},
- {0xC7, {0x63, 0x0, 0x0}, 1},
- {0xC8, {0x65, 0x0, 0x0}, 1},
- {0xC9, {0x65, 0x0, 0x0}, 1},
- {0xCA, {0x65, 0x0, 0x0}, 1},
- {0xCB, {0x65, 0x0, 0x0}, 1},
- {0xCC, {0x69, 0x0, 0x0}, 1},
- {0xCD, {0x69, 0x0, 0x0}, 1},
- {0xCE, {0x69, 0x0, 0x0}, 1},
- {0xCF, {0x69, 0x0, 0x0}, 1},
- {0xD0, {0xF0, 0x0, 0x0}, 1},
- {0xD1, {0x6E, 0x0, 0x0}, 1},
- {0xD2, {0x6F, 0x0, 0x0}, 1},
- {0xD3, {0x6F, 0x0, 0x0}, 1},
- {0xD4, {0x6F, 0x0, 0x0}, 1},
- {0xD5, {0x6F, 0x0, 0x0}, 1},
- {0xD6, {0x6F, 0x0, 0x0}, 1},
- {0xD8, {0xF8, 0x0, 0x0}, 1},
- {0xD9, {0x75, 0x0, 0x0}, 1},
- {0xDA, {0x75, 0x0, 0x0}, 1},
- {0xDB, {0x75, 0x0, 0x0}, 1},
- {0xDC, {0x75, 0x0, 0x0}, 1},
- {0xDD, {0x79, 0x0, 0x0}, 1},
- {0xDE, {0xFE, 0x0, 0x0}, 1},
- {0xE0, {0x61, 0x0, 0x0}, 1},
- {0xE1, {0x61, 0x0, 0x0}, 1},
- {0xE2, {0x61, 0x0, 0x0}, 1},
- {0xE3, {0x61, 0x0, 0x0}, 1},
- {0xE4, {0x61, 0x0, 0x0}, 1},
- {0xE5, {0x61, 0x0, 0x0}, 1},
- {0xE7, {0x63, 0x0, 0x0}, 1},
- {0xE8, {0x65, 0x0, 0x0}, 1},
- {0xE9, {0x65, 0x0, 0x0}, 1},
- {0xEA, {0x65, 0x0, 0x0}, 1},
- {0xEB, {0x65, 0x0, 0x0}, 1},
- {0xEC, {0x69, 0x0, 0x0}, 1},
- {0xED, {0x69, 0x0, 0x0}, 1},
- {0xEE, {0x69, 0x0, 0x0}, 1},
- {0xEF, {0x69, 0x0, 0x0}, 1},
- {0xF1, {0x6E, 0x0, 0x0}, 1},
- {0xF2, {0x6F, 0x0, 0x0}, 1},
- {0xF3, {0x6F, 0x0, 0x0}, 1},
- {0xF4, {0x6F, 0x0, 0x0}, 1},
- {0xF5, {0x6F, 0x0, 0x0}, 1},
- {0xF6, {0x6F, 0x0, 0x0}, 1},
- {0xF9, {0x75, 0x0, 0x0}, 1},
- {0xFA, {0x75, 0x0, 0x0}, 1},
- {0xFB, {0x75, 0x0, 0x0}, 1},
- {0xFC, {0x75, 0x0, 0x0}, 1},
- {0xFD, {0x79, 0x0, 0x0}, 1},
- {0xFF, {0x79, 0x0, 0x0}, 1},
- {0x100, {0x61, 0x0, 0x0}, 1},
- {0x101, {0x61, 0x0, 0x0}, 1},
- {0x102, {0x61, 0x0, 0x0}, 1},
- {0x103, {0x61, 0x0, 0x0}, 1},
- {0x104, {0x61, 0x0, 0x0}, 1},
- {0x105, {0x61, 0x0, 0x0}, 1},
- {0x106, {0x63, 0x0, 0x0}, 1},
- {0x107, {0x63, 0x0, 0x0}, 1},
- {0x108, {0x63, 0x0, 0x0}, 1},
- {0x109, {0x63, 0x0, 0x0}, 1},
- {0x10A, {0x63, 0x0, 0x0}, 1},
- {0x10B, {0x63, 0x0, 0x0}, 1},
- {0x10C, {0x63, 0x0, 0x0}, 1},
- {0x10D, {0x63, 0x0, 0x0}, 1},
- {0x10E, {0x64, 0x0, 0x0}, 1},
- {0x10F, {0x64, 0x0, 0x0}, 1},
- {0x110, {0x111, 0x0, 0x0}, 1},
- {0x112, {0x65, 0x0, 0x0}, 1},
- {0x113, {0x65, 0x0, 0x0}, 1},
- {0x114, {0x65, 0x0, 0x0}, 1},
- {0x115, {0x65, 0x0, 0x0}, 1},
- {0x116, {0x65, 0x0, 0x0}, 1},
- {0x117, {0x65, 0x0, 0x0}, 1},
- {0x118, {0x65, 0x0, 0x0}, 1},
- {0x119, {0x65, 0x0, 0x0}, 1},
- {0x11A, {0x65, 0x0, 0x0}, 1},
- {0x11B, {0x65, 0x0, 0x0}, 1},
- {0x11C, {0x67, 0x0, 0x0}, 1},
- {0x11D, {0x67, 0x0, 0x0}, 1},
- {0x11E, {0x67, 0x0, 0x0}, 1},
- {0x11F, {0x67, 0x0, 0x0}, 1},
- {0x120, {0x67, 0x0, 0x0}, 1},
- {0x121, {0x67, 0x0, 0x0}, 1},
- {0x122, {0x67, 0x0, 0x0}, 1},
- {0x123, {0x67, 0x0, 0x0}, 1},
- {0x124, {0x68, 0x0, 0x0}, 1},
- {0x125, {0x68, 0x0, 0x0}, 1},
- {0x126, {0x127, 0x0, 0x0}, 1},
- {0x128, {0x69, 0x0, 0x0}, 1},
- {0x129, {0x69, 0x0, 0x0}, 1},
- {0x12A, {0x69, 0x0, 0x0}, 1},
- {0x12B, {0x69, 0x0, 0x0}, 1},
- {0x12C, {0x69, 0x0, 0x0}, 1},
- {0x12D, {0x69, 0x0, 0x0}, 1},
- {0x12E, {0x69, 0x0, 0x0}, 1},
- {0x12F, {0x69, 0x0, 0x0}, 1},
- {0x130, {0x69, 0x0, 0x0}, 1},
- {0x132, {0x133, 0x0, 0x0}, 1},
- {0x134, {0x6A, 0x0, 0x0}, 1},
- {0x135, {0x6A, 0x0, 0x0}, 1},
- {0x136, {0x6B, 0x0, 0x0}, 1},
- {0x137, {0x6B, 0x0, 0x0}, 1},
- {0x139, {0x6C, 0x0, 0x0}, 1},
- {0x13A, {0x6C, 0x0, 0x0}, 1},
- {0x13B, {0x6C, 0x0, 0x0}, 1},
- {0x13C, {0x6C, 0x0, 0x0}, 1},
- {0x13D, {0x6C, 0x0, 0x0}, 1},
- {0x13E, {0x6C, 0x0, 0x0}, 1},
- {0x13F, {0x140, 0x0, 0x0}, 1},
- {0x141, {0x142, 0x0, 0x0}, 1},
- {0x143, {0x6E, 0x0, 0x0}, 1},
- {0x144, {0x6E, 0x0, 0x0}, 1},
- {0x145, {0x6E, 0x0, 0x0}, 1},
- {0x146, {0x6E, 0x0, 0x0}, 1},
- {0x147, {0x6E, 0x0, 0x0}, 1},
- {0x148, {0x6E, 0x0, 0x0}, 1},
- {0x14A, {0x14B, 0x0, 0x0}, 1},
- {0x14C, {0x6F, 0x0, 0x0}, 1},
- {0x14D, {0x6F, 0x0, 0x0}, 1},
- {0x14E, {0x6F, 0x0, 0x0}, 1},
- {0x14F, {0x6F, 0x0, 0x0}, 1},
- {0x150, {0x6F, 0x0, 0x0}, 1},
- {0x151, {0x6F, 0x0, 0x0}, 1},
- {0x152, {0x153, 0x0, 0x0}, 1},
- {0x154, {0x72, 0x0, 0x0}, 1},
- {0x155, {0x72, 0x0, 0x0}, 1},
- {0x156, {0x72, 0x0, 0x0}, 1},
- {0x157, {0x72, 0x0, 0x0}, 1},
- {0x158, {0x72, 0x0, 0x0}, 1},
- {0x159, {0x72, 0x0, 0x0}, 1},
- {0x15A, {0x73, 0x0, 0x0}, 1},
- {0x15B, {0x73, 0x0, 0x0}, 1},
- {0x15C, {0x73, 0x0, 0x0}, 1},
- {0x15D, {0x73, 0x0, 0x0}, 1},
- {0x15E, {0x73, 0x0, 0x0}, 1},
- {0x15F, {0x73, 0x0, 0x0}, 1},
- {0x160, {0x73, 0x0, 0x0}, 1},
- {0x161, {0x73, 0x0, 0x0}, 1},
- {0x162, {0x74, 0x0, 0x0}, 1},
- {0x163, {0x74, 0x0, 0x0}, 1},
- {0x164, {0x74, 0x0, 0x0}, 1},
- {0x165, {0x74, 0x0, 0x0}, 1},
- {0x166, {0x167, 0x0, 0x0}, 1},
- {0x168, {0x75, 0x0, 0x0}, 1},
- {0x169, {0x75, 0x0, 0x0}, 1},
- {0x16A, {0x75, 0x0, 0x0}, 1},
- {0x16B, {0x75, 0x0, 0x0}, 1},
- {0x16C, {0x75, 0x0, 0x0}, 1},
- {0x16D, {0x75, 0x0, 0x0}, 1},
- {0x16E, {0x75, 0x0, 0x0}, 1},
- {0x16F, {0x75, 0x0, 0x0}, 1},
- {0x170, {0x75, 0x0, 0x0}, 1},
- {0x171, {0x75, 0x0, 0x0}, 1},
- {0x172, {0x75, 0x0, 0x0}, 1},
- {0x173, {0x75, 0x0, 0x0}, 1},
- {0x174, {0x77, 0x0, 0x0}, 1},
- {0x175, {0x77, 0x0, 0x0}, 1},
- {0x176, {0x79, 0x0, 0x0}, 1},
- {0x177, {0x79, 0x0, 0x0}, 1},
- {0x178, {0x79, 0x0, 0x0}, 1},
- {0x179, {0x7A, 0x0, 0x0}, 1},
- {0x17A, {0x7A, 0x0, 0x0}, 1},
- {0x17B, {0x7A, 0x0, 0x0}, 1},
- {0x17C, {0x7A, 0x0, 0x0}, 1},
- {0x17D, {0x7A, 0x0, 0x0}, 1},
- {0x17E, {0x7A, 0x0, 0x0}, 1},
- {0x181, {0x253, 0x0, 0x0}, 1},
- {0x182, {0x183, 0x0, 0x0}, 1},
- {0x184, {0x185, 0x0, 0x0}, 1},
- {0x186, {0x254, 0x0, 0x0}, 1},
- {0x187, {0x188, 0x0, 0x0}, 1},
- {0x189, {0x256, 0x0, 0x0}, 1},
- {0x18A, {0x257, 0x0, 0x0}, 1},
- {0x18B, {0x18C, 0x0, 0x0}, 1},
- {0x18E, {0x1DD, 0x0, 0x0}, 1},
- {0x18F, {0x259, 0x0, 0x0}, 1},
- {0x190, {0x25B, 0x0, 0x0}, 1},
- {0x191, {0x192, 0x0, 0x0}, 1},
- {0x193, {0x260, 0x0, 0x0}, 1},
- {0x194, {0x263, 0x0, 0x0}, 1},
- {0x196, {0x269, 0x0, 0x0}, 1},
- {0x197, {0x268, 0x0, 0x0}, 1},
- {0x198, {0x199, 0x0, 0x0}, 1},
- {0x19C, {0x26F, 0x0, 0x0}, 1},
- {0x19D, {0x272, 0x0, 0x0}, 1},
- {0x19F, {0x275, 0x0, 0x0}, 1},
- {0x1A0, {0x6F, 0x0, 0x0}, 1},
- {0x1A1, {0x6F, 0x0, 0x0}, 1},
- {0x1A2, {0x1A3, 0x0, 0x0}, 1},
- {0x1A4, {0x1A5, 0x0, 0x0}, 1},
- {0x1A6, {0x280, 0x0, 0x0}, 1},
- {0x1A7, {0x1A8, 0x0, 0x0}, 1},
- {0x1A9, {0x283, 0x0, 0x0}, 1},
- {0x1AC, {0x1AD, 0x0, 0x0}, 1},
- {0x1AE, {0x288, 0x0, 0x0}, 1},
- {0x1AF, {0x75, 0x0, 0x0}, 1},
- {0x1B0, {0x75, 0x0, 0x0}, 1},
- {0x1B1, {0x28A, 0x0, 0x0}, 1},
- {0x1B2, {0x28B, 0x0, 0x0}, 1},
- {0x1B3, {0x1B4, 0x0, 0x0}, 1},
- {0x1B5, {0x1B6, 0x0, 0x0}, 1},
- {0x1B7, {0x292, 0x0, 0x0}, 1},
- {0x1B8, {0x1B9, 0x0, 0x0}, 1},
- {0x1BC, {0x1BD, 0x0, 0x0}, 1},
- {0x1C4, {0x1C6, 0x0, 0x0}, 1},
- {0x1C5, {0x1C6, 0x0, 0x0}, 1},
- {0x1C7, {0x1C9, 0x0, 0x0}, 1},
- {0x1C8, {0x1C9, 0x0, 0x0}, 1},
- {0x1CA, {0x1CC, 0x0, 0x0}, 1},
- {0x1CB, {0x1CC, 0x0, 0x0}, 1},
- {0x1CD, {0x61, 0x0, 0x0}, 1},
- {0x1CE, {0x61, 0x0, 0x0}, 1},
- {0x1CF, {0x69, 0x0, 0x0}, 1},
- {0x1D0, {0x69, 0x0, 0x0}, 1},
- {0x1D1, {0x6F, 0x0, 0x0}, 1},
- {0x1D2, {0x6F, 0x0, 0x0}, 1},
- {0x1D3, {0x75, 0x0, 0x0}, 1},
- {0x1D4, {0x75, 0x0, 0x0}, 1},
- {0x1D5, {0x75, 0x0, 0x0}, 1},
- {0x1D6, {0x75, 0x0, 0x0}, 1},
- {0x1D7, {0x75, 0x0, 0x0}, 1},
- {0x1D8, {0x75, 0x0, 0x0}, 1},
- {0x1D9, {0x75, 0x0, 0x0}, 1},
- {0x1DA, {0x75, 0x0, 0x0}, 1},
- {0x1DB, {0x75, 0x0, 0x0}, 1},
- {0x1DC, {0x75, 0x0, 0x0}, 1},
- {0x1DE, {0x61, 0x0, 0x0}, 1},
- {0x1DF, {0x61, 0x0, 0x0}, 1},
- {0x1E0, {0x61, 0x0, 0x0}, 1},
- {0x1E1, {0x61, 0x0, 0x0}, 1},
- {0x1E2, {0xE6, 0x0, 0x0}, 1},
- {0x1E3, {0xE6, 0x0, 0x0}, 1},
- {0x1E4, {0x1E5, 0x0, 0x0}, 1},
- {0x1E6, {0x67, 0x0, 0x0}, 1},
- {0x1E7, {0x67, 0x0, 0x0}, 1},
- {0x1E8, {0x6B, 0x0, 0x0}, 1},
- {0x1E9, {0x6B, 0x0, 0x0}, 1},
- {0x1EA, {0x6F, 0x0, 0x0}, 1},
- {0x1EB, {0x6F, 0x0, 0x0}, 1},
- {0x1EC, {0x6F, 0x0, 0x0}, 1},
- {0x1ED, {0x6F, 0x0, 0x0}, 1},
- {0x1EE, {0x292, 0x0, 0x0}, 1},
- {0x1EF, {0x292, 0x0, 0x0}, 1},
- {0x1F0, {0x6A, 0x0, 0x0}, 1},
- {0x1F1, {0x1F3, 0x0, 0x0}, 1},
- {0x1F2, {0x1F3, 0x0, 0x0}, 1},
- {0x1F4, {0x67, 0x0, 0x0}, 1},
- {0x1F5, {0x67, 0x0, 0x0}, 1},
- {0x1F6, {0x195, 0x0, 0x0}, 1},
- {0x1F7, {0x1BF, 0x0, 0x0}, 1},
- {0x1F8, {0x6E, 0x0, 0x0}, 1},
- {0x1F9, {0x6E, 0x0, 0x0}, 1},
- {0x1FA, {0x61, 0x0, 0x0}, 1},
- {0x1FB, {0x61, 0x0, 0x0}, 1},
- {0x1FC, {0xE6, 0x0, 0x0}, 1},
- {0x1FD, {0xE6, 0x0, 0x0}, 1},
- {0x1FE, {0xF8, 0x0, 0x0}, 1},
- {0x1FF, {0xF8, 0x0, 0x0}, 1},
- {0x200, {0x61, 0x0, 0x0}, 1},
- {0x201, {0x61, 0x0, 0x0}, 1},
- {0x202, {0x61, 0x0, 0x0}, 1},
- {0x203, {0x61, 0x0, 0x0}, 1},
- {0x204, {0x65, 0x0, 0x0}, 1},
- {0x205, {0x65, 0x0, 0x0}, 1},
- {0x206, {0x65, 0x0, 0x0}, 1},
- {0x207, {0x65, 0x0, 0x0}, 1},
- {0x208, {0x69, 0x0, 0x0}, 1},
- {0x209, {0x69, 0x0, 0x0}, 1},
- {0x20A, {0x69, 0x0, 0x0}, 1},
- {0x20B, {0x69, 0x0, 0x0}, 1},
- {0x20C, {0x6F, 0x0, 0x0}, 1},
- {0x20D, {0x6F, 0x0, 0x0}, 1},
- {0x20E, {0x6F, 0x0, 0x0}, 1},
- {0x20F, {0x6F, 0x0, 0x0}, 1},
- {0x210, {0x72, 0x0, 0x0}, 1},
- {0x211, {0x72, 0x0, 0x0}, 1},
- {0x212, {0x72, 0x0, 0x0}, 1},
- {0x213, {0x72, 0x0, 0x0}, 1},
- {0x214, {0x75, 0x0, 0x0}, 1},
- {0x215, {0x75, 0x0, 0x0}, 1},
- {0x216, {0x75, 0x0, 0x0}, 1},
- {0x217, {0x75, 0x0, 0x0}, 1},
- {0x218, {0x73, 0x0, 0x0}, 1},
- {0x219, {0x73, 0x0, 0x0}, 1},
- {0x21A, {0x74, 0x0, 0x0}, 1},
- {0x21B, {0x74, 0x0, 0x0}, 1},
- {0x21C, {0x21D, 0x0, 0x0}, 1},
- {0x21E, {0x68, 0x0, 0x0}, 1},
- {0x21F, {0x68, 0x0, 0x0}, 1},
- {0x220, {0x19E, 0x0, 0x0}, 1},
- {0x222, {0x223, 0x0, 0x0}, 1},
- {0x224, {0x225, 0x0, 0x0}, 1},
- {0x226, {0x61, 0x0, 0x0}, 1},
- {0x227, {0x61, 0x0, 0x0}, 1},
- {0x228, {0x65, 0x0, 0x0}, 1},
- {0x229, {0x65, 0x0, 0x0}, 1},
- {0x22A, {0x6F, 0x0, 0x0}, 1},
- {0x22B, {0x6F, 0x0, 0x0}, 1},
- {0x22C, {0x6F, 0x0, 0x0}, 1},
- {0x22D, {0x6F, 0x0, 0x0}, 1},
- {0x22E, {0x6F, 0x0, 0x0}, 1},
- {0x22F, {0x6F, 0x0, 0x0}, 1},
- {0x230, {0x6F, 0x0, 0x0}, 1},
- {0x231, {0x6F, 0x0, 0x0}, 1},
- {0x232, {0x79, 0x0, 0x0}, 1},
- {0x233, {0x79, 0x0, 0x0}, 1},
- {0x23A, {0x2C65, 0x0, 0x0}, 1},
- {0x23B, {0x23C, 0x0, 0x0}, 1},
- {0x23D, {0x19A, 0x0, 0x0}, 1},
- {0x23E, {0x2C66, 0x0, 0x0}, 1},
- {0x241, {0x242, 0x0, 0x0}, 1},
- {0x243, {0x180, 0x0, 0x0}, 1},
- {0x244, {0x289, 0x0, 0x0}, 1},
- {0x245, {0x28C, 0x0, 0x0}, 1},
- {0x246, {0x247, 0x0, 0x0}, 1},
- {0x248, {0x249, 0x0, 0x0}, 1},
- {0x24A, {0x24B, 0x0, 0x0}, 1},
- {0x24C, {0x24D, 0x0, 0x0}, 1},
- {0x24E, {0x24F, 0x0, 0x0}, 1},
- {0x300, {0x0, 0x0, 0x0}, 0},
- {0x301, {0x0, 0x0, 0x0}, 0},
- {0x302, {0x0, 0x0, 0x0}, 0},
- {0x303, {0x0, 0x0, 0x0}, 0},
- {0x304, {0x0, 0x0, 0x0}, 0},
- {0x305, {0x0, 0x0, 0x0}, 0},
- {0x306, {0x0, 0x0, 0x0}, 0},
- {0x307, {0x0, 0x0, 0x0}, 0},
- {0x308, {0x0, 0x0, 0x0}, 0},
- {0x309, {0x0, 0x0, 0x0}, 0},
- {0x30A, {0x0, 0x0, 0x0}, 0},
- {0x30B, {0x0, 0x0, 0x0}, 0},
- {0x30C, {0x0, 0x0, 0x0}, 0},
- {0x30D, {0x0, 0x0, 0x0}, 0},
- {0x30E, {0x0, 0x0, 0x0}, 0},
- {0x30F, {0x0, 0x0, 0x0}, 0},
- {0x310, {0x0, 0x0, 0x0}, 0},
- {0x311, {0x0, 0x0, 0x0}, 0},
- {0x312, {0x0, 0x0, 0x0}, 0},
- {0x313, {0x0, 0x0, 0x0}, 0},
- {0x314, {0x0, 0x0, 0x0}, 0},
- {0x315, {0x0, 0x0, 0x0}, 0},
- {0x316, {0x0, 0x0, 0x0}, 0},
- {0x317, {0x0, 0x0, 0x0}, 0},
- {0x318, {0x0, 0x0, 0x0}, 0},
- {0x319, {0x0, 0x0, 0x0}, 0},
- {0x31A, {0x0, 0x0, 0x0}, 0},
- {0x31B, {0x0, 0x0, 0x0}, 0},
- {0x31C, {0x0, 0x0, 0x0}, 0},
- {0x31D, {0x0, 0x0, 0x0}, 0},
- {0x31E, {0x0, 0x0, 0x0}, 0},
- {0x31F, {0x0, 0x0, 0x0}, 0},
- {0x320, {0x0, 0x0, 0x0}, 0},
- {0x321, {0x0, 0x0, 0x0}, 0},
- {0x322, {0x0, 0x0, 0x0}, 0},
- {0x323, {0x0, 0x0, 0x0}, 0},
- {0x324, {0x0, 0x0, 0x0}, 0},
- {0x325, {0x0, 0x0, 0x0}, 0},
- {0x326, {0x0, 0x0, 0x0}, 0},
- {0x327, {0x0, 0x0, 0x0}, 0},
- {0x328, {0x0, 0x0, 0x0}, 0},
- {0x329, {0x0, 0x0, 0x0}, 0},
- {0x32A, {0x0, 0x0, 0x0}, 0},
- {0x32B, {0x0, 0x0, 0x0}, 0},
- {0x32C, {0x0, 0x0, 0x0}, 0},
- {0x32D, {0x0, 0x0, 0x0}, 0},
- {0x32E, {0x0, 0x0, 0x0}, 0},
- {0x32F, {0x0, 0x0, 0x0}, 0},
- {0x330, {0x0, 0x0, 0x0}, 0},
- {0x331, {0x0, 0x0, 0x0}, 0},
- {0x332, {0x0, 0x0, 0x0}, 0},
- {0x333, {0x0, 0x0, 0x0}, 0},
- {0x334, {0x0, 0x0, 0x0}, 0},
- {0x335, {0x0, 0x0, 0x0}, 0},
- {0x336, {0x0, 0x0, 0x0}, 0},
- {0x337, {0x0, 0x0, 0x0}, 0},
- {0x338, {0x0, 0x0, 0x0}, 0},
- {0x339, {0x0, 0x0, 0x0}, 0},
- {0x33A, {0x0, 0x0, 0x0}, 0},
- {0x33B, {0x0, 0x0, 0x0}, 0},
- {0x33C, {0x0, 0x0, 0x0}, 0},
- {0x33D, {0x0, 0x0, 0x0}, 0},
- {0x33E, {0x0, 0x0, 0x0}, 0},
- {0x33F, {0x0, 0x0, 0x0}, 0},
- {0x340, {0x0, 0x0, 0x0}, 0},
- {0x341, {0x0, 0x0, 0x0}, 0},
- {0x342, {0x0, 0x0, 0x0}, 0},
- {0x343, {0x0, 0x0, 0x0}, 0},
- {0x344, {0x0, 0x0, 0x0}, 0},
- {0x345, {0x0, 0x0, 0x0}, 0},
- {0x346, {0x0, 0x0, 0x0}, 0},
- {0x347, {0x0, 0x0, 0x0}, 0},
- {0x348, {0x0, 0x0, 0x0}, 0},
- {0x349, {0x0, 0x0, 0x0}, 0},
- {0x34A, {0x0, 0x0, 0x0}, 0},
- {0x34B, {0x0, 0x0, 0x0}, 0},
- {0x34C, {0x0, 0x0, 0x0}, 0},
- {0x34D, {0x0, 0x0, 0x0}, 0},
- {0x34E, {0x0, 0x0, 0x0}, 0},
- {0x34F, {0x0, 0x0, 0x0}, 0},
- {0x350, {0x0, 0x0, 0x0}, 0},
- {0x351, {0x0, 0x0, 0x0}, 0},
- {0x352, {0x0, 0x0, 0x0}, 0},
- {0x353, {0x0, 0x0, 0x0}, 0},
- {0x354, {0x0, 0x0, 0x0}, 0},
- {0x355, {0x0, 0x0, 0x0}, 0},
- {0x356, {0x0, 0x0, 0x0}, 0},
- {0x357, {0x0, 0x0, 0x0}, 0},
- {0x358, {0x0, 0x0, 0x0}, 0},
- {0x359, {0x0, 0x0, 0x0}, 0},
- {0x35A, {0x0, 0x0, 0x0}, 0},
- {0x35B, {0x0, 0x0, 0x0}, 0},
- {0x35C, {0x0, 0x0, 0x0}, 0},
- {0x35D, {0x0, 0x0, 0x0}, 0},
- {0x35E, {0x0, 0x0, 0x0}, 0},
- {0x35F, {0x0, 0x0, 0x0}, 0},
- {0x360, {0x0, 0x0, 0x0}, 0},
- {0x361, {0x0, 0x0, 0x0}, 0},
- {0x362, {0x0, 0x0, 0x0}, 0},
- {0x363, {0x0, 0x0, 0x0}, 0},
- {0x364, {0x0, 0x0, 0x0}, 0},
- {0x365, {0x0, 0x0, 0x0}, 0},
- {0x366, {0x0, 0x0, 0x0}, 0},
- {0x367, {0x0, 0x0, 0x0}, 0},
- {0x368, {0x0, 0x0, 0x0}, 0},
- {0x369, {0x0, 0x0, 0x0}, 0},
- {0x36A, {0x0, 0x0, 0x0}, 0},
- {0x36B, {0x0, 0x0, 0x0}, 0},
- {0x36C, {0x0, 0x0, 0x0}, 0},
- {0x36D, {0x0, 0x0, 0x0}, 0},
- {0x36E, {0x0, 0x0, 0x0}, 0},
- {0x36F, {0x0, 0x0, 0x0}, 0},
- {0x370, {0x371, 0x0, 0x0}, 1},
- {0x372, {0x373, 0x0, 0x0}, 1},
- {0x374, {0x2B9, 0x0, 0x0}, 1},
- {0x376, {0x377, 0x0, 0x0}, 1},
- {0x37E, {0x3B, 0x0, 0x0}, 1},
- {0x37F, {0x3F3, 0x0, 0x0}, 1},
- {0x385, {0xA8, 0x0, 0x0}, 1},
- {0x386, {0x3B1, 0x0, 0x0}, 1},
- {0x387, {0xB7, 0x0, 0x0}, 1},
- {0x388, {0x3B5, 0x0, 0x0}, 1},
- {0x389, {0x3B7, 0x0, 0x0}, 1},
- {0x38A, {0x3B9, 0x0, 0x0}, 1},
- {0x38C, {0x3BF, 0x0, 0x0}, 1},
- {0x38E, {0x3C5, 0x0, 0x0}, 1},
- {0x38F, {0x3C9, 0x0, 0x0}, 1},
- {0x390, {0x3B9, 0x0, 0x0}, 1},
- {0x391, {0x3B1, 0x0, 0x0}, 1},
- {0x392, {0x3B2, 0x0, 0x0}, 1},
- {0x393, {0x3B3, 0x0, 0x0}, 1},
- {0x394, {0x3B4, 0x0, 0x0}, 1},
- {0x395, {0x3B5, 0x0, 0x0}, 1},
- {0x396, {0x3B6, 0x0, 0x0}, 1},
- {0x397, {0x3B7, 0x0, 0x0}, 1},
- {0x398, {0x3B8, 0x0, 0x0}, 1},
- {0x399, {0x3B9, 0x0, 0x0}, 1},
- {0x39A, {0x3BA, 0x0, 0x0}, 1},
- {0x39B, {0x3BB, 0x0, 0x0}, 1},
- {0x39C, {0x3BC, 0x0, 0x0}, 1},
- {0x39D, {0x3BD, 0x0, 0x0}, 1},
- {0x39E, {0x3BE, 0x0, 0x0}, 1},
- {0x39F, {0x3BF, 0x0, 0x0}, 1},
- {0x3A0, {0x3C0, 0x0, 0x0}, 1},
- {0x3A1, {0x3C1, 0x0, 0x0}, 1},
- {0x3A3, {0x3C3, 0x0, 0x0}, 1},
- {0x3A4, {0x3C4, 0x0, 0x0}, 1},
- {0x3A5, {0x3C5, 0x0, 0x0}, 1},
- {0x3A6, {0x3C6, 0x0, 0x0}, 1},
- {0x3A7, {0x3C7, 0x0, 0x0}, 1},
- {0x3A8, {0x3C8, 0x0, 0x0}, 1},
- {0x3A9, {0x3C9, 0x0, 0x0}, 1},
- {0x3AA, {0x3B9, 0x0, 0x0}, 1},
- {0x3AB, {0x3C5, 0x0, 0x0}, 1},
- {0x3AC, {0x3B1, 0x0, 0x0}, 1},
- {0x3AD, {0x3B5, 0x0, 0x0}, 1},
- {0x3AE, {0x3B7, 0x0, 0x0}, 1},
- {0x3AF, {0x3B9, 0x0, 0x0}, 1},
- {0x3B0, {0x3C5, 0x0, 0x0}, 1},
- {0x3CA, {0x3B9, 0x0, 0x0}, 1},
- {0x3CB, {0x3C5, 0x0, 0x0}, 1},
- {0x3CC, {0x3BF, 0x0, 0x0}, 1},
- {0x3CD, {0x3C5, 0x0, 0x0}, 1},
- {0x3CE, {0x3C9, 0x0, 0x0}, 1},
- {0x3CF, {0x3D7, 0x0, 0x0}, 1},
- {0x3D3, {0x3D2, 0x0, 0x0}, 1},
- {0x3D4, {0x3D2, 0x0, 0x0}, 1},
- {0x3D8, {0x3D9, 0x0, 0x0}, 1},
- {0x3DA, {0x3DB, 0x0, 0x0}, 1},
- {0x3DC, {0x3DD, 0x0, 0x0}, 1},
- {0x3DE, {0x3DF, 0x0, 0x0}, 1},
- {0x3E0, {0x3E1, 0x0, 0x0}, 1},
- {0x3E2, {0x3E3, 0x0, 0x0}, 1},
- {0x3E4, {0x3E5, 0x0, 0x0}, 1},
- {0x3E6, {0x3E7, 0x0, 0x0}, 1},
- {0x3E8, {0x3E9, 0x0, 0x0}, 1},
- {0x3EA, {0x3EB, 0x0, 0x0}, 1},
- {0x3EC, {0x3ED, 0x0, 0x0}, 1},
- {0x3EE, {0x3EF, 0x0, 0x0}, 1},
- {0x3F4, {0x3B8, 0x0, 0x0}, 1},
- {0x3F7, {0x3F8, 0x0, 0x0}, 1},
- {0x3F9, {0x3F2, 0x0, 0x0}, 1},
- {0x3FA, {0x3FB, 0x0, 0x0}, 1},
- {0x3FD, {0x37B, 0x0, 0x0}, 1},
- {0x3FE, {0x37C, 0x0, 0x0}, 1},
- {0x3FF, {0x37D, 0x0, 0x0}, 1},
- {0x400, {0x435, 0x0, 0x0}, 1},
- {0x401, {0x435, 0x0, 0x0}, 1},
- {0x402, {0x452, 0x0, 0x0}, 1},
- {0x403, {0x433, 0x0, 0x0}, 1},
- {0x404, {0x454, 0x0, 0x0}, 1},
- {0x405, {0x455, 0x0, 0x0}, 1},
- {0x406, {0x456, 0x0, 0x0}, 1},
- {0x407, {0x456, 0x0, 0x0}, 1},
- {0x408, {0x458, 0x0, 0x0}, 1},
- {0x409, {0x459, 0x0, 0x0}, 1},
- {0x40A, {0x45A, 0x0, 0x0}, 1},
- {0x40B, {0x45B, 0x0, 0x0}, 1},
- {0x40C, {0x43A, 0x0, 0x0}, 1},
- {0x40D, {0x438, 0x0, 0x0}, 1},
- {0x40E, {0x443, 0x0, 0x0}, 1},
- {0x40F, {0x45F, 0x0, 0x0}, 1},
- {0x410, {0x430, 0x0, 0x0}, 1},
- {0x411, {0x431, 0x0, 0x0}, 1},
- {0x412, {0x432, 0x0, 0x0}, 1},
- {0x413, {0x433, 0x0, 0x0}, 1},
- {0x414, {0x434, 0x0, 0x0}, 1},
- {0x415, {0x435, 0x0, 0x0}, 1},
- {0x416, {0x436, 0x0, 0x0}, 1},
- {0x417, {0x437, 0x0, 0x0}, 1},
- {0x418, {0x438, 0x0, 0x0}, 1},
- {0x419, {0x438, 0x0, 0x0}, 1},
- {0x41A, {0x43A, 0x0, 0x0}, 1},
- {0x41B, {0x43B, 0x0, 0x0}, 1},
- {0x41C, {0x43C, 0x0, 0x0}, 1},
- {0x41D, {0x43D, 0x0, 0x0}, 1},
- {0x41E, {0x43E, 0x0, 0x0}, 1},
- {0x41F, {0x43F, 0x0, 0x0}, 1},
- {0x420, {0x440, 0x0, 0x0}, 1},
- {0x421, {0x441, 0x0, 0x0}, 1},
- {0x422, {0x442, 0x0, 0x0}, 1},
- {0x423, {0x443, 0x0, 0x0}, 1},
- {0x424, {0x444, 0x0, 0x0}, 1},
- {0x425, {0x445, 0x0, 0x0}, 1},
- {0x426, {0x446, 0x0, 0x0}, 1},
- {0x427, {0x447, 0x0, 0x0}, 1},
- {0x428, {0x448, 0x0, 0x0}, 1},
- {0x429, {0x449, 0x0, 0x0}, 1},
- {0x42A, {0x44A, 0x0, 0x0}, 1},
- {0x42B, {0x44B, 0x0, 0x0}, 1},
- {0x42C, {0x44C, 0x0, 0x0}, 1},
- {0x42D, {0x44D, 0x0, 0x0}, 1},
- {0x42E, {0x44E, 0x0, 0x0}, 1},
- {0x42F, {0x44F, 0x0, 0x0}, 1},
- {0x439, {0x438, 0x0, 0x0}, 1},
- {0x450, {0x435, 0x0, 0x0}, 1},
- {0x451, {0x435, 0x0, 0x0}, 1},
- {0x453, {0x433, 0x0, 0x0}, 1},
- {0x457, {0x456, 0x0, 0x0}, 1},
- {0x45C, {0x43A, 0x0, 0x0}, 1},
- {0x45D, {0x438, 0x0, 0x0}, 1},
- {0x45E, {0x443, 0x0, 0x0}, 1},
- {0x460, {0x461, 0x0, 0x0}, 1},
- {0x462, {0x463, 0x0, 0x0}, 1},
- {0x464, {0x465, 0x0, 0x0}, 1},
- {0x466, {0x467, 0x0, 0x0}, 1},
- {0x468, {0x469, 0x0, 0x0}, 1},
- {0x46A, {0x46B, 0x0, 0x0}, 1},
- {0x46C, {0x46D, 0x0, 0x0}, 1},
- {0x46E, {0x46F, 0x0, 0x0}, 1},
- {0x470, {0x471, 0x0, 0x0}, 1},
- {0x472, {0x473, 0x0, 0x0}, 1},
- {0x474, {0x475, 0x0, 0x0}, 1},
- {0x476, {0x475, 0x0, 0x0}, 1},
- {0x477, {0x475, 0x0, 0x0}, 1},
- {0x478, {0x479, 0x0, 0x0}, 1},
- {0x47A, {0x47B, 0x0, 0x0}, 1},
- {0x47C, {0x47D, 0x0, 0x0}, 1},
- {0x47E, {0x47F, 0x0, 0x0}, 1},
- {0x480, {0x481, 0x0, 0x0}, 1},
- {0x483, {0x0, 0x0, 0x0}, 0},
- {0x484, {0x0, 0x0, 0x0}, 0},
- {0x485, {0x0, 0x0, 0x0}, 0},
- {0x486, {0x0, 0x0, 0x0}, 0},
- {0x487, {0x0, 0x0, 0x0}, 0},
- {0x48A, {0x48B, 0x0, 0x0}, 1},
- {0x48C, {0x48D, 0x0, 0x0}, 1},
- {0x48E, {0x48F, 0x0, 0x0}, 1},
- {0x490, {0x491, 0x0, 0x0}, 1},
- {0x492, {0x493, 0x0, 0x0}, 1},
- {0x494, {0x495, 0x0, 0x0}, 1},
- {0x496, {0x497, 0x0, 0x0}, 1},
- {0x498, {0x499, 0x0, 0x0}, 1},
- {0x49A, {0x49B, 0x0, 0x0}, 1},
- {0x49C, {0x49D, 0x0, 0x0}, 1},
- {0x49E, {0x49F, 0x0, 0x0}, 1},
- {0x4A0, {0x4A1, 0x0, 0x0}, 1},
- {0x4A2, {0x4A3, 0x0, 0x0}, 1},
- {0x4A4, {0x4A5, 0x0, 0x0}, 1},
- {0x4A6, {0x4A7, 0x0, 0x0}, 1},
- {0x4A8, {0x4A9, 0x0, 0x0}, 1},
- {0x4AA, {0x4AB, 0x0, 0x0}, 1},
- {0x4AC, {0x4AD, 0x0, 0x0}, 1},
- {0x4AE, {0x4AF, 0x0, 0x0}, 1},
- {0x4B0, {0x4B1, 0x0, 0x0}, 1},
- {0x4B2, {0x4B3, 0x0, 0x0}, 1},
- {0x4B4, {0x4B5, 0x0, 0x0}, 1},
- {0x4B6, {0x4B7, 0x0, 0x0}, 1},
- {0x4B8, {0x4B9, 0x0, 0x0}, 1},
- {0x4BA, {0x4BB, 0x0, 0x0}, 1},
- {0x4BC, {0x4BD, 0x0, 0x0}, 1},
- {0x4BE, {0x4BF, 0x0, 0x0}, 1},
- {0x4C0, {0x4CF, 0x0, 0x0}, 1},
- {0x4C1, {0x436, 0x0, 0x0}, 1},
- {0x4C2, {0x436, 0x0, 0x0}, 1},
- {0x4C3, {0x4C4, 0x0, 0x0}, 1},
- {0x4C5, {0x4C6, 0x0, 0x0}, 1},
- {0x4C7, {0x4C8, 0x0, 0x0}, 1},
- {0x4C9, {0x4CA, 0x0, 0x0}, 1},
- {0x4CB, {0x4CC, 0x0, 0x0}, 1},
- {0x4CD, {0x4CE, 0x0, 0x0}, 1},
- {0x4D0, {0x430, 0x0, 0x0}, 1},
- {0x4D1, {0x430, 0x0, 0x0}, 1},
- {0x4D2, {0x430, 0x0, 0x0}, 1},
- {0x4D3, {0x430, 0x0, 0x0}, 1},
- {0x4D4, {0x4D5, 0x0, 0x0}, 1},
- {0x4D6, {0x435, 0x0, 0x0}, 1},
- {0x4D7, {0x435, 0x0, 0x0}, 1},
- {0x4D8, {0x4D9, 0x0, 0x0}, 1},
- {0x4DA, {0x4D9, 0x0, 0x0}, 1},
- {0x4DB, {0x4D9, 0x0, 0x0}, 1},
- {0x4DC, {0x436, 0x0, 0x0}, 1},
- {0x4DD, {0x436, 0x0, 0x0}, 1},
- {0x4DE, {0x437, 0x0, 0x0}, 1},
- {0x4DF, {0x437, 0x0, 0x0}, 1},
- {0x4E0, {0x4E1, 0x0, 0x0}, 1},
- {0x4E2, {0x438, 0x0, 0x0}, 1},
- {0x4E3, {0x438, 0x0, 0x0}, 1},
- {0x4E4, {0x438, 0x0, 0x0}, 1},
- {0x4E5, {0x438, 0x0, 0x0}, 1},
- {0x4E6, {0x43E, 0x0, 0x0}, 1},
- {0x4E7, {0x43E, 0x0, 0x0}, 1},
- {0x4E8, {0x4E9, 0x0, 0x0}, 1},
- {0x4EA, {0x4E9, 0x0, 0x0}, 1},
- {0x4EB, {0x4E9, 0x0, 0x0}, 1},
- {0x4EC, {0x44D, 0x0, 0x0}, 1},
- {0x4ED, {0x44D, 0x0, 0x0}, 1},
- {0x4EE, {0x443, 0x0, 0x0}, 1},
- {0x4EF, {0x443, 0x0, 0x0}, 1},
- {0x4F0, {0x443, 0x0, 0x0}, 1},
- {0x4F1, {0x443, 0x0, 0x0}, 1},
- {0x4F2, {0x443, 0x0, 0x0}, 1},
- {0x4F3, {0x443, 0x0, 0x0}, 1},
- {0x4F4, {0x447, 0x0, 0x0}, 1},
- {0x4F5, {0x447, 0x0, 0x0}, 1},
- {0x4F6, {0x4F7, 0x0, 0x0}, 1},
- {0x4F8, {0x44B, 0x0, 0x0}, 1},
- {0x4F9, {0x44B, 0x0, 0x0}, 1},
- {0x4FA, {0x4FB, 0x0, 0x0}, 1},
- {0x4FC, {0x4FD, 0x0, 0x0}, 1},
- {0x4FE, {0x4FF, 0x0, 0x0}, 1},
- {0x500, {0x501, 0x0, 0x0}, 1},
- {0x502, {0x503, 0x0, 0x0}, 1},
- {0x504, {0x505, 0x0, 0x0}, 1},
- {0x506, {0x507, 0x0, 0x0}, 1},
- {0x508, {0x509, 0x0, 0x0}, 1},
- {0x50A, {0x50B, 0x0, 0x0}, 1},
- {0x50C, {0x50D, 0x0, 0x0}, 1},
- {0x50E, {0x50F, 0x0, 0x0}, 1},
- {0x510, {0x511, 0x0, 0x0}, 1},
- {0x512, {0x513, 0x0, 0x0}, 1},
- {0x514, {0x515, 0x0, 0x0}, 1},
- {0x516, {0x517, 0x0, 0x0}, 1},
- {0x518, {0x519, 0x0, 0x0}, 1},
- {0x51A, {0x51B, 0x0, 0x0}, 1},
- {0x51C, {0x51D, 0x0, 0x0}, 1},
- {0x51E, {0x51F, 0x0, 0x0}, 1},
- {0x520, {0x521, 0x0, 0x0}, 1},
- {0x522, {0x523, 0x0, 0x0}, 1},
- {0x524, {0x525, 0x0, 0x0}, 1},
- {0x526, {0x527, 0x0, 0x0}, 1},
- {0x528, {0x529, 0x0, 0x0}, 1},
- {0x52A, {0x52B, 0x0, 0x0}, 1},
- {0x52C, {0x52D, 0x0, 0x0}, 1},
- {0x52E, {0x52F, 0x0, 0x0}, 1},
- {0x531, {0x561, 0x0, 0x0}, 1},
- {0x532, {0x562, 0x0, 0x0}, 1},
- {0x533, {0x563, 0x0, 0x0}, 1},
- {0x534, {0x564, 0x0, 0x0}, 1},
- {0x535, {0x565, 0x0, 0x0}, 1},
- {0x536, {0x566, 0x0, 0x0}, 1},
- {0x537, {0x567, 0x0, 0x0}, 1},
- {0x538, {0x568, 0x0, 0x0}, 1},
- {0x539, {0x569, 0x0, 0x0}, 1},
- {0x53A, {0x56A, 0x0, 0x0}, 1},
- {0x53B, {0x56B, 0x0, 0x0}, 1},
- {0x53C, {0x56C, 0x0, 0x0}, 1},
- {0x53D, {0x56D, 0x0, 0x0}, 1},
- {0x53E, {0x56E, 0x0, 0x0}, 1},
- {0x53F, {0x56F, 0x0, 0x0}, 1},
- {0x540, {0x570, 0x0, 0x0}, 1},
- {0x541, {0x571, 0x0, 0x0}, 1},
- {0x542, {0x572, 0x0, 0x0}, 1},
- {0x543, {0x573, 0x0, 0x0}, 1},
- {0x544, {0x574, 0x0, 0x0}, 1},
- {0x545, {0x575, 0x0, 0x0}, 1},
- {0x546, {0x576, 0x0, 0x0}, 1},
- {0x547, {0x577, 0x0, 0x0}, 1},
- {0x548, {0x578, 0x0, 0x0}, 1},
- {0x549, {0x579, 0x0, 0x0}, 1},
- {0x54A, {0x57A, 0x0, 0x0}, 1},
- {0x54B, {0x57B, 0x0, 0x0}, 1},
- {0x54C, {0x57C, 0x0, 0x0}, 1},
- {0x54D, {0x57D, 0x0, 0x0}, 1},
- {0x54E, {0x57E, 0x0, 0x0}, 1},
- {0x54F, {0x57F, 0x0, 0x0}, 1},
- {0x550, {0x580, 0x0, 0x0}, 1},
- {0x551, {0x581, 0x0, 0x0}, 1},
- {0x552, {0x582, 0x0, 0x0}, 1},
- {0x553, {0x583, 0x0, 0x0}, 1},
- {0x554, {0x584, 0x0, 0x0}, 1},
- {0x555, {0x585, 0x0, 0x0}, 1},
- {0x556, {0x586, 0x0, 0x0}, 1},
- {0x591, {0x0, 0x0, 0x0}, 0},
- {0x592, {0x0, 0x0, 0x0}, 0},
- {0x593, {0x0, 0x0, 0x0}, 0},
- {0x594, {0x0, 0x0, 0x0}, 0},
- {0x595, {0x0, 0x0, 0x0}, 0},
- {0x596, {0x0, 0x0, 0x0}, 0},
- {0x597, {0x0, 0x0, 0x0}, 0},
- {0x598, {0x0, 0x0, 0x0}, 0},
- {0x599, {0x0, 0x0, 0x0}, 0},
- {0x59A, {0x0, 0x0, 0x0}, 0},
- {0x59B, {0x0, 0x0, 0x0}, 0},
- {0x59C, {0x0, 0x0, 0x0}, 0},
- {0x59D, {0x0, 0x0, 0x0}, 0},
- {0x59E, {0x0, 0x0, 0x0}, 0},
- {0x59F, {0x0, 0x0, 0x0}, 0},
- {0x5A0, {0x0, 0x0, 0x0}, 0},
- {0x5A1, {0x0, 0x0, 0x0}, 0},
- {0x5A2, {0x0, 0x0, 0x0}, 0},
- {0x5A3, {0x0, 0x0, 0x0}, 0},
- {0x5A4, {0x0, 0x0, 0x0}, 0},
- {0x5A5, {0x0, 0x0, 0x0}, 0},
- {0x5A6, {0x0, 0x0, 0x0}, 0},
- {0x5A7, {0x0, 0x0, 0x0}, 0},
- {0x5A8, {0x0, 0x0, 0x0}, 0},
- {0x5A9, {0x0, 0x0, 0x0}, 0},
- {0x5AA, {0x0, 0x0, 0x0}, 0},
- {0x5AB, {0x0, 0x0, 0x0}, 0},
- {0x5AC, {0x0, 0x0, 0x0}, 0},
- {0x5AD, {0x0, 0x0, 0x0}, 0},
- {0x5AE, {0x0, 0x0, 0x0}, 0},
- {0x5AF, {0x0, 0x0, 0x0}, 0},
- {0x5B0, {0x0, 0x0, 0x0}, 0},
- {0x5B1, {0x0, 0x0, 0x0}, 0},
- {0x5B2, {0x0, 0x0, 0x0}, 0},
- {0x5B3, {0x0, 0x0, 0x0}, 0},
- {0x5B4, {0x0, 0x0, 0x0}, 0},
- {0x5B5, {0x0, 0x0, 0x0}, 0},
- {0x5B6, {0x0, 0x0, 0x0}, 0},
- {0x5B7, {0x0, 0x0, 0x0}, 0},
- {0x5B8, {0x0, 0x0, 0x0}, 0},
- {0x5B9, {0x0, 0x0, 0x0}, 0},
- {0x5BA, {0x0, 0x0, 0x0}, 0},
- {0x5BB, {0x0, 0x0, 0x0}, 0},
- {0x5BC, {0x0, 0x0, 0x0}, 0},
- {0x5BD, {0x0, 0x0, 0x0}, 0},
- {0x5BF, {0x0, 0x0, 0x0}, 0},
- {0x5C1, {0x0, 0x0, 0x0}, 0},
- {0x5C2, {0x0, 0x0, 0x0}, 0},
- {0x5C4, {0x0, 0x0, 0x0}, 0},
- {0x5C5, {0x0, 0x0, 0x0}, 0},
- {0x5C7, {0x0, 0x0, 0x0}, 0},
- {0x610, {0x0, 0x0, 0x0}, 0},
- {0x611, {0x0, 0x0, 0x0}, 0},
- {0x612, {0x0, 0x0, 0x0}, 0},
- {0x613, {0x0, 0x0, 0x0}, 0},
- {0x614, {0x0, 0x0, 0x0}, 0},
- {0x615, {0x0, 0x0, 0x0}, 0},
- {0x616, {0x0, 0x0, 0x0}, 0},
- {0x617, {0x0, 0x0, 0x0}, 0},
- {0x618, {0x0, 0x0, 0x0}, 0},
- {0x619, {0x0, 0x0, 0x0}, 0},
- {0x61A, {0x0, 0x0, 0x0}, 0},
- {0x622, {0x627, 0x0, 0x0}, 1},
- {0x623, {0x627, 0x0, 0x0}, 1},
- {0x624, {0x648, 0x0, 0x0}, 1},
- {0x625, {0x627, 0x0, 0x0}, 1},
- {0x626, {0x64A, 0x0, 0x0}, 1},
- {0x64B, {0x0, 0x0, 0x0}, 0},
- {0x64C, {0x0, 0x0, 0x0}, 0},
- {0x64D, {0x0, 0x0, 0x0}, 0},
- {0x64E, {0x0, 0x0, 0x0}, 0},
- {0x64F, {0x0, 0x0, 0x0}, 0},
- {0x650, {0x0, 0x0, 0x0}, 0},
- {0x651, {0x0, 0x0, 0x0}, 0},
- {0x652, {0x0, 0x0, 0x0}, 0},
- {0x653, {0x0, 0x0, 0x0}, 0},
- {0x654, {0x0, 0x0, 0x0}, 0},
- {0x655, {0x0, 0x0, 0x0}, 0},
- {0x656, {0x0, 0x0, 0x0}, 0},
- {0x657, {0x0, 0x0, 0x0}, 0},
- {0x658, {0x0, 0x0, 0x0}, 0},
- {0x659, {0x0, 0x0, 0x0}, 0},
- {0x65A, {0x0, 0x0, 0x0}, 0},
- {0x65B, {0x0, 0x0, 0x0}, 0},
- {0x65C, {0x0, 0x0, 0x0}, 0},
- {0x65D, {0x0, 0x0, 0x0}, 0},
- {0x65E, {0x0, 0x0, 0x0}, 0},
- {0x65F, {0x0, 0x0, 0x0}, 0},
- {0x670, {0x0, 0x0, 0x0}, 0},
- {0x6C0, {0x6D5, 0x0, 0x0}, 1},
- {0x6C2, {0x6C1, 0x0, 0x0}, 1},
- {0x6D3, {0x6D2, 0x0, 0x0}, 1},
- {0x6D6, {0x0, 0x0, 0x0}, 0},
- {0x6D7, {0x0, 0x0, 0x0}, 0},
- {0x6D8, {0x0, 0x0, 0x0}, 0},
- {0x6D9, {0x0, 0x0, 0x0}, 0},
- {0x6DA, {0x0, 0x0, 0x0}, 0},
- {0x6DB, {0x0, 0x0, 0x0}, 0},
- {0x6DC, {0x0, 0x0, 0x0}, 0},
- {0x6DF, {0x0, 0x0, 0x0}, 0},
- {0x6E0, {0x0, 0x0, 0x0}, 0},
- {0x6E1, {0x0, 0x0, 0x0}, 0},
- {0x6E2, {0x0, 0x0, 0x0}, 0},
- {0x6E3, {0x0, 0x0, 0x0}, 0},
- {0x6E4, {0x0, 0x0, 0x0}, 0},
- {0x6E7, {0x0, 0x0, 0x0}, 0},
- {0x6E8, {0x0, 0x0, 0x0}, 0},
- {0x6EA, {0x0, 0x0, 0x0}, 0},
- {0x6EB, {0x0, 0x0, 0x0}, 0},
- {0x6EC, {0x0, 0x0, 0x0}, 0},
- {0x6ED, {0x0, 0x0, 0x0}, 0},
- {0x711, {0x0, 0x0, 0x0}, 0},
- {0x730, {0x0, 0x0, 0x0}, 0},
- {0x731, {0x0, 0x0, 0x0}, 0},
- {0x732, {0x0, 0x0, 0x0}, 0},
- {0x733, {0x0, 0x0, 0x0}, 0},
- {0x734, {0x0, 0x0, 0x0}, 0},
- {0x735, {0x0, 0x0, 0x0}, 0},
- {0x736, {0x0, 0x0, 0x0}, 0},
- {0x737, {0x0, 0x0, 0x0}, 0},
- {0x738, {0x0, 0x0, 0x0}, 0},
- {0x739, {0x0, 0x0, 0x0}, 0},
- {0x73A, {0x0, 0x0, 0x0}, 0},
- {0x73B, {0x0, 0x0, 0x0}, 0},
- {0x73C, {0x0, 0x0, 0x0}, 0},
- {0x73D, {0x0, 0x0, 0x0}, 0},
- {0x73E, {0x0, 0x0, 0x0}, 0},
- {0x73F, {0x0, 0x0, 0x0}, 0},
- {0x740, {0x0, 0x0, 0x0}, 0},
- {0x741, {0x0, 0x0, 0x0}, 0},
- {0x742, {0x0, 0x0, 0x0}, 0},
- {0x743, {0x0, 0x0, 0x0}, 0},
- {0x744, {0x0, 0x0, 0x0}, 0},
- {0x745, {0x0, 0x0, 0x0}, 0},
- {0x746, {0x0, 0x0, 0x0}, 0},
- {0x747, {0x0, 0x0, 0x0}, 0},
- {0x748, {0x0, 0x0, 0x0}, 0},
- {0x749, {0x0, 0x0, 0x0}, 0},
- {0x74A, {0x0, 0x0, 0x0}, 0},
- {0x7A6, {0x0, 0x0, 0x0}, 0},
- {0x7A7, {0x0, 0x0, 0x0}, 0},
- {0x7A8, {0x0, 0x0, 0x0}, 0},
- {0x7A9, {0x0, 0x0, 0x0}, 0},
- {0x7AA, {0x0, 0x0, 0x0}, 0},
- {0x7AB, {0x0, 0x0, 0x0}, 0},
- {0x7AC, {0x0, 0x0, 0x0}, 0},
- {0x7AD, {0x0, 0x0, 0x0}, 0},
- {0x7AE, {0x0, 0x0, 0x0}, 0},
- {0x7AF, {0x0, 0x0, 0x0}, 0},
- {0x7B0, {0x0, 0x0, 0x0}, 0},
- {0x7EB, {0x0, 0x0, 0x0}, 0},
- {0x7EC, {0x0, 0x0, 0x0}, 0},
- {0x7ED, {0x0, 0x0, 0x0}, 0},
- {0x7EE, {0x0, 0x0, 0x0}, 0},
- {0x7EF, {0x0, 0x0, 0x0}, 0},
- {0x7F0, {0x0, 0x0, 0x0}, 0},
- {0x7F1, {0x0, 0x0, 0x0}, 0},
- {0x7F2, {0x0, 0x0, 0x0}, 0},
- {0x7F3, {0x0, 0x0, 0x0}, 0},
- {0x7FD, {0x0, 0x0, 0x0}, 0},
- {0x816, {0x0, 0x0, 0x0}, 0},
- {0x817, {0x0, 0x0, 0x0}, 0},
- {0x818, {0x0, 0x0, 0x0}, 0},
- {0x819, {0x0, 0x0, 0x0}, 0},
- {0x81B, {0x0, 0x0, 0x0}, 0},
- {0x81C, {0x0, 0x0, 0x0}, 0},
- {0x81D, {0x0, 0x0, 0x0}, 0},
- {0x81E, {0x0, 0x0, 0x0}, 0},
- {0x81F, {0x0, 0x0, 0x0}, 0},
- {0x820, {0x0, 0x0, 0x0}, 0},
- {0x821, {0x0, 0x0, 0x0}, 0},
- {0x822, {0x0, 0x0, 0x0}, 0},
- {0x823, {0x0, 0x0, 0x0}, 0},
- {0x825, {0x0, 0x0, 0x0}, 0},
- {0x826, {0x0, 0x0, 0x0}, 0},
- {0x827, {0x0, 0x0, 0x0}, 0},
- {0x829, {0x0, 0x0, 0x0}, 0},
- {0x82A, {0x0, 0x0, 0x0}, 0},
- {0x82B, {0x0, 0x0, 0x0}, 0},
- {0x82C, {0x0, 0x0, 0x0}, 0},
- {0x82D, {0x0, 0x0, 0x0}, 0},
- {0x859, {0x0, 0x0, 0x0}, 0},
- {0x85A, {0x0, 0x0, 0x0}, 0},
- {0x85B, {0x0, 0x0, 0x0}, 0},
- {0x8D3, {0x0, 0x0, 0x0}, 0},
- {0x8D4, {0x0, 0x0, 0x0}, 0},
- {0x8D5, {0x0, 0x0, 0x0}, 0},
- {0x8D6, {0x0, 0x0, 0x0}, 0},
- {0x8D7, {0x0, 0x0, 0x0}, 0},
- {0x8D8, {0x0, 0x0, 0x0}, 0},
- {0x8D9, {0x0, 0x0, 0x0}, 0},
- {0x8DA, {0x0, 0x0, 0x0}, 0},
- {0x8DB, {0x0, 0x0, 0x0}, 0},
- {0x8DC, {0x0, 0x0, 0x0}, 0},
- {0x8DD, {0x0, 0x0, 0x0}, 0},
- {0x8DE, {0x0, 0x0, 0x0}, 0},
- {0x8DF, {0x0, 0x0, 0x0}, 0},
- {0x8E0, {0x0, 0x0, 0x0}, 0},
- {0x8E1, {0x0, 0x0, 0x0}, 0},
- {0x8E3, {0x0, 0x0, 0x0}, 0},
- {0x8E4, {0x0, 0x0, 0x0}, 0},
- {0x8E5, {0x0, 0x0, 0x0}, 0},
- {0x8E6, {0x0, 0x0, 0x0}, 0},
- {0x8E7, {0x0, 0x0, 0x0}, 0},
- {0x8E8, {0x0, 0x0, 0x0}, 0},
- {0x8E9, {0x0, 0x0, 0x0}, 0},
- {0x8EA, {0x0, 0x0, 0x0}, 0},
- {0x8EB, {0x0, 0x0, 0x0}, 0},
- {0x8EC, {0x0, 0x0, 0x0}, 0},
- {0x8ED, {0x0, 0x0, 0x0}, 0},
- {0x8EE, {0x0, 0x0, 0x0}, 0},
- {0x8EF, {0x0, 0x0, 0x0}, 0},
- {0x8F0, {0x0, 0x0, 0x0}, 0},
- {0x8F1, {0x0, 0x0, 0x0}, 0},
- {0x8F2, {0x0, 0x0, 0x0}, 0},
- {0x8F3, {0x0, 0x0, 0x0}, 0},
- {0x8F4, {0x0, 0x0, 0x0}, 0},
- {0x8F5, {0x0, 0x0, 0x0}, 0},
- {0x8F6, {0x0, 0x0, 0x0}, 0},
- {0x8F7, {0x0, 0x0, 0x0}, 0},
- {0x8F8, {0x0, 0x0, 0x0}, 0},
- {0x8F9, {0x0, 0x0, 0x0}, 0},
- {0x8FA, {0x0, 0x0, 0x0}, 0},
- {0x8FB, {0x0, 0x0, 0x0}, 0},
- {0x8FC, {0x0, 0x0, 0x0}, 0},
- {0x8FD, {0x0, 0x0, 0x0}, 0},
- {0x8FE, {0x0, 0x0, 0x0}, 0},
- {0x8FF, {0x0, 0x0, 0x0}, 0},
- {0x900, {0x0, 0x0, 0x0}, 0},
- {0x901, {0x0, 0x0, 0x0}, 0},
- {0x902, {0x0, 0x0, 0x0}, 0},
- {0x929, {0x928, 0x0, 0x0}, 1},
- {0x931, {0x930, 0x0, 0x0}, 1},
- {0x934, {0x933, 0x0, 0x0}, 1},
- {0x93A, {0x0, 0x0, 0x0}, 0},
- {0x93C, {0x0, 0x0, 0x0}, 0},
- {0x941, {0x0, 0x0, 0x0}, 0},
- {0x942, {0x0, 0x0, 0x0}, 0},
- {0x943, {0x0, 0x0, 0x0}, 0},
- {0x944, {0x0, 0x0, 0x0}, 0},
- {0x945, {0x0, 0x0, 0x0}, 0},
- {0x946, {0x0, 0x0, 0x0}, 0},
- {0x947, {0x0, 0x0, 0x0}, 0},
- {0x948, {0x0, 0x0, 0x0}, 0},
- {0x94D, {0x0, 0x0, 0x0}, 0},
- {0x951, {0x0, 0x0, 0x0}, 0},
- {0x952, {0x0, 0x0, 0x0}, 0},
- {0x953, {0x0, 0x0, 0x0}, 0},
- {0x954, {0x0, 0x0, 0x0}, 0},
- {0x955, {0x0, 0x0, 0x0}, 0},
- {0x956, {0x0, 0x0, 0x0}, 0},
- {0x957, {0x0, 0x0, 0x0}, 0},
- {0x958, {0x915, 0x0, 0x0}, 1},
- {0x959, {0x916, 0x0, 0x0}, 1},
- {0x95A, {0x917, 0x0, 0x0}, 1},
- {0x95B, {0x91C, 0x0, 0x0}, 1},
- {0x95C, {0x921, 0x0, 0x0}, 1},
- {0x95D, {0x922, 0x0, 0x0}, 1},
- {0x95E, {0x92B, 0x0, 0x0}, 1},
- {0x95F, {0x92F, 0x0, 0x0}, 1},
- {0x962, {0x0, 0x0, 0x0}, 0},
- {0x963, {0x0, 0x0, 0x0}, 0},
- {0x981, {0x0, 0x0, 0x0}, 0},
- {0x9BC, {0x0, 0x0, 0x0}, 0},
- {0x9C1, {0x0, 0x0, 0x0}, 0},
- {0x9C2, {0x0, 0x0, 0x0}, 0},
- {0x9C3, {0x0, 0x0, 0x0}, 0},
- {0x9C4, {0x0, 0x0, 0x0}, 0},
- {0x9CB, {0x9C7, 0x9BE, 0x0}, 2},
- {0x9CC, {0x9C7, 0x9D7, 0x0}, 2},
- {0x9CD, {0x0, 0x0, 0x0}, 0},
- {0x9DC, {0x9A1, 0x0, 0x0}, 1},
- {0x9DD, {0x9A2, 0x0, 0x0}, 1},
- {0x9DF, {0x9AF, 0x0, 0x0}, 1},
- {0x9E2, {0x0, 0x0, 0x0}, 0},
- {0x9E3, {0x0, 0x0, 0x0}, 0},
- {0x9FE, {0x0, 0x0, 0x0}, 0},
- {0xA01, {0x0, 0x0, 0x0}, 0},
- {0xA02, {0x0, 0x0, 0x0}, 0},
- {0xA33, {0xA32, 0x0, 0x0}, 1},
- {0xA36, {0xA38, 0x0, 0x0}, 1},
- {0xA3C, {0x0, 0x0, 0x0}, 0},
- {0xA41, {0x0, 0x0, 0x0}, 0},
- {0xA42, {0x0, 0x0, 0x0}, 0},
- {0xA47, {0x0, 0x0, 0x0}, 0},
- {0xA48, {0x0, 0x0, 0x0}, 0},
- {0xA4B, {0x0, 0x0, 0x0}, 0},
- {0xA4C, {0x0, 0x0, 0x0}, 0},
- {0xA4D, {0x0, 0x0, 0x0}, 0},
- {0xA51, {0x0, 0x0, 0x0}, 0},
- {0xA59, {0xA16, 0x0, 0x0}, 1},
- {0xA5A, {0xA17, 0x0, 0x0}, 1},
- {0xA5B, {0xA1C, 0x0, 0x0}, 1},
- {0xA5E, {0xA2B, 0x0, 0x0}, 1},
- {0xA70, {0x0, 0x0, 0x0}, 0},
- {0xA71, {0x0, 0x0, 0x0}, 0},
- {0xA75, {0x0, 0x0, 0x0}, 0},
- {0xA81, {0x0, 0x0, 0x0}, 0},
- {0xA82, {0x0, 0x0, 0x0}, 0},
- {0xABC, {0x0, 0x0, 0x0}, 0},
- {0xAC1, {0x0, 0x0, 0x0}, 0},
- {0xAC2, {0x0, 0x0, 0x0}, 0},
- {0xAC3, {0x0, 0x0, 0x0}, 0},
- {0xAC4, {0x0, 0x0, 0x0}, 0},
- {0xAC5, {0x0, 0x0, 0x0}, 0},
- {0xAC7, {0x0, 0x0, 0x0}, 0},
- {0xAC8, {0x0, 0x0, 0x0}, 0},
- {0xACD, {0x0, 0x0, 0x0}, 0},
- {0xAE2, {0x0, 0x0, 0x0}, 0},
- {0xAE3, {0x0, 0x0, 0x0}, 0},
- {0xAFA, {0x0, 0x0, 0x0}, 0},
- {0xAFB, {0x0, 0x0, 0x0}, 0},
- {0xAFC, {0x0, 0x0, 0x0}, 0},
- {0xAFD, {0x0, 0x0, 0x0}, 0},
- {0xAFE, {0x0, 0x0, 0x0}, 0},
- {0xAFF, {0x0, 0x0, 0x0}, 0},
- {0xB01, {0x0, 0x0, 0x0}, 0},
- {0xB3C, {0x0, 0x0, 0x0}, 0},
- {0xB3F, {0x0, 0x0, 0x0}, 0},
- {0xB41, {0x0, 0x0, 0x0}, 0},
- {0xB42, {0x0, 0x0, 0x0}, 0},
- {0xB43, {0x0, 0x0, 0x0}, 0},
- {0xB44, {0x0, 0x0, 0x0}, 0},
- {0xB48, {0xB47, 0x0, 0x0}, 1},
- {0xB4B, {0xB47, 0xB3E, 0x0}, 2},
- {0xB4C, {0xB47, 0xB57, 0x0}, 2},
- {0xB4D, {0x0, 0x0, 0x0}, 0},
- {0xB55, {0x0, 0x0, 0x0}, 0},
- {0xB56, {0x0, 0x0, 0x0}, 0},
- {0xB5C, {0xB21, 0x0, 0x0}, 1},
- {0xB5D, {0xB22, 0x0, 0x0}, 1},
- {0xB62, {0x0, 0x0, 0x0}, 0},
- {0xB63, {0x0, 0x0, 0x0}, 0},
- {0xB82, {0x0, 0x0, 0x0}, 0},
- {0xB94, {0xB92, 0xBD7, 0x0}, 2},
- {0xBC0, {0x0, 0x0, 0x0}, 0},
- {0xBCA, {0xBC6, 0xBBE, 0x0}, 2},
- {0xBCB, {0xBC7, 0xBBE, 0x0}, 2},
- {0xBCC, {0xBC6, 0xBD7, 0x0}, 2},
- {0xBCD, {0x0, 0x0, 0x0}, 0},
- {0xC00, {0x0, 0x0, 0x0}, 0},
- {0xC04, {0x0, 0x0, 0x0}, 0},
- {0xC3E, {0x0, 0x0, 0x0}, 0},
- {0xC3F, {0x0, 0x0, 0x0}, 0},
- {0xC40, {0x0, 0x0, 0x0}, 0},
- {0xC46, {0x0, 0x0, 0x0}, 0},
- {0xC47, {0x0, 0x0, 0x0}, 0},
- {0xC48, {0x0, 0x0, 0x0}, 0},
- {0xC4A, {0x0, 0x0, 0x0}, 0},
- {0xC4B, {0x0, 0x0, 0x0}, 0},
- {0xC4C, {0x0, 0x0, 0x0}, 0},
- {0xC4D, {0x0, 0x0, 0x0}, 0},
- {0xC55, {0x0, 0x0, 0x0}, 0},
- {0xC56, {0x0, 0x0, 0x0}, 0},
- {0xC62, {0x0, 0x0, 0x0}, 0},
- {0xC63, {0x0, 0x0, 0x0}, 0},
- {0xC81, {0x0, 0x0, 0x0}, 0},
- {0xCBC, {0x0, 0x0, 0x0}, 0},
- {0xCBF, {0x0, 0x0, 0x0}, 0},
- {0xCC0, {0xCD5, 0x0, 0x0}, 1},
- {0xCC6, {0x0, 0x0, 0x0}, 0},
- {0xCC7, {0xCD5, 0x0, 0x0}, 1},
- {0xCC8, {0xCD6, 0x0, 0x0}, 1},
- {0xCCA, {0xCC2, 0x0, 0x0}, 1},
- {0xCCB, {0xCC2, 0xCD5, 0x0}, 2},
- {0xCCC, {0x0, 0x0, 0x0}, 0},
- {0xCCD, {0x0, 0x0, 0x0}, 0},
- {0xCE2, {0x0, 0x0, 0x0}, 0},
- {0xCE3, {0x0, 0x0, 0x0}, 0},
- {0xD00, {0x0, 0x0, 0x0}, 0},
- {0xD01, {0x0, 0x0, 0x0}, 0},
- {0xD3B, {0x0, 0x0, 0x0}, 0},
- {0xD3C, {0x0, 0x0, 0x0}, 0},
- {0xD41, {0x0, 0x0, 0x0}, 0},
- {0xD42, {0x0, 0x0, 0x0}, 0},
- {0xD43, {0x0, 0x0, 0x0}, 0},
- {0xD44, {0x0, 0x0, 0x0}, 0},
- {0xD4A, {0xD46, 0xD3E, 0x0}, 2},
- {0xD4B, {0xD47, 0xD3E, 0x0}, 2},
- {0xD4C, {0xD46, 0xD57, 0x0}, 2},
- {0xD4D, {0x0, 0x0, 0x0}, 0},
- {0xD62, {0x0, 0x0, 0x0}, 0},
- {0xD63, {0x0, 0x0, 0x0}, 0},
- {0xD81, {0x0, 0x0, 0x0}, 0},
- {0xDCA, {0x0, 0x0, 0x0}, 0},
- {0xDD2, {0x0, 0x0, 0x0}, 0},
- {0xDD3, {0x0, 0x0, 0x0}, 0},
- {0xDD4, {0x0, 0x0, 0x0}, 0},
- {0xDD6, {0x0, 0x0, 0x0}, 0},
- {0xDDA, {0xDD9, 0x0, 0x0}, 1},
- {0xDDC, {0xDD9, 0xDCF, 0x0}, 2},
- {0xDDD, {0xDD9, 0xDCF, 0x0}, 2},
- {0xDDE, {0xDD9, 0xDDF, 0x0}, 2},
- {0xE31, {0x0, 0x0, 0x0}, 0},
- {0xE34, {0x0, 0x0, 0x0}, 0},
- {0xE35, {0x0, 0x0, 0x0}, 0},
- {0xE36, {0x0, 0x0, 0x0}, 0},
- {0xE37, {0x0, 0x0, 0x0}, 0},
- {0xE38, {0x0, 0x0, 0x0}, 0},
- {0xE39, {0x0, 0x0, 0x0}, 0},
- {0xE3A, {0x0, 0x0, 0x0}, 0},
- {0xE47, {0x0, 0x0, 0x0}, 0},
- {0xE48, {0x0, 0x0, 0x0}, 0},
- {0xE49, {0x0, 0x0, 0x0}, 0},
- {0xE4A, {0x0, 0x0, 0x0}, 0},
- {0xE4B, {0x0, 0x0, 0x0}, 0},
- {0xE4C, {0x0, 0x0, 0x0}, 0},
- {0xE4D, {0x0, 0x0, 0x0}, 0},
- {0xE4E, {0x0, 0x0, 0x0}, 0},
- {0xEB1, {0x0, 0x0, 0x0}, 0},
- {0xEB4, {0x0, 0x0, 0x0}, 0},
- {0xEB5, {0x0, 0x0, 0x0}, 0},
- {0xEB6, {0x0, 0x0, 0x0}, 0},
- {0xEB7, {0x0, 0x0, 0x0}, 0},
- {0xEB8, {0x0, 0x0, 0x0}, 0},
- {0xEB9, {0x0, 0x0, 0x0}, 0},
- {0xEBA, {0x0, 0x0, 0x0}, 0},
- {0xEBB, {0x0, 0x0, 0x0}, 0},
- {0xEBC, {0x0, 0x0, 0x0}, 0},
- {0xEC8, {0x0, 0x0, 0x0}, 0},
- {0xEC9, {0x0, 0x0, 0x0}, 0},
- {0xECA, {0x0, 0x0, 0x0}, 0},
- {0xECB, {0x0, 0x0, 0x0}, 0},
- {0xECC, {0x0, 0x0, 0x0}, 0},
- {0xECD, {0x0, 0x0, 0x0}, 0},
- {0xF18, {0x0, 0x0, 0x0}, 0},
- {0xF19, {0x0, 0x0, 0x0}, 0},
- {0xF35, {0x0, 0x0, 0x0}, 0},
- {0xF37, {0x0, 0x0, 0x0}, 0},
- {0xF39, {0x0, 0x0, 0x0}, 0},
- {0xF43, {0xF42, 0x0, 0x0}, 1},
- {0xF4D, {0xF4C, 0x0, 0x0}, 1},
- {0xF52, {0xF51, 0x0, 0x0}, 1},
- {0xF57, {0xF56, 0x0, 0x0}, 1},
- {0xF5C, {0xF5B, 0x0, 0x0}, 1},
- {0xF69, {0xF40, 0x0, 0x0}, 1},
- {0xF71, {0x0, 0x0, 0x0}, 0},
- {0xF72, {0x0, 0x0, 0x0}, 0},
- {0xF73, {0x0, 0x0, 0x0}, 0},
- {0xF74, {0x0, 0x0, 0x0}, 0},
- {0xF75, {0x0, 0x0, 0x0}, 0},
- {0xF76, {0x0, 0x0, 0x0}, 0},
- {0xF77, {0x0, 0x0, 0x0}, 0},
- {0xF78, {0x0, 0x0, 0x0}, 0},
- {0xF79, {0x0, 0x0, 0x0}, 0},
- {0xF7A, {0x0, 0x0, 0x0}, 0},
- {0xF7B, {0x0, 0x0, 0x0}, 0},
- {0xF7C, {0x0, 0x0, 0x0}, 0},
- {0xF7D, {0x0, 0x0, 0x0}, 0},
- {0xF7E, {0x0, 0x0, 0x0}, 0},
- {0xF80, {0x0, 0x0, 0x0}, 0},
- {0xF81, {0x0, 0x0, 0x0}, 0},
- {0xF82, {0x0, 0x0, 0x0}, 0},
- {0xF83, {0x0, 0x0, 0x0}, 0},
- {0xF84, {0x0, 0x0, 0x0}, 0},
- {0xF86, {0x0, 0x0, 0x0}, 0},
- {0xF87, {0x0, 0x0, 0x0}, 0},
- {0xF8D, {0x0, 0x0, 0x0}, 0},
- {0xF8E, {0x0, 0x0, 0x0}, 0},
- {0xF8F, {0x0, 0x0, 0x0}, 0},
- {0xF90, {0x0, 0x0, 0x0}, 0},
- {0xF91, {0x0, 0x0, 0x0}, 0},
- {0xF92, {0x0, 0x0, 0x0}, 0},
- {0xF93, {0x0, 0x0, 0x0}, 0},
- {0xF94, {0x0, 0x0, 0x0}, 0},
- {0xF95, {0x0, 0x0, 0x0}, 0},
- {0xF96, {0x0, 0x0, 0x0}, 0},
- {0xF97, {0x0, 0x0, 0x0}, 0},
- {0xF99, {0x0, 0x0, 0x0}, 0},
- {0xF9A, {0x0, 0x0, 0x0}, 0},
- {0xF9B, {0x0, 0x0, 0x0}, 0},
- {0xF9C, {0x0, 0x0, 0x0}, 0},
- {0xF9D, {0x0, 0x0, 0x0}, 0},
- {0xF9E, {0x0, 0x0, 0x0}, 0},
- {0xF9F, {0x0, 0x0, 0x0}, 0},
- {0xFA0, {0x0, 0x0, 0x0}, 0},
- {0xFA1, {0x0, 0x0, 0x0}, 0},
- {0xFA2, {0x0, 0x0, 0x0}, 0},
- {0xFA3, {0x0, 0x0, 0x0}, 0},
- {0xFA4, {0x0, 0x0, 0x0}, 0},
- {0xFA5, {0x0, 0x0, 0x0}, 0},
- {0xFA6, {0x0, 0x0, 0x0}, 0},
- {0xFA7, {0x0, 0x0, 0x0}, 0},
- {0xFA8, {0x0, 0x0, 0x0}, 0},
- {0xFA9, {0x0, 0x0, 0x0}, 0},
- {0xFAA, {0x0, 0x0, 0x0}, 0},
- {0xFAB, {0x0, 0x0, 0x0}, 0},
- {0xFAC, {0x0, 0x0, 0x0}, 0},
- {0xFAD, {0x0, 0x0, 0x0}, 0},
- {0xFAE, {0x0, 0x0, 0x0}, 0},
- {0xFAF, {0x0, 0x0, 0x0}, 0},
- {0xFB0, {0x0, 0x0, 0x0}, 0},
- {0xFB1, {0x0, 0x0, 0x0}, 0},
- {0xFB2, {0x0, 0x0, 0x0}, 0},
- {0xFB3, {0x0, 0x0, 0x0}, 0},
- {0xFB4, {0x0, 0x0, 0x0}, 0},
- {0xFB5, {0x0, 0x0, 0x0}, 0},
- {0xFB6, {0x0, 0x0, 0x0}, 0},
- {0xFB7, {0x0, 0x0, 0x0}, 0},
- {0xFB8, {0x0, 0x0, 0x0}, 0},
- {0xFB9, {0x0, 0x0, 0x0}, 0},
- {0xFBA, {0x0, 0x0, 0x0}, 0},
- {0xFBB, {0x0, 0x0, 0x0}, 0},
- {0xFBC, {0x0, 0x0, 0x0}, 0},
- {0xFC6, {0x0, 0x0, 0x0}, 0},
- {0x1026, {0x1025, 0x0, 0x0}, 1},
- {0x102D, {0x0, 0x0, 0x0}, 0},
- {0x102E, {0x0, 0x0, 0x0}, 0},
- {0x102F, {0x0, 0x0, 0x0}, 0},
- {0x1030, {0x0, 0x0, 0x0}, 0},
- {0x1032, {0x0, 0x0, 0x0}, 0},
- {0x1033, {0x0, 0x0, 0x0}, 0},
- {0x1034, {0x0, 0x0, 0x0}, 0},
- {0x1035, {0x0, 0x0, 0x0}, 0},
- {0x1036, {0x0, 0x0, 0x0}, 0},
- {0x1037, {0x0, 0x0, 0x0}, 0},
- {0x1039, {0x0, 0x0, 0x0}, 0},
- {0x103A, {0x0, 0x0, 0x0}, 0},
- {0x103D, {0x0, 0x0, 0x0}, 0},
- {0x103E, {0x0, 0x0, 0x0}, 0},
- {0x1058, {0x0, 0x0, 0x0}, 0},
- {0x1059, {0x0, 0x0, 0x0}, 0},
- {0x105E, {0x0, 0x0, 0x0}, 0},
- {0x105F, {0x0, 0x0, 0x0}, 0},
- {0x1060, {0x0, 0x0, 0x0}, 0},
- {0x1071, {0x0, 0x0, 0x0}, 0},
- {0x1072, {0x0, 0x0, 0x0}, 0},
- {0x1073, {0x0, 0x0, 0x0}, 0},
- {0x1074, {0x0, 0x0, 0x0}, 0},
- {0x1082, {0x0, 0x0, 0x0}, 0},
- {0x1085, {0x0, 0x0, 0x0}, 0},
- {0x1086, {0x0, 0x0, 0x0}, 0},
- {0x108D, {0x0, 0x0, 0x0}, 0},
- {0x109D, {0x0, 0x0, 0x0}, 0},
- {0x10A0, {0x2D00, 0x0, 0x0}, 1},
- {0x10A1, {0x2D01, 0x0, 0x0}, 1},
- {0x10A2, {0x2D02, 0x0, 0x0}, 1},
- {0x10A3, {0x2D03, 0x0, 0x0}, 1},
- {0x10A4, {0x2D04, 0x0, 0x0}, 1},
- {0x10A5, {0x2D05, 0x0, 0x0}, 1},
- {0x10A6, {0x2D06, 0x0, 0x0}, 1},
- {0x10A7, {0x2D07, 0x0, 0x0}, 1},
- {0x10A8, {0x2D08, 0x0, 0x0}, 1},
- {0x10A9, {0x2D09, 0x0, 0x0}, 1},
- {0x10AA, {0x2D0A, 0x0, 0x0}, 1},
- {0x10AB, {0x2D0B, 0x0, 0x0}, 1},
- {0x10AC, {0x2D0C, 0x0, 0x0}, 1},
- {0x10AD, {0x2D0D, 0x0, 0x0}, 1},
- {0x10AE, {0x2D0E, 0x0, 0x0}, 1},
- {0x10AF, {0x2D0F, 0x0, 0x0}, 1},
- {0x10B0, {0x2D10, 0x0, 0x0}, 1},
- {0x10B1, {0x2D11, 0x0, 0x0}, 1},
- {0x10B2, {0x2D12, 0x0, 0x0}, 1},
- {0x10B3, {0x2D13, 0x0, 0x0}, 1},
- {0x10B4, {0x2D14, 0x0, 0x0}, 1},
- {0x10B5, {0x2D15, 0x0, 0x0}, 1},
- {0x10B6, {0x2D16, 0x0, 0x0}, 1},
- {0x10B7, {0x2D17, 0x0, 0x0}, 1},
- {0x10B8, {0x2D18, 0x0, 0x0}, 1},
- {0x10B9, {0x2D19, 0x0, 0x0}, 1},
- {0x10BA, {0x2D1A, 0x0, 0x0}, 1},
- {0x10BB, {0x2D1B, 0x0, 0x0}, 1},
- {0x10BC, {0x2D1C, 0x0, 0x0}, 1},
- {0x10BD, {0x2D1D, 0x0, 0x0}, 1},
- {0x10BE, {0x2D1E, 0x0, 0x0}, 1},
- {0x10BF, {0x2D1F, 0x0, 0x0}, 1},
- {0x10C0, {0x2D20, 0x0, 0x0}, 1},
- {0x10C1, {0x2D21, 0x0, 0x0}, 1},
- {0x10C2, {0x2D22, 0x0, 0x0}, 1},
- {0x10C3, {0x2D23, 0x0, 0x0}, 1},
- {0x10C4, {0x2D24, 0x0, 0x0}, 1},
- {0x10C5, {0x2D25, 0x0, 0x0}, 1},
- {0x10C7, {0x2D27, 0x0, 0x0}, 1},
- {0x10CD, {0x2D2D, 0x0, 0x0}, 1},
- {0x135D, {0x0, 0x0, 0x0}, 0},
- {0x135E, {0x0, 0x0, 0x0}, 0},
- {0x135F, {0x0, 0x0, 0x0}, 0},
- {0x13A0, {0xAB70, 0x0, 0x0}, 1},
- {0x13A1, {0xAB71, 0x0, 0x0}, 1},
- {0x13A2, {0xAB72, 0x0, 0x0}, 1},
- {0x13A3, {0xAB73, 0x0, 0x0}, 1},
- {0x13A4, {0xAB74, 0x0, 0x0}, 1},
- {0x13A5, {0xAB75, 0x0, 0x0}, 1},
- {0x13A6, {0xAB76, 0x0, 0x0}, 1},
- {0x13A7, {0xAB77, 0x0, 0x0}, 1},
- {0x13A8, {0xAB78, 0x0, 0x0}, 1},
- {0x13A9, {0xAB79, 0x0, 0x0}, 1},
- {0x13AA, {0xAB7A, 0x0, 0x0}, 1},
- {0x13AB, {0xAB7B, 0x0, 0x0}, 1},
- {0x13AC, {0xAB7C, 0x0, 0x0}, 1},
- {0x13AD, {0xAB7D, 0x0, 0x0}, 1},
- {0x13AE, {0xAB7E, 0x0, 0x0}, 1},
- {0x13AF, {0xAB7F, 0x0, 0x0}, 1},
- {0x13B0, {0xAB80, 0x0, 0x0}, 1},
- {0x13B1, {0xAB81, 0x0, 0x0}, 1},
- {0x13B2, {0xAB82, 0x0, 0x0}, 1},
- {0x13B3, {0xAB83, 0x0, 0x0}, 1},
- {0x13B4, {0xAB84, 0x0, 0x0}, 1},
- {0x13B5, {0xAB85, 0x0, 0x0}, 1},
- {0x13B6, {0xAB86, 0x0, 0x0}, 1},
- {0x13B7, {0xAB87, 0x0, 0x0}, 1},
- {0x13B8, {0xAB88, 0x0, 0x0}, 1},
- {0x13B9, {0xAB89, 0x0, 0x0}, 1},
- {0x13BA, {0xAB8A, 0x0, 0x0}, 1},
- {0x13BB, {0xAB8B, 0x0, 0x0}, 1},
- {0x13BC, {0xAB8C, 0x0, 0x0}, 1},
- {0x13BD, {0xAB8D, 0x0, 0x0}, 1},
- {0x13BE, {0xAB8E, 0x0, 0x0}, 1},
- {0x13BF, {0xAB8F, 0x0, 0x0}, 1},
- {0x13C0, {0xAB90, 0x0, 0x0}, 1},
- {0x13C1, {0xAB91, 0x0, 0x0}, 1},
- {0x13C2, {0xAB92, 0x0, 0x0}, 1},
- {0x13C3, {0xAB93, 0x0, 0x0}, 1},
- {0x13C4, {0xAB94, 0x0, 0x0}, 1},
- {0x13C5, {0xAB95, 0x0, 0x0}, 1},
- {0x13C6, {0xAB96, 0x0, 0x0}, 1},
- {0x13C7, {0xAB97, 0x0, 0x0}, 1},
- {0x13C8, {0xAB98, 0x0, 0x0}, 1},
- {0x13C9, {0xAB99, 0x0, 0x0}, 1},
- {0x13CA, {0xAB9A, 0x0, 0x0}, 1},
- {0x13CB, {0xAB9B, 0x0, 0x0}, 1},
- {0x13CC, {0xAB9C, 0x0, 0x0}, 1},
- {0x13CD, {0xAB9D, 0x0, 0x0}, 1},
- {0x13CE, {0xAB9E, 0x0, 0x0}, 1},
- {0x13CF, {0xAB9F, 0x0, 0x0}, 1},
- {0x13D0, {0xABA0, 0x0, 0x0}, 1},
- {0x13D1, {0xABA1, 0x0, 0x0}, 1},
- {0x13D2, {0xABA2, 0x0, 0x0}, 1},
- {0x13D3, {0xABA3, 0x0, 0x0}, 1},
- {0x13D4, {0xABA4, 0x0, 0x0}, 1},
- {0x13D5, {0xABA5, 0x0, 0x0}, 1},
- {0x13D6, {0xABA6, 0x0, 0x0}, 1},
- {0x13D7, {0xABA7, 0x0, 0x0}, 1},
- {0x13D8, {0xABA8, 0x0, 0x0}, 1},
- {0x13D9, {0xABA9, 0x0, 0x0}, 1},
- {0x13DA, {0xABAA, 0x0, 0x0}, 1},
- {0x13DB, {0xABAB, 0x0, 0x0}, 1},
- {0x13DC, {0xABAC, 0x0, 0x0}, 1},
- {0x13DD, {0xABAD, 0x0, 0x0}, 1},
- {0x13DE, {0xABAE, 0x0, 0x0}, 1},
- {0x13DF, {0xABAF, 0x0, 0x0}, 1},
- {0x13E0, {0xABB0, 0x0, 0x0}, 1},
- {0x13E1, {0xABB1, 0x0, 0x0}, 1},
- {0x13E2, {0xABB2, 0x0, 0x0}, 1},
- {0x13E3, {0xABB3, 0x0, 0x0}, 1},
- {0x13E4, {0xABB4, 0x0, 0x0}, 1},
- {0x13E5, {0xABB5, 0x0, 0x0}, 1},
- {0x13E6, {0xABB6, 0x0, 0x0}, 1},
- {0x13E7, {0xABB7, 0x0, 0x0}, 1},
- {0x13E8, {0xABB8, 0x0, 0x0}, 1},
- {0x13E9, {0xABB9, 0x0, 0x0}, 1},
- {0x13EA, {0xABBA, 0x0, 0x0}, 1},
- {0x13EB, {0xABBB, 0x0, 0x0}, 1},
- {0x13EC, {0xABBC, 0x0, 0x0}, 1},
- {0x13ED, {0xABBD, 0x0, 0x0}, 1},
- {0x13EE, {0xABBE, 0x0, 0x0}, 1},
- {0x13EF, {0xABBF, 0x0, 0x0}, 1},
- {0x13F0, {0x13F8, 0x0, 0x0}, 1},
- {0x13F1, {0x13F9, 0x0, 0x0}, 1},
- {0x13F2, {0x13FA, 0x0, 0x0}, 1},
- {0x13F3, {0x13FB, 0x0, 0x0}, 1},
- {0x13F4, {0x13FC, 0x0, 0x0}, 1},
- {0x13F5, {0x13FD, 0x0, 0x0}, 1},
- {0x1712, {0x0, 0x0, 0x0}, 0},
- {0x1713, {0x0, 0x0, 0x0}, 0},
- {0x1714, {0x0, 0x0, 0x0}, 0},
- {0x1732, {0x0, 0x0, 0x0}, 0},
- {0x1733, {0x0, 0x0, 0x0}, 0},
- {0x1734, {0x0, 0x0, 0x0}, 0},
- {0x1752, {0x0, 0x0, 0x0}, 0},
- {0x1753, {0x0, 0x0, 0x0}, 0},
- {0x1772, {0x0, 0x0, 0x0}, 0},
- {0x1773, {0x0, 0x0, 0x0}, 0},
- {0x17B4, {0x0, 0x0, 0x0}, 0},
- {0x17B5, {0x0, 0x0, 0x0}, 0},
- {0x17B7, {0x0, 0x0, 0x0}, 0},
- {0x17B8, {0x0, 0x0, 0x0}, 0},
- {0x17B9, {0x0, 0x0, 0x0}, 0},
- {0x17BA, {0x0, 0x0, 0x0}, 0},
- {0x17BB, {0x0, 0x0, 0x0}, 0},
- {0x17BC, {0x0, 0x0, 0x0}, 0},
- {0x17BD, {0x0, 0x0, 0x0}, 0},
- {0x17C6, {0x0, 0x0, 0x0}, 0},
- {0x17C9, {0x0, 0x0, 0x0}, 0},
- {0x17CA, {0x0, 0x0, 0x0}, 0},
- {0x17CB, {0x0, 0x0, 0x0}, 0},
- {0x17CC, {0x0, 0x0, 0x0}, 0},
- {0x17CD, {0x0, 0x0, 0x0}, 0},
- {0x17CE, {0x0, 0x0, 0x0}, 0},
- {0x17CF, {0x0, 0x0, 0x0}, 0},
- {0x17D0, {0x0, 0x0, 0x0}, 0},
- {0x17D1, {0x0, 0x0, 0x0}, 0},
- {0x17D2, {0x0, 0x0, 0x0}, 0},
- {0x17D3, {0x0, 0x0, 0x0}, 0},
- {0x17DD, {0x0, 0x0, 0x0}, 0},
- {0x180B, {0x0, 0x0, 0x0}, 0},
- {0x180C, {0x0, 0x0, 0x0}, 0},
- {0x180D, {0x0, 0x0, 0x0}, 0},
- {0x1885, {0x0, 0x0, 0x0}, 0},
- {0x1886, {0x0, 0x0, 0x0}, 0},
- {0x18A9, {0x0, 0x0, 0x0}, 0},
- {0x1920, {0x0, 0x0, 0x0}, 0},
- {0x1921, {0x0, 0x0, 0x0}, 0},
- {0x1922, {0x0, 0x0, 0x0}, 0},
- {0x1927, {0x0, 0x0, 0x0}, 0},
- {0x1928, {0x0, 0x0, 0x0}, 0},
- {0x1932, {0x0, 0x0, 0x0}, 0},
- {0x1939, {0x0, 0x0, 0x0}, 0},
- {0x193A, {0x0, 0x0, 0x0}, 0},
- {0x193B, {0x0, 0x0, 0x0}, 0},
- {0x1A17, {0x0, 0x0, 0x0}, 0},
- {0x1A18, {0x0, 0x0, 0x0}, 0},
- {0x1A1B, {0x0, 0x0, 0x0}, 0},
- {0x1A56, {0x0, 0x0, 0x0}, 0},
- {0x1A58, {0x0, 0x0, 0x0}, 0},
- {0x1A59, {0x0, 0x0, 0x0}, 0},
- {0x1A5A, {0x0, 0x0, 0x0}, 0},
- {0x1A5B, {0x0, 0x0, 0x0}, 0},
- {0x1A5C, {0x0, 0x0, 0x0}, 0},
- {0x1A5D, {0x0, 0x0, 0x0}, 0},
- {0x1A5E, {0x0, 0x0, 0x0}, 0},
- {0x1A60, {0x0, 0x0, 0x0}, 0},
- {0x1A62, {0x0, 0x0, 0x0}, 0},
- {0x1A65, {0x0, 0x0, 0x0}, 0},
- {0x1A66, {0x0, 0x0, 0x0}, 0},
- {0x1A67, {0x0, 0x0, 0x0}, 0},
- {0x1A68, {0x0, 0x0, 0x0}, 0},
- {0x1A69, {0x0, 0x0, 0x0}, 0},
- {0x1A6A, {0x0, 0x0, 0x0}, 0},
- {0x1A6B, {0x0, 0x0, 0x0}, 0},
- {0x1A6C, {0x0, 0x0, 0x0}, 0},
- {0x1A73, {0x0, 0x0, 0x0}, 0},
- {0x1A74, {0x0, 0x0, 0x0}, 0},
- {0x1A75, {0x0, 0x0, 0x0}, 0},
- {0x1A76, {0x0, 0x0, 0x0}, 0},
- {0x1A77, {0x0, 0x0, 0x0}, 0},
- {0x1A78, {0x0, 0x0, 0x0}, 0},
- {0x1A79, {0x0, 0x0, 0x0}, 0},
- {0x1A7A, {0x0, 0x0, 0x0}, 0},
- {0x1A7B, {0x0, 0x0, 0x0}, 0},
- {0x1A7C, {0x0, 0x0, 0x0}, 0},
- {0x1A7F, {0x0, 0x0, 0x0}, 0},
- {0x1AB0, {0x0, 0x0, 0x0}, 0},
- {0x1AB1, {0x0, 0x0, 0x0}, 0},
- {0x1AB2, {0x0, 0x0, 0x0}, 0},
- {0x1AB3, {0x0, 0x0, 0x0}, 0},
- {0x1AB4, {0x0, 0x0, 0x0}, 0},
- {0x1AB5, {0x0, 0x0, 0x0}, 0},
- {0x1AB6, {0x0, 0x0, 0x0}, 0},
- {0x1AB7, {0x0, 0x0, 0x0}, 0},
- {0x1AB8, {0x0, 0x0, 0x0}, 0},
- {0x1AB9, {0x0, 0x0, 0x0}, 0},
- {0x1ABA, {0x0, 0x0, 0x0}, 0},
- {0x1ABB, {0x0, 0x0, 0x0}, 0},
- {0x1ABC, {0x0, 0x0, 0x0}, 0},
- {0x1ABD, {0x0, 0x0, 0x0}, 0},
- {0x1ABF, {0x0, 0x0, 0x0}, 0},
- {0x1AC0, {0x0, 0x0, 0x0}, 0},
- {0x1B00, {0x0, 0x0, 0x0}, 0},
- {0x1B01, {0x0, 0x0, 0x0}, 0},
- {0x1B02, {0x0, 0x0, 0x0}, 0},
- {0x1B03, {0x0, 0x0, 0x0}, 0},
- {0x1B06, {0x1B05, 0x1B35, 0x0}, 2},
- {0x1B08, {0x1B07, 0x1B35, 0x0}, 2},
- {0x1B0A, {0x1B09, 0x1B35, 0x0}, 2},
- {0x1B0C, {0x1B0B, 0x1B35, 0x0}, 2},
- {0x1B0E, {0x1B0D, 0x1B35, 0x0}, 2},
- {0x1B12, {0x1B11, 0x1B35, 0x0}, 2},
- {0x1B34, {0x0, 0x0, 0x0}, 0},
- {0x1B36, {0x0, 0x0, 0x0}, 0},
- {0x1B37, {0x0, 0x0, 0x0}, 0},
- {0x1B38, {0x0, 0x0, 0x0}, 0},
- {0x1B39, {0x0, 0x0, 0x0}, 0},
- {0x1B3A, {0x0, 0x0, 0x0}, 0},
- {0x1B3B, {0x1B35, 0x0, 0x0}, 1},
- {0x1B3C, {0x0, 0x0, 0x0}, 0},
- {0x1B3D, {0x1B35, 0x0, 0x0}, 1},
- {0x1B40, {0x1B3E, 0x1B35, 0x0}, 2},
- {0x1B41, {0x1B3F, 0x1B35, 0x0}, 2},
- {0x1B42, {0x0, 0x0, 0x0}, 0},
- {0x1B43, {0x1B35, 0x0, 0x0}, 1},
- {0x1B6B, {0x0, 0x0, 0x0}, 0},
- {0x1B6C, {0x0, 0x0, 0x0}, 0},
- {0x1B6D, {0x0, 0x0, 0x0}, 0},
- {0x1B6E, {0x0, 0x0, 0x0}, 0},
- {0x1B6F, {0x0, 0x0, 0x0}, 0},
- {0x1B70, {0x0, 0x0, 0x0}, 0},
- {0x1B71, {0x0, 0x0, 0x0}, 0},
- {0x1B72, {0x0, 0x0, 0x0}, 0},
- {0x1B73, {0x0, 0x0, 0x0}, 0},
- {0x1B80, {0x0, 0x0, 0x0}, 0},
- {0x1B81, {0x0, 0x0, 0x0}, 0},
- {0x1BA2, {0x0, 0x0, 0x0}, 0},
- {0x1BA3, {0x0, 0x0, 0x0}, 0},
- {0x1BA4, {0x0, 0x0, 0x0}, 0},
- {0x1BA5, {0x0, 0x0, 0x0}, 0},
- {0x1BA8, {0x0, 0x0, 0x0}, 0},
- {0x1BA9, {0x0, 0x0, 0x0}, 0},
- {0x1BAB, {0x0, 0x0, 0x0}, 0},
- {0x1BAC, {0x0, 0x0, 0x0}, 0},
- {0x1BAD, {0x0, 0x0, 0x0}, 0},
- {0x1BE6, {0x0, 0x0, 0x0}, 0},
- {0x1BE8, {0x0, 0x0, 0x0}, 0},
- {0x1BE9, {0x0, 0x0, 0x0}, 0},
- {0x1BED, {0x0, 0x0, 0x0}, 0},
- {0x1BEF, {0x0, 0x0, 0x0}, 0},
- {0x1BF0, {0x0, 0x0, 0x0}, 0},
- {0x1BF1, {0x0, 0x0, 0x0}, 0},
- {0x1C2C, {0x0, 0x0, 0x0}, 0},
- {0x1C2D, {0x0, 0x0, 0x0}, 0},
- {0x1C2E, {0x0, 0x0, 0x0}, 0},
- {0x1C2F, {0x0, 0x0, 0x0}, 0},
- {0x1C30, {0x0, 0x0, 0x0}, 0},
- {0x1C31, {0x0, 0x0, 0x0}, 0},
- {0x1C32, {0x0, 0x0, 0x0}, 0},
- {0x1C33, {0x0, 0x0, 0x0}, 0},
- {0x1C36, {0x0, 0x0, 0x0}, 0},
- {0x1C37, {0x0, 0x0, 0x0}, 0},
- {0x1C90, {0x10D0, 0x0, 0x0}, 1},
- {0x1C91, {0x10D1, 0x0, 0x0}, 1},
- {0x1C92, {0x10D2, 0x0, 0x0}, 1},
- {0x1C93, {0x10D3, 0x0, 0x0}, 1},
- {0x1C94, {0x10D4, 0x0, 0x0}, 1},
- {0x1C95, {0x10D5, 0x0, 0x0}, 1},
- {0x1C96, {0x10D6, 0x0, 0x0}, 1},
- {0x1C97, {0x10D7, 0x0, 0x0}, 1},
- {0x1C98, {0x10D8, 0x0, 0x0}, 1},
- {0x1C99, {0x10D9, 0x0, 0x0}, 1},
- {0x1C9A, {0x10DA, 0x0, 0x0}, 1},
- {0x1C9B, {0x10DB, 0x0, 0x0}, 1},
- {0x1C9C, {0x10DC, 0x0, 0x0}, 1},
- {0x1C9D, {0x10DD, 0x0, 0x0}, 1},
- {0x1C9E, {0x10DE, 0x0, 0x0}, 1},
- {0x1C9F, {0x10DF, 0x0, 0x0}, 1},
- {0x1CA0, {0x10E0, 0x0, 0x0}, 1},
- {0x1CA1, {0x10E1, 0x0, 0x0}, 1},
- {0x1CA2, {0x10E2, 0x0, 0x0}, 1},
- {0x1CA3, {0x10E3, 0x0, 0x0}, 1},
- {0x1CA4, {0x10E4, 0x0, 0x0}, 1},
- {0x1CA5, {0x10E5, 0x0, 0x0}, 1},
- {0x1CA6, {0x10E6, 0x0, 0x0}, 1},
- {0x1CA7, {0x10E7, 0x0, 0x0}, 1},
- {0x1CA8, {0x10E8, 0x0, 0x0}, 1},
- {0x1CA9, {0x10E9, 0x0, 0x0}, 1},
- {0x1CAA, {0x10EA, 0x0, 0x0}, 1},
- {0x1CAB, {0x10EB, 0x0, 0x0}, 1},
- {0x1CAC, {0x10EC, 0x0, 0x0}, 1},
- {0x1CAD, {0x10ED, 0x0, 0x0}, 1},
- {0x1CAE, {0x10EE, 0x0, 0x0}, 1},
- {0x1CAF, {0x10EF, 0x0, 0x0}, 1},
- {0x1CB0, {0x10F0, 0x0, 0x0}, 1},
- {0x1CB1, {0x10F1, 0x0, 0x0}, 1},
- {0x1CB2, {0x10F2, 0x0, 0x0}, 1},
- {0x1CB3, {0x10F3, 0x0, 0x0}, 1},
- {0x1CB4, {0x10F4, 0x0, 0x0}, 1},
- {0x1CB5, {0x10F5, 0x0, 0x0}, 1},
- {0x1CB6, {0x10F6, 0x0, 0x0}, 1},
- {0x1CB7, {0x10F7, 0x0, 0x0}, 1},
- {0x1CB8, {0x10F8, 0x0, 0x0}, 1},
- {0x1CB9, {0x10F9, 0x0, 0x0}, 1},
- {0x1CBA, {0x10FA, 0x0, 0x0}, 1},
- {0x1CBD, {0x10FD, 0x0, 0x0}, 1},
- {0x1CBE, {0x10FE, 0x0, 0x0}, 1},
- {0x1CBF, {0x10FF, 0x0, 0x0}, 1},
- {0x1CD0, {0x0, 0x0, 0x0}, 0},
- {0x1CD1, {0x0, 0x0, 0x0}, 0},
- {0x1CD2, {0x0, 0x0, 0x0}, 0},
- {0x1CD4, {0x0, 0x0, 0x0}, 0},
- {0x1CD5, {0x0, 0x0, 0x0}, 0},
- {0x1CD6, {0x0, 0x0, 0x0}, 0},
- {0x1CD7, {0x0, 0x0, 0x0}, 0},
- {0x1CD8, {0x0, 0x0, 0x0}, 0},
- {0x1CD9, {0x0, 0x0, 0x0}, 0},
- {0x1CDA, {0x0, 0x0, 0x0}, 0},
- {0x1CDB, {0x0, 0x0, 0x0}, 0},
- {0x1CDC, {0x0, 0x0, 0x0}, 0},
- {0x1CDD, {0x0, 0x0, 0x0}, 0},
- {0x1CDE, {0x0, 0x0, 0x0}, 0},
- {0x1CDF, {0x0, 0x0, 0x0}, 0},
- {0x1CE0, {0x0, 0x0, 0x0}, 0},
- {0x1CE2, {0x0, 0x0, 0x0}, 0},
- {0x1CE3, {0x0, 0x0, 0x0}, 0},
- {0x1CE4, {0x0, 0x0, 0x0}, 0},
- {0x1CE5, {0x0, 0x0, 0x0}, 0},
- {0x1CE6, {0x0, 0x0, 0x0}, 0},
- {0x1CE7, {0x0, 0x0, 0x0}, 0},
- {0x1CE8, {0x0, 0x0, 0x0}, 0},
- {0x1CED, {0x0, 0x0, 0x0}, 0},
- {0x1CF4, {0x0, 0x0, 0x0}, 0},
- {0x1CF8, {0x0, 0x0, 0x0}, 0},
- {0x1CF9, {0x0, 0x0, 0x0}, 0},
- {0x1DC0, {0x0, 0x0, 0x0}, 0},
- {0x1DC1, {0x0, 0x0, 0x0}, 0},
- {0x1DC2, {0x0, 0x0, 0x0}, 0},
- {0x1DC3, {0x0, 0x0, 0x0}, 0},
- {0x1DC4, {0x0, 0x0, 0x0}, 0},
- {0x1DC5, {0x0, 0x0, 0x0}, 0},
- {0x1DC6, {0x0, 0x0, 0x0}, 0},
- {0x1DC7, {0x0, 0x0, 0x0}, 0},
- {0x1DC8, {0x0, 0x0, 0x0}, 0},
- {0x1DC9, {0x0, 0x0, 0x0}, 0},
- {0x1DCA, {0x0, 0x0, 0x0}, 0},
- {0x1DCB, {0x0, 0x0, 0x0}, 0},
- {0x1DCC, {0x0, 0x0, 0x0}, 0},
- {0x1DCD, {0x0, 0x0, 0x0}, 0},
- {0x1DCE, {0x0, 0x0, 0x0}, 0},
- {0x1DCF, {0x0, 0x0, 0x0}, 0},
- {0x1DD0, {0x0, 0x0, 0x0}, 0},
- {0x1DD1, {0x0, 0x0, 0x0}, 0},
- {0x1DD2, {0x0, 0x0, 0x0}, 0},
- {0x1DD3, {0x0, 0x0, 0x0}, 0},
- {0x1DD4, {0x0, 0x0, 0x0}, 0},
- {0x1DD5, {0x0, 0x0, 0x0}, 0},
- {0x1DD6, {0x0, 0x0, 0x0}, 0},
- {0x1DD7, {0x0, 0x0, 0x0}, 0},
- {0x1DD8, {0x0, 0x0, 0x0}, 0},
- {0x1DD9, {0x0, 0x0, 0x0}, 0},
- {0x1DDA, {0x0, 0x0, 0x0}, 0},
- {0x1DDB, {0x0, 0x0, 0x0}, 0},
- {0x1DDC, {0x0, 0x0, 0x0}, 0},
- {0x1DDD, {0x0, 0x0, 0x0}, 0},
- {0x1DDE, {0x0, 0x0, 0x0}, 0},
- {0x1DDF, {0x0, 0x0, 0x0}, 0},
- {0x1DE0, {0x0, 0x0, 0x0}, 0},
- {0x1DE1, {0x0, 0x0, 0x0}, 0},
- {0x1DE2, {0x0, 0x0, 0x0}, 0},
- {0x1DE3, {0x0, 0x0, 0x0}, 0},
- {0x1DE4, {0x0, 0x0, 0x0}, 0},
- {0x1DE5, {0x0, 0x0, 0x0}, 0},
- {0x1DE6, {0x0, 0x0, 0x0}, 0},
- {0x1DE7, {0x0, 0x0, 0x0}, 0},
- {0x1DE8, {0x0, 0x0, 0x0}, 0},
- {0x1DE9, {0x0, 0x0, 0x0}, 0},
- {0x1DEA, {0x0, 0x0, 0x0}, 0},
- {0x1DEB, {0x0, 0x0, 0x0}, 0},
- {0x1DEC, {0x0, 0x0, 0x0}, 0},
- {0x1DED, {0x0, 0x0, 0x0}, 0},
- {0x1DEE, {0x0, 0x0, 0x0}, 0},
- {0x1DEF, {0x0, 0x0, 0x0}, 0},
- {0x1DF0, {0x0, 0x0, 0x0}, 0},
- {0x1DF1, {0x0, 0x0, 0x0}, 0},
- {0x1DF2, {0x0, 0x0, 0x0}, 0},
- {0x1DF3, {0x0, 0x0, 0x0}, 0},
- {0x1DF4, {0x0, 0x0, 0x0}, 0},
- {0x1DF5, {0x0, 0x0, 0x0}, 0},
- {0x1DF6, {0x0, 0x0, 0x0}, 0},
- {0x1DF7, {0x0, 0x0, 0x0}, 0},
- {0x1DF8, {0x0, 0x0, 0x0}, 0},
- {0x1DF9, {0x0, 0x0, 0x0}, 0},
- {0x1DFB, {0x0, 0x0, 0x0}, 0},
- {0x1DFC, {0x0, 0x0, 0x0}, 0},
- {0x1DFD, {0x0, 0x0, 0x0}, 0},
- {0x1DFE, {0x0, 0x0, 0x0}, 0},
- {0x1DFF, {0x0, 0x0, 0x0}, 0},
- {0x1E00, {0x61, 0x0, 0x0}, 1},
- {0x1E01, {0x61, 0x0, 0x0}, 1},
- {0x1E02, {0x62, 0x0, 0x0}, 1},
- {0x1E03, {0x62, 0x0, 0x0}, 1},
- {0x1E04, {0x62, 0x0, 0x0}, 1},
- {0x1E05, {0x62, 0x0, 0x0}, 1},
- {0x1E06, {0x62, 0x0, 0x0}, 1},
- {0x1E07, {0x62, 0x0, 0x0}, 1},
- {0x1E08, {0x63, 0x0, 0x0}, 1},
- {0x1E09, {0x63, 0x0, 0x0}, 1},
- {0x1E0A, {0x64, 0x0, 0x0}, 1},
- {0x1E0B, {0x64, 0x0, 0x0}, 1},
- {0x1E0C, {0x64, 0x0, 0x0}, 1},
- {0x1E0D, {0x64, 0x0, 0x0}, 1},
- {0x1E0E, {0x64, 0x0, 0x0}, 1},
- {0x1E0F, {0x64, 0x0, 0x0}, 1},
- {0x1E10, {0x64, 0x0, 0x0}, 1},
- {0x1E11, {0x64, 0x0, 0x0}, 1},
- {0x1E12, {0x64, 0x0, 0x0}, 1},
- {0x1E13, {0x64, 0x0, 0x0}, 1},
- {0x1E14, {0x65, 0x0, 0x0}, 1},
- {0x1E15, {0x65, 0x0, 0x0}, 1},
- {0x1E16, {0x65, 0x0, 0x0}, 1},
- {0x1E17, {0x65, 0x0, 0x0}, 1},
- {0x1E18, {0x65, 0x0, 0x0}, 1},
- {0x1E19, {0x65, 0x0, 0x0}, 1},
- {0x1E1A, {0x65, 0x0, 0x0}, 1},
- {0x1E1B, {0x65, 0x0, 0x0}, 1},
- {0x1E1C, {0x65, 0x0, 0x0}, 1},
- {0x1E1D, {0x65, 0x0, 0x0}, 1},
- {0x1E1E, {0x66, 0x0, 0x0}, 1},
- {0x1E1F, {0x66, 0x0, 0x0}, 1},
- {0x1E20, {0x67, 0x0, 0x0}, 1},
- {0x1E21, {0x67, 0x0, 0x0}, 1},
- {0x1E22, {0x68, 0x0, 0x0}, 1},
- {0x1E23, {0x68, 0x0, 0x0}, 1},
- {0x1E24, {0x68, 0x0, 0x0}, 1},
- {0x1E25, {0x68, 0x0, 0x0}, 1},
- {0x1E26, {0x68, 0x0, 0x0}, 1},
- {0x1E27, {0x68, 0x0, 0x0}, 1},
- {0x1E28, {0x68, 0x0, 0x0}, 1},
- {0x1E29, {0x68, 0x0, 0x0}, 1},
- {0x1E2A, {0x68, 0x0, 0x0}, 1},
- {0x1E2B, {0x68, 0x0, 0x0}, 1},
- {0x1E2C, {0x69, 0x0, 0x0}, 1},
- {0x1E2D, {0x69, 0x0, 0x0}, 1},
- {0x1E2E, {0x69, 0x0, 0x0}, 1},
- {0x1E2F, {0x69, 0x0, 0x0}, 1},
- {0x1E30, {0x6B, 0x0, 0x0}, 1},
- {0x1E31, {0x6B, 0x0, 0x0}, 1},
- {0x1E32, {0x6B, 0x0, 0x0}, 1},
- {0x1E33, {0x6B, 0x0, 0x0}, 1},
- {0x1E34, {0x6B, 0x0, 0x0}, 1},
- {0x1E35, {0x6B, 0x0, 0x0}, 1},
- {0x1E36, {0x6C, 0x0, 0x0}, 1},
- {0x1E37, {0x6C, 0x0, 0x0}, 1},
- {0x1E38, {0x6C, 0x0, 0x0}, 1},
- {0x1E39, {0x6C, 0x0, 0x0}, 1},
- {0x1E3A, {0x6C, 0x0, 0x0}, 1},
- {0x1E3B, {0x6C, 0x0, 0x0}, 1},
- {0x1E3C, {0x6C, 0x0, 0x0}, 1},
- {0x1E3D, {0x6C, 0x0, 0x0}, 1},
- {0x1E3E, {0x6D, 0x0, 0x0}, 1},
- {0x1E3F, {0x6D, 0x0, 0x0}, 1},
- {0x1E40, {0x6D, 0x0, 0x0}, 1},
- {0x1E41, {0x6D, 0x0, 0x0}, 1},
- {0x1E42, {0x6D, 0x0, 0x0}, 1},
- {0x1E43, {0x6D, 0x0, 0x0}, 1},
- {0x1E44, {0x6E, 0x0, 0x0}, 1},
- {0x1E45, {0x6E, 0x0, 0x0}, 1},
- {0x1E46, {0x6E, 0x0, 0x0}, 1},
- {0x1E47, {0x6E, 0x0, 0x0}, 1},
- {0x1E48, {0x6E, 0x0, 0x0}, 1},
- {0x1E49, {0x6E, 0x0, 0x0}, 1},
- {0x1E4A, {0x6E, 0x0, 0x0}, 1},
- {0x1E4B, {0x6E, 0x0, 0x0}, 1},
- {0x1E4C, {0x6F, 0x0, 0x0}, 1},
- {0x1E4D, {0x6F, 0x0, 0x0}, 1},
- {0x1E4E, {0x6F, 0x0, 0x0}, 1},
- {0x1E4F, {0x6F, 0x0, 0x0}, 1},
- {0x1E50, {0x6F, 0x0, 0x0}, 1},
- {0x1E51, {0x6F, 0x0, 0x0}, 1},
- {0x1E52, {0x6F, 0x0, 0x0}, 1},
- {0x1E53, {0x6F, 0x0, 0x0}, 1},
- {0x1E54, {0x70, 0x0, 0x0}, 1},
- {0x1E55, {0x70, 0x0, 0x0}, 1},
- {0x1E56, {0x70, 0x0, 0x0}, 1},
- {0x1E57, {0x70, 0x0, 0x0}, 1},
- {0x1E58, {0x72, 0x0, 0x0}, 1},
- {0x1E59, {0x72, 0x0, 0x0}, 1},
- {0x1E5A, {0x72, 0x0, 0x0}, 1},
- {0x1E5B, {0x72, 0x0, 0x0}, 1},
- {0x1E5C, {0x72, 0x0, 0x0}, 1},
- {0x1E5D, {0x72, 0x0, 0x0}, 1},
- {0x1E5E, {0x72, 0x0, 0x0}, 1},
- {0x1E5F, {0x72, 0x0, 0x0}, 1},
- {0x1E60, {0x73, 0x0, 0x0}, 1},
- {0x1E61, {0x73, 0x0, 0x0}, 1},
- {0x1E62, {0x73, 0x0, 0x0}, 1},
- {0x1E63, {0x73, 0x0, 0x0}, 1},
- {0x1E64, {0x73, 0x0, 0x0}, 1},
- {0x1E65, {0x73, 0x0, 0x0}, 1},
- {0x1E66, {0x73, 0x0, 0x0}, 1},
- {0x1E67, {0x73, 0x0, 0x0}, 1},
- {0x1E68, {0x73, 0x0, 0x0}, 1},
- {0x1E69, {0x73, 0x0, 0x0}, 1},
- {0x1E6A, {0x74, 0x0, 0x0}, 1},
- {0x1E6B, {0x74, 0x0, 0x0}, 1},
- {0x1E6C, {0x74, 0x0, 0x0}, 1},
- {0x1E6D, {0x74, 0x0, 0x0}, 1},
- {0x1E6E, {0x74, 0x0, 0x0}, 1},
- {0x1E6F, {0x74, 0x0, 0x0}, 1},
- {0x1E70, {0x74, 0x0, 0x0}, 1},
- {0x1E71, {0x74, 0x0, 0x0}, 1},
- {0x1E72, {0x75, 0x0, 0x0}, 1},
- {0x1E73, {0x75, 0x0, 0x0}, 1},
- {0x1E74, {0x75, 0x0, 0x0}, 1},
- {0x1E75, {0x75, 0x0, 0x0}, 1},
- {0x1E76, {0x75, 0x0, 0x0}, 1},
- {0x1E77, {0x75, 0x0, 0x0}, 1},
- {0x1E78, {0x75, 0x0, 0x0}, 1},
- {0x1E79, {0x75, 0x0, 0x0}, 1},
- {0x1E7A, {0x75, 0x0, 0x0}, 1},
- {0x1E7B, {0x75, 0x0, 0x0}, 1},
- {0x1E7C, {0x76, 0x0, 0x0}, 1},
- {0x1E7D, {0x76, 0x0, 0x0}, 1},
- {0x1E7E, {0x76, 0x0, 0x0}, 1},
- {0x1E7F, {0x76, 0x0, 0x0}, 1},
- {0x1E80, {0x77, 0x0, 0x0}, 1},
- {0x1E81, {0x77, 0x0, 0x0}, 1},
- {0x1E82, {0x77, 0x0, 0x0}, 1},
- {0x1E83, {0x77, 0x0, 0x0}, 1},
- {0x1E84, {0x77, 0x0, 0x0}, 1},
- {0x1E85, {0x77, 0x0, 0x0}, 1},
- {0x1E86, {0x77, 0x0, 0x0}, 1},
- {0x1E87, {0x77, 0x0, 0x0}, 1},
- {0x1E88, {0x77, 0x0, 0x0}, 1},
- {0x1E89, {0x77, 0x0, 0x0}, 1},
- {0x1E8A, {0x78, 0x0, 0x0}, 1},
- {0x1E8B, {0x78, 0x0, 0x0}, 1},
- {0x1E8C, {0x78, 0x0, 0x0}, 1},
- {0x1E8D, {0x78, 0x0, 0x0}, 1},
- {0x1E8E, {0x79, 0x0, 0x0}, 1},
- {0x1E8F, {0x79, 0x0, 0x0}, 1},
- {0x1E90, {0x7A, 0x0, 0x0}, 1},
- {0x1E91, {0x7A, 0x0, 0x0}, 1},
- {0x1E92, {0x7A, 0x0, 0x0}, 1},
- {0x1E93, {0x7A, 0x0, 0x0}, 1},
- {0x1E94, {0x7A, 0x0, 0x0}, 1},
- {0x1E95, {0x7A, 0x0, 0x0}, 1},
- {0x1E96, {0x68, 0x0, 0x0}, 1},
- {0x1E97, {0x74, 0x0, 0x0}, 1},
- {0x1E98, {0x77, 0x0, 0x0}, 1},
- {0x1E99, {0x79, 0x0, 0x0}, 1},
- {0x1E9B, {0x17F, 0x0, 0x0}, 1},
- {0x1E9E, {0xDF, 0x0, 0x0}, 1},
- {0x1EA0, {0x61, 0x0, 0x0}, 1},
- {0x1EA1, {0x61, 0x0, 0x0}, 1},
- {0x1EA2, {0x61, 0x0, 0x0}, 1},
- {0x1EA3, {0x61, 0x0, 0x0}, 1},
- {0x1EA4, {0x61, 0x0, 0x0}, 1},
- {0x1EA5, {0x61, 0x0, 0x0}, 1},
- {0x1EA6, {0x61, 0x0, 0x0}, 1},
- {0x1EA7, {0x61, 0x0, 0x0}, 1},
- {0x1EA8, {0x61, 0x0, 0x0}, 1},
- {0x1EA9, {0x61, 0x0, 0x0}, 1},
- {0x1EAA, {0x61, 0x0, 0x0}, 1},
- {0x1EAB, {0x61, 0x0, 0x0}, 1},
- {0x1EAC, {0x61, 0x0, 0x0}, 1},
- {0x1EAD, {0x61, 0x0, 0x0}, 1},
- {0x1EAE, {0x61, 0x0, 0x0}, 1},
- {0x1EAF, {0x61, 0x0, 0x0}, 1},
- {0x1EB0, {0x61, 0x0, 0x0}, 1},
- {0x1EB1, {0x61, 0x0, 0x0}, 1},
- {0x1EB2, {0x61, 0x0, 0x0}, 1},
- {0x1EB3, {0x61, 0x0, 0x0}, 1},
- {0x1EB4, {0x61, 0x0, 0x0}, 1},
- {0x1EB5, {0x61, 0x0, 0x0}, 1},
- {0x1EB6, {0x61, 0x0, 0x0}, 1},
- {0x1EB7, {0x61, 0x0, 0x0}, 1},
- {0x1EB8, {0x65, 0x0, 0x0}, 1},
- {0x1EB9, {0x65, 0x0, 0x0}, 1},
- {0x1EBA, {0x65, 0x0, 0x0}, 1},
- {0x1EBB, {0x65, 0x0, 0x0}, 1},
- {0x1EBC, {0x65, 0x0, 0x0}, 1},
- {0x1EBD, {0x65, 0x0, 0x0}, 1},
- {0x1EBE, {0x65, 0x0, 0x0}, 1},
- {0x1EBF, {0x65, 0x0, 0x0}, 1},
- {0x1EC0, {0x65, 0x0, 0x0}, 1},
- {0x1EC1, {0x65, 0x0, 0x0}, 1},
- {0x1EC2, {0x65, 0x0, 0x0}, 1},
- {0x1EC3, {0x65, 0x0, 0x0}, 1},
- {0x1EC4, {0x65, 0x0, 0x0}, 1},
- {0x1EC5, {0x65, 0x0, 0x0}, 1},
- {0x1EC6, {0x65, 0x0, 0x0}, 1},
- {0x1EC7, {0x65, 0x0, 0x0}, 1},
- {0x1EC8, {0x69, 0x0, 0x0}, 1},
- {0x1EC9, {0x69, 0x0, 0x0}, 1},
- {0x1ECA, {0x69, 0x0, 0x0}, 1},
- {0x1ECB, {0x69, 0x0, 0x0}, 1},
- {0x1ECC, {0x6F, 0x0, 0x0}, 1},
- {0x1ECD, {0x6F, 0x0, 0x0}, 1},
- {0x1ECE, {0x6F, 0x0, 0x0}, 1},
- {0x1ECF, {0x6F, 0x0, 0x0}, 1},
- {0x1ED0, {0x6F, 0x0, 0x0}, 1},
- {0x1ED1, {0x6F, 0x0, 0x0}, 1},
- {0x1ED2, {0x6F, 0x0, 0x0}, 1},
- {0x1ED3, {0x6F, 0x0, 0x0}, 1},
- {0x1ED4, {0x6F, 0x0, 0x0}, 1},
- {0x1ED5, {0x6F, 0x0, 0x0}, 1},
- {0x1ED6, {0x6F, 0x0, 0x0}, 1},
- {0x1ED7, {0x6F, 0x0, 0x0}, 1},
- {0x1ED8, {0x6F, 0x0, 0x0}, 1},
- {0x1ED9, {0x6F, 0x0, 0x0}, 1},
- {0x1EDA, {0x6F, 0x0, 0x0}, 1},
- {0x1EDB, {0x6F, 0x0, 0x0}, 1},
- {0x1EDC, {0x6F, 0x0, 0x0}, 1},
- {0x1EDD, {0x6F, 0x0, 0x0}, 1},
- {0x1EDE, {0x6F, 0x0, 0x0}, 1},
- {0x1EDF, {0x6F, 0x0, 0x0}, 1},
- {0x1EE0, {0x6F, 0x0, 0x0}, 1},
- {0x1EE1, {0x6F, 0x0, 0x0}, 1},
- {0x1EE2, {0x6F, 0x0, 0x0}, 1},
- {0x1EE3, {0x6F, 0x0, 0x0}, 1},
- {0x1EE4, {0x75, 0x0, 0x0}, 1},
- {0x1EE5, {0x75, 0x0, 0x0}, 1},
- {0x1EE6, {0x75, 0x0, 0x0}, 1},
- {0x1EE7, {0x75, 0x0, 0x0}, 1},
- {0x1EE8, {0x75, 0x0, 0x0}, 1},
- {0x1EE9, {0x75, 0x0, 0x0}, 1},
- {0x1EEA, {0x75, 0x0, 0x0}, 1},
- {0x1EEB, {0x75, 0x0, 0x0}, 1},
- {0x1EEC, {0x75, 0x0, 0x0}, 1},
- {0x1EED, {0x75, 0x0, 0x0}, 1},
- {0x1EEE, {0x75, 0x0, 0x0}, 1},
- {0x1EEF, {0x75, 0x0, 0x0}, 1},
- {0x1EF0, {0x75, 0x0, 0x0}, 1},
- {0x1EF1, {0x75, 0x0, 0x0}, 1},
- {0x1EF2, {0x79, 0x0, 0x0}, 1},
- {0x1EF3, {0x79, 0x0, 0x0}, 1},
- {0x1EF4, {0x79, 0x0, 0x0}, 1},
- {0x1EF5, {0x79, 0x0, 0x0}, 1},
- {0x1EF6, {0x79, 0x0, 0x0}, 1},
- {0x1EF7, {0x79, 0x0, 0x0}, 1},
- {0x1EF8, {0x79, 0x0, 0x0}, 1},
- {0x1EF9, {0x79, 0x0, 0x0}, 1},
- {0x1EFA, {0x1EFB, 0x0, 0x0}, 1},
- {0x1EFC, {0x1EFD, 0x0, 0x0}, 1},
- {0x1EFE, {0x1EFF, 0x0, 0x0}, 1},
- {0x1F00, {0x3B1, 0x0, 0x0}, 1},
- {0x1F01, {0x3B1, 0x0, 0x0}, 1},
- {0x1F02, {0x3B1, 0x0, 0x0}, 1},
- {0x1F03, {0x3B1, 0x0, 0x0}, 1},
- {0x1F04, {0x3B1, 0x0, 0x0}, 1},
- {0x1F05, {0x3B1, 0x0, 0x0}, 1},
- {0x1F06, {0x3B1, 0x0, 0x0}, 1},
- {0x1F07, {0x3B1, 0x0, 0x0}, 1},
- {0x1F08, {0x3B1, 0x0, 0x0}, 1},
- {0x1F09, {0x3B1, 0x0, 0x0}, 1},
- {0x1F0A, {0x3B1, 0x0, 0x0}, 1},
- {0x1F0B, {0x3B1, 0x0, 0x0}, 1},
- {0x1F0C, {0x3B1, 0x0, 0x0}, 1},
- {0x1F0D, {0x3B1, 0x0, 0x0}, 1},
- {0x1F0E, {0x3B1, 0x0, 0x0}, 1},
- {0x1F0F, {0x3B1, 0x0, 0x0}, 1},
- {0x1F10, {0x3B5, 0x0, 0x0}, 1},
- {0x1F11, {0x3B5, 0x0, 0x0}, 1},
- {0x1F12, {0x3B5, 0x0, 0x0}, 1},
- {0x1F13, {0x3B5, 0x0, 0x0}, 1},
- {0x1F14, {0x3B5, 0x0, 0x0}, 1},
- {0x1F15, {0x3B5, 0x0, 0x0}, 1},
- {0x1F18, {0x3B5, 0x0, 0x0}, 1},
- {0x1F19, {0x3B5, 0x0, 0x0}, 1},
- {0x1F1A, {0x3B5, 0x0, 0x0}, 1},
- {0x1F1B, {0x3B5, 0x0, 0x0}, 1},
- {0x1F1C, {0x3B5, 0x0, 0x0}, 1},
- {0x1F1D, {0x3B5, 0x0, 0x0}, 1},
- {0x1F20, {0x3B7, 0x0, 0x0}, 1},
- {0x1F21, {0x3B7, 0x0, 0x0}, 1},
- {0x1F22, {0x3B7, 0x0, 0x0}, 1},
- {0x1F23, {0x3B7, 0x0, 0x0}, 1},
- {0x1F24, {0x3B7, 0x0, 0x0}, 1},
- {0x1F25, {0x3B7, 0x0, 0x0}, 1},
- {0x1F26, {0x3B7, 0x0, 0x0}, 1},
- {0x1F27, {0x3B7, 0x0, 0x0}, 1},
- {0x1F28, {0x3B7, 0x0, 0x0}, 1},
- {0x1F29, {0x3B7, 0x0, 0x0}, 1},
- {0x1F2A, {0x3B7, 0x0, 0x0}, 1},
- {0x1F2B, {0x3B7, 0x0, 0x0}, 1},
- {0x1F2C, {0x3B7, 0x0, 0x0}, 1},
- {0x1F2D, {0x3B7, 0x0, 0x0}, 1},
- {0x1F2E, {0x3B7, 0x0, 0x0}, 1},
- {0x1F2F, {0x3B7, 0x0, 0x0}, 1},
- {0x1F30, {0x3B9, 0x0, 0x0}, 1},
- {0x1F31, {0x3B9, 0x0, 0x0}, 1},
- {0x1F32, {0x3B9, 0x0, 0x0}, 1},
- {0x1F33, {0x3B9, 0x0, 0x0}, 1},
- {0x1F34, {0x3B9, 0x0, 0x0}, 1},
- {0x1F35, {0x3B9, 0x0, 0x0}, 1},
- {0x1F36, {0x3B9, 0x0, 0x0}, 1},
- {0x1F37, {0x3B9, 0x0, 0x0}, 1},
- {0x1F38, {0x3B9, 0x0, 0x0}, 1},
- {0x1F39, {0x3B9, 0x0, 0x0}, 1},
- {0x1F3A, {0x3B9, 0x0, 0x0}, 1},
- {0x1F3B, {0x3B9, 0x0, 0x0}, 1},
- {0x1F3C, {0x3B9, 0x0, 0x0}, 1},
- {0x1F3D, {0x3B9, 0x0, 0x0}, 1},
- {0x1F3E, {0x3B9, 0x0, 0x0}, 1},
- {0x1F3F, {0x3B9, 0x0, 0x0}, 1},
- {0x1F40, {0x3BF, 0x0, 0x0}, 1},
- {0x1F41, {0x3BF, 0x0, 0x0}, 1},
- {0x1F42, {0x3BF, 0x0, 0x0}, 1},
- {0x1F43, {0x3BF, 0x0, 0x0}, 1},
- {0x1F44, {0x3BF, 0x0, 0x0}, 1},
- {0x1F45, {0x3BF, 0x0, 0x0}, 1},
- {0x1F48, {0x3BF, 0x0, 0x0}, 1},
- {0x1F49, {0x3BF, 0x0, 0x0}, 1},
- {0x1F4A, {0x3BF, 0x0, 0x0}, 1},
- {0x1F4B, {0x3BF, 0x0, 0x0}, 1},
- {0x1F4C, {0x3BF, 0x0, 0x0}, 1},
- {0x1F4D, {0x3BF, 0x0, 0x0}, 1},
- {0x1F50, {0x3C5, 0x0, 0x0}, 1},
- {0x1F51, {0x3C5, 0x0, 0x0}, 1},
- {0x1F52, {0x3C5, 0x0, 0x0}, 1},
- {0x1F53, {0x3C5, 0x0, 0x0}, 1},
- {0x1F54, {0x3C5, 0x0, 0x0}, 1},
- {0x1F55, {0x3C5, 0x0, 0x0}, 1},
- {0x1F56, {0x3C5, 0x0, 0x0}, 1},
- {0x1F57, {0x3C5, 0x0, 0x0}, 1},
- {0x1F59, {0x3C5, 0x0, 0x0}, 1},
- {0x1F5B, {0x3C5, 0x0, 0x0}, 1},
- {0x1F5D, {0x3C5, 0x0, 0x0}, 1},
- {0x1F5F, {0x3C5, 0x0, 0x0}, 1},
- {0x1F60, {0x3C9, 0x0, 0x0}, 1},
- {0x1F61, {0x3C9, 0x0, 0x0}, 1},
- {0x1F62, {0x3C9, 0x0, 0x0}, 1},
- {0x1F63, {0x3C9, 0x0, 0x0}, 1},
- {0x1F64, {0x3C9, 0x0, 0x0}, 1},
- {0x1F65, {0x3C9, 0x0, 0x0}, 1},
- {0x1F66, {0x3C9, 0x0, 0x0}, 1},
- {0x1F67, {0x3C9, 0x0, 0x0}, 1},
- {0x1F68, {0x3C9, 0x0, 0x0}, 1},
- {0x1F69, {0x3C9, 0x0, 0x0}, 1},
- {0x1F6A, {0x3C9, 0x0, 0x0}, 1},
- {0x1F6B, {0x3C9, 0x0, 0x0}, 1},
- {0x1F6C, {0x3C9, 0x0, 0x0}, 1},
- {0x1F6D, {0x3C9, 0x0, 0x0}, 1},
- {0x1F6E, {0x3C9, 0x0, 0x0}, 1},
- {0x1F6F, {0x3C9, 0x0, 0x0}, 1},
- {0x1F70, {0x3B1, 0x0, 0x0}, 1},
- {0x1F71, {0x3B1, 0x0, 0x0}, 1},
- {0x1F72, {0x3B5, 0x0, 0x0}, 1},
- {0x1F73, {0x3B5, 0x0, 0x0}, 1},
- {0x1F74, {0x3B7, 0x0, 0x0}, 1},
- {0x1F75, {0x3B7, 0x0, 0x0}, 1},
- {0x1F76, {0x3B9, 0x0, 0x0}, 1},
- {0x1F77, {0x3B9, 0x0, 0x0}, 1},
- {0x1F78, {0x3BF, 0x0, 0x0}, 1},
- {0x1F79, {0x3BF, 0x0, 0x0}, 1},
- {0x1F7A, {0x3C5, 0x0, 0x0}, 1},
- {0x1F7B, {0x3C5, 0x0, 0x0}, 1},
- {0x1F7C, {0x3C9, 0x0, 0x0}, 1},
- {0x1F7D, {0x3C9, 0x0, 0x0}, 1},
- {0x1F80, {0x3B1, 0x0, 0x0}, 1},
- {0x1F81, {0x3B1, 0x0, 0x0}, 1},
- {0x1F82, {0x3B1, 0x0, 0x0}, 1},
- {0x1F83, {0x3B1, 0x0, 0x0}, 1},
- {0x1F84, {0x3B1, 0x0, 0x0}, 1},
- {0x1F85, {0x3B1, 0x0, 0x0}, 1},
- {0x1F86, {0x3B1, 0x0, 0x0}, 1},
- {0x1F87, {0x3B1, 0x0, 0x0}, 1},
- {0x1F88, {0x3B1, 0x0, 0x0}, 1},
- {0x1F89, {0x3B1, 0x0, 0x0}, 1},
- {0x1F8A, {0x3B1, 0x0, 0x0}, 1},
- {0x1F8B, {0x3B1, 0x0, 0x0}, 1},
- {0x1F8C, {0x3B1, 0x0, 0x0}, 1},
- {0x1F8D, {0x3B1, 0x0, 0x0}, 1},
- {0x1F8E, {0x3B1, 0x0, 0x0}, 1},
- {0x1F8F, {0x3B1, 0x0, 0x0}, 1},
- {0x1F90, {0x3B7, 0x0, 0x0}, 1},
- {0x1F91, {0x3B7, 0x0, 0x0}, 1},
- {0x1F92, {0x3B7, 0x0, 0x0}, 1},
- {0x1F93, {0x3B7, 0x0, 0x0}, 1},
- {0x1F94, {0x3B7, 0x0, 0x0}, 1},
- {0x1F95, {0x3B7, 0x0, 0x0}, 1},
- {0x1F96, {0x3B7, 0x0, 0x0}, 1},
- {0x1F97, {0x3B7, 0x0, 0x0}, 1},
- {0x1F98, {0x3B7, 0x0, 0x0}, 1},
- {0x1F99, {0x3B7, 0x0, 0x0}, 1},
- {0x1F9A, {0x3B7, 0x0, 0x0}, 1},
- {0x1F9B, {0x3B7, 0x0, 0x0}, 1},
- {0x1F9C, {0x3B7, 0x0, 0x0}, 1},
- {0x1F9D, {0x3B7, 0x0, 0x0}, 1},
- {0x1F9E, {0x3B7, 0x0, 0x0}, 1},
- {0x1F9F, {0x3B7, 0x0, 0x0}, 1},
- {0x1FA0, {0x3C9, 0x0, 0x0}, 1},
- {0x1FA1, {0x3C9, 0x0, 0x0}, 1},
- {0x1FA2, {0x3C9, 0x0, 0x0}, 1},
- {0x1FA3, {0x3C9, 0x0, 0x0}, 1},
- {0x1FA4, {0x3C9, 0x0, 0x0}, 1},
- {0x1FA5, {0x3C9, 0x0, 0x0}, 1},
- {0x1FA6, {0x3C9, 0x0, 0x0}, 1},
- {0x1FA7, {0x3C9, 0x0, 0x0}, 1},
- {0x1FA8, {0x3C9, 0x0, 0x0}, 1},
- {0x1FA9, {0x3C9, 0x0, 0x0}, 1},
- {0x1FAA, {0x3C9, 0x0, 0x0}, 1},
- {0x1FAB, {0x3C9, 0x0, 0x0}, 1},
- {0x1FAC, {0x3C9, 0x0, 0x0}, 1},
- {0x1FAD, {0x3C9, 0x0, 0x0}, 1},
- {0x1FAE, {0x3C9, 0x0, 0x0}, 1},
- {0x1FAF, {0x3C9, 0x0, 0x0}, 1},
- {0x1FB0, {0x3B1, 0x0, 0x0}, 1},
- {0x1FB1, {0x3B1, 0x0, 0x0}, 1},
- {0x1FB2, {0x3B1, 0x0, 0x0}, 1},
- {0x1FB3, {0x3B1, 0x0, 0x0}, 1},
- {0x1FB4, {0x3B1, 0x0, 0x0}, 1},
- {0x1FB6, {0x3B1, 0x0, 0x0}, 1},
- {0x1FB7, {0x3B1, 0x0, 0x0}, 1},
- {0x1FB8, {0x3B1, 0x0, 0x0}, 1},
- {0x1FB9, {0x3B1, 0x0, 0x0}, 1},
- {0x1FBA, {0x3B1, 0x0, 0x0}, 1},
- {0x1FBB, {0x3B1, 0x0, 0x0}, 1},
- {0x1FBC, {0x3B1, 0x0, 0x0}, 1},
- {0x1FBE, {0x3B9, 0x0, 0x0}, 1},
- {0x1FC1, {0xA8, 0x0, 0x0}, 1},
- {0x1FC2, {0x3B7, 0x0, 0x0}, 1},
- {0x1FC3, {0x3B7, 0x0, 0x0}, 1},
- {0x1FC4, {0x3B7, 0x0, 0x0}, 1},
- {0x1FC6, {0x3B7, 0x0, 0x0}, 1},
- {0x1FC7, {0x3B7, 0x0, 0x0}, 1},
- {0x1FC8, {0x3B5, 0x0, 0x0}, 1},
- {0x1FC9, {0x3B5, 0x0, 0x0}, 1},
- {0x1FCA, {0x3B7, 0x0, 0x0}, 1},
- {0x1FCB, {0x3B7, 0x0, 0x0}, 1},
- {0x1FCC, {0x3B7, 0x0, 0x0}, 1},
- {0x1FCD, {0x1FBF, 0x0, 0x0}, 1},
- {0x1FCE, {0x1FBF, 0x0, 0x0}, 1},
- {0x1FCF, {0x1FBF, 0x0, 0x0}, 1},
- {0x1FD0, {0x3B9, 0x0, 0x0}, 1},
- {0x1FD1, {0x3B9, 0x0, 0x0}, 1},
- {0x1FD2, {0x3B9, 0x0, 0x0}, 1},
- {0x1FD3, {0x3B9, 0x0, 0x0}, 1},
- {0x1FD6, {0x3B9, 0x0, 0x0}, 1},
- {0x1FD7, {0x3B9, 0x0, 0x0}, 1},
- {0x1FD8, {0x3B9, 0x0, 0x0}, 1},
- {0x1FD9, {0x3B9, 0x0, 0x0}, 1},
- {0x1FDA, {0x3B9, 0x0, 0x0}, 1},
- {0x1FDB, {0x3B9, 0x0, 0x0}, 1},
- {0x1FDD, {0x1FFE, 0x0, 0x0}, 1},
- {0x1FDE, {0x1FFE, 0x0, 0x0}, 1},
- {0x1FDF, {0x1FFE, 0x0, 0x0}, 1},
- {0x1FE0, {0x3C5, 0x0, 0x0}, 1},
- {0x1FE1, {0x3C5, 0x0, 0x0}, 1},
- {0x1FE2, {0x3C5, 0x0, 0x0}, 1},
- {0x1FE3, {0x3C5, 0x0, 0x0}, 1},
- {0x1FE4, {0x3C1, 0x0, 0x0}, 1},
- {0x1FE5, {0x3C1, 0x0, 0x0}, 1},
- {0x1FE6, {0x3C5, 0x0, 0x0}, 1},
- {0x1FE7, {0x3C5, 0x0, 0x0}, 1},
- {0x1FE8, {0x3C5, 0x0, 0x0}, 1},
- {0x1FE9, {0x3C5, 0x0, 0x0}, 1},
- {0x1FEA, {0x3C5, 0x0, 0x0}, 1},
- {0x1FEB, {0x3C5, 0x0, 0x0}, 1},
- {0x1FEC, {0x3C1, 0x0, 0x0}, 1},
- {0x1FED, {0xA8, 0x0, 0x0}, 1},
- {0x1FEE, {0xA8, 0x0, 0x0}, 1},
- {0x1FEF, {0x60, 0x0, 0x0}, 1},
- {0x1FF2, {0x3C9, 0x0, 0x0}, 1},
- {0x1FF3, {0x3C9, 0x0, 0x0}, 1},
- {0x1FF4, {0x3C9, 0x0, 0x0}, 1},
- {0x1FF6, {0x3C9, 0x0, 0x0}, 1},
- {0x1FF7, {0x3C9, 0x0, 0x0}, 1},
- {0x1FF8, {0x3BF, 0x0, 0x0}, 1},
- {0x1FF9, {0x3BF, 0x0, 0x0}, 1},
- {0x1FFA, {0x3C9, 0x0, 0x0}, 1},
- {0x1FFB, {0x3C9, 0x0, 0x0}, 1},
- {0x1FFC, {0x3C9, 0x0, 0x0}, 1},
- {0x1FFD, {0xB4, 0x0, 0x0}, 1},
- {0x2000, {0x2002, 0x0, 0x0}, 1},
- {0x2001, {0x2003, 0x0, 0x0}, 1},
- {0x20D0, {0x0, 0x0, 0x0}, 0},
- {0x20D1, {0x0, 0x0, 0x0}, 0},
- {0x20D2, {0x0, 0x0, 0x0}, 0},
- {0x20D3, {0x0, 0x0, 0x0}, 0},
- {0x20D4, {0x0, 0x0, 0x0}, 0},
- {0x20D5, {0x0, 0x0, 0x0}, 0},
- {0x20D6, {0x0, 0x0, 0x0}, 0},
- {0x20D7, {0x0, 0x0, 0x0}, 0},
- {0x20D8, {0x0, 0x0, 0x0}, 0},
- {0x20D9, {0x0, 0x0, 0x0}, 0},
- {0x20DA, {0x0, 0x0, 0x0}, 0},
- {0x20DB, {0x0, 0x0, 0x0}, 0},
- {0x20DC, {0x0, 0x0, 0x0}, 0},
- {0x20E1, {0x0, 0x0, 0x0}, 0},
- {0x20E5, {0x0, 0x0, 0x0}, 0},
- {0x20E6, {0x0, 0x0, 0x0}, 0},
- {0x20E7, {0x0, 0x0, 0x0}, 0},
- {0x20E8, {0x0, 0x0, 0x0}, 0},
- {0x20E9, {0x0, 0x0, 0x0}, 0},
- {0x20EA, {0x0, 0x0, 0x0}, 0},
- {0x20EB, {0x0, 0x0, 0x0}, 0},
- {0x20EC, {0x0, 0x0, 0x0}, 0},
- {0x20ED, {0x0, 0x0, 0x0}, 0},
- {0x20EE, {0x0, 0x0, 0x0}, 0},
- {0x20EF, {0x0, 0x0, 0x0}, 0},
- {0x20F0, {0x0, 0x0, 0x0}, 0},
- {0x2126, {0x3C9, 0x0, 0x0}, 1},
- {0x212A, {0x6B, 0x0, 0x0}, 1},
- {0x212B, {0x61, 0x0, 0x0}, 1},
- {0x2132, {0x214E, 0x0, 0x0}, 1},
- {0x2160, {0x2170, 0x0, 0x0}, 1},
- {0x2161, {0x2171, 0x0, 0x0}, 1},
- {0x2162, {0x2172, 0x0, 0x0}, 1},
- {0x2163, {0x2173, 0x0, 0x0}, 1},
- {0x2164, {0x2174, 0x0, 0x0}, 1},
- {0x2165, {0x2175, 0x0, 0x0}, 1},
- {0x2166, {0x2176, 0x0, 0x0}, 1},
- {0x2167, {0x2177, 0x0, 0x0}, 1},
- {0x2168, {0x2178, 0x0, 0x0}, 1},
- {0x2169, {0x2179, 0x0, 0x0}, 1},
- {0x216A, {0x217A, 0x0, 0x0}, 1},
- {0x216B, {0x217B, 0x0, 0x0}, 1},
- {0x216C, {0x217C, 0x0, 0x0}, 1},
- {0x216D, {0x217D, 0x0, 0x0}, 1},
- {0x216E, {0x217E, 0x0, 0x0}, 1},
- {0x216F, {0x217F, 0x0, 0x0}, 1},
- {0x2183, {0x2184, 0x0, 0x0}, 1},
- {0x219A, {0x2190, 0x0, 0x0}, 1},
- {0x219B, {0x2192, 0x0, 0x0}, 1},
- {0x21AE, {0x2194, 0x0, 0x0}, 1},
- {0x21CD, {0x21D0, 0x0, 0x0}, 1},
- {0x21CE, {0x21D4, 0x0, 0x0}, 1},
- {0x21CF, {0x21D2, 0x0, 0x0}, 1},
- {0x2204, {0x2203, 0x0, 0x0}, 1},
- {0x2209, {0x2208, 0x0, 0x0}, 1},
- {0x220C, {0x220B, 0x0, 0x0}, 1},
- {0x2224, {0x2223, 0x0, 0x0}, 1},
- {0x2226, {0x2225, 0x0, 0x0}, 1},
- {0x2241, {0x223C, 0x0, 0x0}, 1},
- {0x2244, {0x2243, 0x0, 0x0}, 1},
- {0x2247, {0x2245, 0x0, 0x0}, 1},
- {0x2249, {0x2248, 0x0, 0x0}, 1},
- {0x2260, {0x3D, 0x0, 0x0}, 1},
- {0x2262, {0x2261, 0x0, 0x0}, 1},
- {0x226D, {0x224D, 0x0, 0x0}, 1},
- {0x226E, {0x3C, 0x0, 0x0}, 1},
- {0x226F, {0x3E, 0x0, 0x0}, 1},
- {0x2270, {0x2264, 0x0, 0x0}, 1},
- {0x2271, {0x2265, 0x0, 0x0}, 1},
- {0x2274, {0x2272, 0x0, 0x0}, 1},
- {0x2275, {0x2273, 0x0, 0x0}, 1},
- {0x2278, {0x2276, 0x0, 0x0}, 1},
- {0x2279, {0x2277, 0x0, 0x0}, 1},
- {0x2280, {0x227A, 0x0, 0x0}, 1},
- {0x2281, {0x227B, 0x0, 0x0}, 1},
- {0x2284, {0x2282, 0x0, 0x0}, 1},
- {0x2285, {0x2283, 0x0, 0x0}, 1},
- {0x2288, {0x2286, 0x0, 0x0}, 1},
- {0x2289, {0x2287, 0x0, 0x0}, 1},
- {0x22AC, {0x22A2, 0x0, 0x0}, 1},
- {0x22AD, {0x22A8, 0x0, 0x0}, 1},
- {0x22AE, {0x22A9, 0x0, 0x0}, 1},
- {0x22AF, {0x22AB, 0x0, 0x0}, 1},
- {0x22E0, {0x227C, 0x0, 0x0}, 1},
- {0x22E1, {0x227D, 0x0, 0x0}, 1},
- {0x22E2, {0x2291, 0x0, 0x0}, 1},
- {0x22E3, {0x2292, 0x0, 0x0}, 1},
- {0x22EA, {0x22B2, 0x0, 0x0}, 1},
- {0x22EB, {0x22B3, 0x0, 0x0}, 1},
- {0x22EC, {0x22B4, 0x0, 0x0}, 1},
- {0x22ED, {0x22B5, 0x0, 0x0}, 1},
- {0x2329, {0x3008, 0x0, 0x0}, 1},
- {0x232A, {0x3009, 0x0, 0x0}, 1},
- {0x24B6, {0x24D0, 0x0, 0x0}, 1},
- {0x24B7, {0x24D1, 0x0, 0x0}, 1},
- {0x24B8, {0x24D2, 0x0, 0x0}, 1},
- {0x24B9, {0x24D3, 0x0, 0x0}, 1},
- {0x24BA, {0x24D4, 0x0, 0x0}, 1},
- {0x24BB, {0x24D5, 0x0, 0x0}, 1},
- {0x24BC, {0x24D6, 0x0, 0x0}, 1},
- {0x24BD, {0x24D7, 0x0, 0x0}, 1},
- {0x24BE, {0x24D8, 0x0, 0x0}, 1},
- {0x24BF, {0x24D9, 0x0, 0x0}, 1},
- {0x24C0, {0x24DA, 0x0, 0x0}, 1},
- {0x24C1, {0x24DB, 0x0, 0x0}, 1},
- {0x24C2, {0x24DC, 0x0, 0x0}, 1},
- {0x24C3, {0x24DD, 0x0, 0x0}, 1},
- {0x24C4, {0x24DE, 0x0, 0x0}, 1},
- {0x24C5, {0x24DF, 0x0, 0x0}, 1},
- {0x24C6, {0x24E0, 0x0, 0x0}, 1},
- {0x24C7, {0x24E1, 0x0, 0x0}, 1},
- {0x24C8, {0x24E2, 0x0, 0x0}, 1},
- {0x24C9, {0x24E3, 0x0, 0x0}, 1},
- {0x24CA, {0x24E4, 0x0, 0x0}, 1},
- {0x24CB, {0x24E5, 0x0, 0x0}, 1},
- {0x24CC, {0x24E6, 0x0, 0x0}, 1},
- {0x24CD, {0x24E7, 0x0, 0x0}, 1},
- {0x24CE, {0x24E8, 0x0, 0x0}, 1},
- {0x24CF, {0x24E9, 0x0, 0x0}, 1},
- {0x2ADC, {0x2ADD, 0x0, 0x0}, 1},
- {0x2C00, {0x2C30, 0x0, 0x0}, 1},
- {0x2C01, {0x2C31, 0x0, 0x0}, 1},
- {0x2C02, {0x2C32, 0x0, 0x0}, 1},
- {0x2C03, {0x2C33, 0x0, 0x0}, 1},
- {0x2C04, {0x2C34, 0x0, 0x0}, 1},
- {0x2C05, {0x2C35, 0x0, 0x0}, 1},
- {0x2C06, {0x2C36, 0x0, 0x0}, 1},
- {0x2C07, {0x2C37, 0x0, 0x0}, 1},
- {0x2C08, {0x2C38, 0x0, 0x0}, 1},
- {0x2C09, {0x2C39, 0x0, 0x0}, 1},
- {0x2C0A, {0x2C3A, 0x0, 0x0}, 1},
- {0x2C0B, {0x2C3B, 0x0, 0x0}, 1},
- {0x2C0C, {0x2C3C, 0x0, 0x0}, 1},
- {0x2C0D, {0x2C3D, 0x0, 0x0}, 1},
- {0x2C0E, {0x2C3E, 0x0, 0x0}, 1},
- {0x2C0F, {0x2C3F, 0x0, 0x0}, 1},
- {0x2C10, {0x2C40, 0x0, 0x0}, 1},
- {0x2C11, {0x2C41, 0x0, 0x0}, 1},
- {0x2C12, {0x2C42, 0x0, 0x0}, 1},
- {0x2C13, {0x2C43, 0x0, 0x0}, 1},
- {0x2C14, {0x2C44, 0x0, 0x0}, 1},
- {0x2C15, {0x2C45, 0x0, 0x0}, 1},
- {0x2C16, {0x2C46, 0x0, 0x0}, 1},
- {0x2C17, {0x2C47, 0x0, 0x0}, 1},
- {0x2C18, {0x2C48, 0x0, 0x0}, 1},
- {0x2C19, {0x2C49, 0x0, 0x0}, 1},
- {0x2C1A, {0x2C4A, 0x0, 0x0}, 1},
- {0x2C1B, {0x2C4B, 0x0, 0x0}, 1},
- {0x2C1C, {0x2C4C, 0x0, 0x0}, 1},
- {0x2C1D, {0x2C4D, 0x0, 0x0}, 1},
- {0x2C1E, {0x2C4E, 0x0, 0x0}, 1},
- {0x2C1F, {0x2C4F, 0x0, 0x0}, 1},
- {0x2C20, {0x2C50, 0x0, 0x0}, 1},
- {0x2C21, {0x2C51, 0x0, 0x0}, 1},
- {0x2C22, {0x2C52, 0x0, 0x0}, 1},
- {0x2C23, {0x2C53, 0x0, 0x0}, 1},
- {0x2C24, {0x2C54, 0x0, 0x0}, 1},
- {0x2C25, {0x2C55, 0x0, 0x0}, 1},
- {0x2C26, {0x2C56, 0x0, 0x0}, 1},
- {0x2C27, {0x2C57, 0x0, 0x0}, 1},
- {0x2C28, {0x2C58, 0x0, 0x0}, 1},
- {0x2C29, {0x2C59, 0x0, 0x0}, 1},
- {0x2C2A, {0x2C5A, 0x0, 0x0}, 1},
- {0x2C2B, {0x2C5B, 0x0, 0x0}, 1},
- {0x2C2C, {0x2C5C, 0x0, 0x0}, 1},
- {0x2C2D, {0x2C5D, 0x0, 0x0}, 1},
- {0x2C2E, {0x2C5E, 0x0, 0x0}, 1},
- {0x2C60, {0x2C61, 0x0, 0x0}, 1},
- {0x2C62, {0x26B, 0x0, 0x0}, 1},
- {0x2C63, {0x1D7D, 0x0, 0x0}, 1},
- {0x2C64, {0x27D, 0x0, 0x0}, 1},
- {0x2C67, {0x2C68, 0x0, 0x0}, 1},
- {0x2C69, {0x2C6A, 0x0, 0x0}, 1},
- {0x2C6B, {0x2C6C, 0x0, 0x0}, 1},
- {0x2C6D, {0x251, 0x0, 0x0}, 1},
- {0x2C6E, {0x271, 0x0, 0x0}, 1},
- {0x2C6F, {0x250, 0x0, 0x0}, 1},
- {0x2C70, {0x252, 0x0, 0x0}, 1},
- {0x2C72, {0x2C73, 0x0, 0x0}, 1},
- {0x2C75, {0x2C76, 0x0, 0x0}, 1},
- {0x2C7E, {0x23F, 0x0, 0x0}, 1},
- {0x2C7F, {0x240, 0x0, 0x0}, 1},
- {0x2C80, {0x2C81, 0x0, 0x0}, 1},
- {0x2C82, {0x2C83, 0x0, 0x0}, 1},
- {0x2C84, {0x2C85, 0x0, 0x0}, 1},
- {0x2C86, {0x2C87, 0x0, 0x0}, 1},
- {0x2C88, {0x2C89, 0x0, 0x0}, 1},
- {0x2C8A, {0x2C8B, 0x0, 0x0}, 1},
- {0x2C8C, {0x2C8D, 0x0, 0x0}, 1},
- {0x2C8E, {0x2C8F, 0x0, 0x0}, 1},
- {0x2C90, {0x2C91, 0x0, 0x0}, 1},
- {0x2C92, {0x2C93, 0x0, 0x0}, 1},
- {0x2C94, {0x2C95, 0x0, 0x0}, 1},
- {0x2C96, {0x2C97, 0x0, 0x0}, 1},
- {0x2C98, {0x2C99, 0x0, 0x0}, 1},
- {0x2C9A, {0x2C9B, 0x0, 0x0}, 1},
- {0x2C9C, {0x2C9D, 0x0, 0x0}, 1},
- {0x2C9E, {0x2C9F, 0x0, 0x0}, 1},
- {0x2CA0, {0x2CA1, 0x0, 0x0}, 1},
- {0x2CA2, {0x2CA3, 0x0, 0x0}, 1},
- {0x2CA4, {0x2CA5, 0x0, 0x0}, 1},
- {0x2CA6, {0x2CA7, 0x0, 0x0}, 1},
- {0x2CA8, {0x2CA9, 0x0, 0x0}, 1},
- {0x2CAA, {0x2CAB, 0x0, 0x0}, 1},
- {0x2CAC, {0x2CAD, 0x0, 0x0}, 1},
- {0x2CAE, {0x2CAF, 0x0, 0x0}, 1},
- {0x2CB0, {0x2CB1, 0x0, 0x0}, 1},
- {0x2CB2, {0x2CB3, 0x0, 0x0}, 1},
- {0x2CB4, {0x2CB5, 0x0, 0x0}, 1},
- {0x2CB6, {0x2CB7, 0x0, 0x0}, 1},
- {0x2CB8, {0x2CB9, 0x0, 0x0}, 1},
- {0x2CBA, {0x2CBB, 0x0, 0x0}, 1},
- {0x2CBC, {0x2CBD, 0x0, 0x0}, 1},
- {0x2CBE, {0x2CBF, 0x0, 0x0}, 1},
- {0x2CC0, {0x2CC1, 0x0, 0x0}, 1},
- {0x2CC2, {0x2CC3, 0x0, 0x0}, 1},
- {0x2CC4, {0x2CC5, 0x0, 0x0}, 1},
- {0x2CC6, {0x2CC7, 0x0, 0x0}, 1},
- {0x2CC8, {0x2CC9, 0x0, 0x0}, 1},
- {0x2CCA, {0x2CCB, 0x0, 0x0}, 1},
- {0x2CCC, {0x2CCD, 0x0, 0x0}, 1},
- {0x2CCE, {0x2CCF, 0x0, 0x0}, 1},
- {0x2CD0, {0x2CD1, 0x0, 0x0}, 1},
- {0x2CD2, {0x2CD3, 0x0, 0x0}, 1},
- {0x2CD4, {0x2CD5, 0x0, 0x0}, 1},
- {0x2CD6, {0x2CD7, 0x0, 0x0}, 1},
- {0x2CD8, {0x2CD9, 0x0, 0x0}, 1},
- {0x2CDA, {0x2CDB, 0x0, 0x0}, 1},
- {0x2CDC, {0x2CDD, 0x0, 0x0}, 1},
- {0x2CDE, {0x2CDF, 0x0, 0x0}, 1},
- {0x2CE0, {0x2CE1, 0x0, 0x0}, 1},
- {0x2CE2, {0x2CE3, 0x0, 0x0}, 1},
- {0x2CEB, {0x2CEC, 0x0, 0x0}, 1},
- {0x2CED, {0x2CEE, 0x0, 0x0}, 1},
- {0x2CEF, {0x0, 0x0, 0x0}, 0},
- {0x2CF0, {0x0, 0x0, 0x0}, 0},
- {0x2CF1, {0x0, 0x0, 0x0}, 0},
- {0x2CF2, {0x2CF3, 0x0, 0x0}, 1},
- {0x2D7F, {0x0, 0x0, 0x0}, 0},
- {0x2DE0, {0x0, 0x0, 0x0}, 0},
- {0x2DE1, {0x0, 0x0, 0x0}, 0},
- {0x2DE2, {0x0, 0x0, 0x0}, 0},
- {0x2DE3, {0x0, 0x0, 0x0}, 0},
- {0x2DE4, {0x0, 0x0, 0x0}, 0},
- {0x2DE5, {0x0, 0x0, 0x0}, 0},
- {0x2DE6, {0x0, 0x0, 0x0}, 0},
- {0x2DE7, {0x0, 0x0, 0x0}, 0},
- {0x2DE8, {0x0, 0x0, 0x0}, 0},
- {0x2DE9, {0x0, 0x0, 0x0}, 0},
- {0x2DEA, {0x0, 0x0, 0x0}, 0},
- {0x2DEB, {0x0, 0x0, 0x0}, 0},
- {0x2DEC, {0x0, 0x0, 0x0}, 0},
- {0x2DED, {0x0, 0x0, 0x0}, 0},
- {0x2DEE, {0x0, 0x0, 0x0}, 0},
- {0x2DEF, {0x0, 0x0, 0x0}, 0},
- {0x2DF0, {0x0, 0x0, 0x0}, 0},
- {0x2DF1, {0x0, 0x0, 0x0}, 0},
- {0x2DF2, {0x0, 0x0, 0x0}, 0},
- {0x2DF3, {0x0, 0x0, 0x0}, 0},
- {0x2DF4, {0x0, 0x0, 0x0}, 0},
- {0x2DF5, {0x0, 0x0, 0x0}, 0},
- {0x2DF6, {0x0, 0x0, 0x0}, 0},
- {0x2DF7, {0x0, 0x0, 0x0}, 0},
- {0x2DF8, {0x0, 0x0, 0x0}, 0},
- {0x2DF9, {0x0, 0x0, 0x0}, 0},
- {0x2DFA, {0x0, 0x0, 0x0}, 0},
- {0x2DFB, {0x0, 0x0, 0x0}, 0},
- {0x2DFC, {0x0, 0x0, 0x0}, 0},
- {0x2DFD, {0x0, 0x0, 0x0}, 0},
- {0x2DFE, {0x0, 0x0, 0x0}, 0},
- {0x2DFF, {0x0, 0x0, 0x0}, 0},
- {0x302A, {0x0, 0x0, 0x0}, 0},
- {0x302B, {0x0, 0x0, 0x0}, 0},
- {0x302C, {0x0, 0x0, 0x0}, 0},
- {0x302D, {0x0, 0x0, 0x0}, 0},
- {0x304C, {0x304B, 0x0, 0x0}, 1},
- {0x304E, {0x304D, 0x0, 0x0}, 1},
- {0x3050, {0x304F, 0x0, 0x0}, 1},
- {0x3052, {0x3051, 0x0, 0x0}, 1},
- {0x3054, {0x3053, 0x0, 0x0}, 1},
- {0x3056, {0x3055, 0x0, 0x0}, 1},
- {0x3058, {0x3057, 0x0, 0x0}, 1},
- {0x305A, {0x3059, 0x0, 0x0}, 1},
- {0x305C, {0x305B, 0x0, 0x0}, 1},
- {0x305E, {0x305D, 0x0, 0x0}, 1},
- {0x3060, {0x305F, 0x0, 0x0}, 1},
- {0x3062, {0x3061, 0x0, 0x0}, 1},
- {0x3065, {0x3064, 0x0, 0x0}, 1},
- {0x3067, {0x3066, 0x0, 0x0}, 1},
- {0x3069, {0x3068, 0x0, 0x0}, 1},
- {0x3070, {0x306F, 0x0, 0x0}, 1},
- {0x3071, {0x306F, 0x0, 0x0}, 1},
- {0x3073, {0x3072, 0x0, 0x0}, 1},
- {0x3074, {0x3072, 0x0, 0x0}, 1},
- {0x3076, {0x3075, 0x0, 0x0}, 1},
- {0x3077, {0x3075, 0x0, 0x0}, 1},
- {0x3079, {0x3078, 0x0, 0x0}, 1},
- {0x307A, {0x3078, 0x0, 0x0}, 1},
- {0x307C, {0x307B, 0x0, 0x0}, 1},
- {0x307D, {0x307B, 0x0, 0x0}, 1},
- {0x3094, {0x3046, 0x0, 0x0}, 1},
- {0x3099, {0x0, 0x0, 0x0}, 0},
- {0x309A, {0x0, 0x0, 0x0}, 0},
- {0x309E, {0x309D, 0x0, 0x0}, 1},
- {0x30AC, {0x30AB, 0x0, 0x0}, 1},
- {0x30AE, {0x30AD, 0x0, 0x0}, 1},
- {0x30B0, {0x30AF, 0x0, 0x0}, 1},
- {0x30B2, {0x30B1, 0x0, 0x0}, 1},
- {0x30B4, {0x30B3, 0x0, 0x0}, 1},
- {0x30B6, {0x30B5, 0x0, 0x0}, 1},
- {0x30B8, {0x30B7, 0x0, 0x0}, 1},
- {0x30BA, {0x30B9, 0x0, 0x0}, 1},
- {0x30BC, {0x30BB, 0x0, 0x0}, 1},
- {0x30BE, {0x30BD, 0x0, 0x0}, 1},
- {0x30C0, {0x30BF, 0x0, 0x0}, 1},
- {0x30C2, {0x30C1, 0x0, 0x0}, 1},
- {0x30C5, {0x30C4, 0x0, 0x0}, 1},
- {0x30C7, {0x30C6, 0x0, 0x0}, 1},
- {0x30C9, {0x30C8, 0x0, 0x0}, 1},
- {0x30D0, {0x30CF, 0x0, 0x0}, 1},
- {0x30D1, {0x30CF, 0x0, 0x0}, 1},
- {0x30D3, {0x30D2, 0x0, 0x0}, 1},
- {0x30D4, {0x30D2, 0x0, 0x0}, 1},
- {0x30D6, {0x30D5, 0x0, 0x0}, 1},
- {0x30D7, {0x30D5, 0x0, 0x0}, 1},
- {0x30D9, {0x30D8, 0x0, 0x0}, 1},
- {0x30DA, {0x30D8, 0x0, 0x0}, 1},
- {0x30DC, {0x30DB, 0x0, 0x0}, 1},
- {0x30DD, {0x30DB, 0x0, 0x0}, 1},
- {0x30F4, {0x30A6, 0x0, 0x0}, 1},
- {0x30F7, {0x30EF, 0x0, 0x0}, 1},
- {0x30F8, {0x30F0, 0x0, 0x0}, 1},
- {0x30F9, {0x30F1, 0x0, 0x0}, 1},
- {0x30FA, {0x30F2, 0x0, 0x0}, 1},
- {0x30FE, {0x30FD, 0x0, 0x0}, 1},
- {0xA640, {0xA641, 0x0, 0x0}, 1},
- {0xA642, {0xA643, 0x0, 0x0}, 1},
- {0xA644, {0xA645, 0x0, 0x0}, 1},
- {0xA646, {0xA647, 0x0, 0x0}, 1},
- {0xA648, {0xA649, 0x0, 0x0}, 1},
- {0xA64A, {0xA64B, 0x0, 0x0}, 1},
- {0xA64C, {0xA64D, 0x0, 0x0}, 1},
- {0xA64E, {0xA64F, 0x0, 0x0}, 1},
- {0xA650, {0xA651, 0x0, 0x0}, 1},
- {0xA652, {0xA653, 0x0, 0x0}, 1},
- {0xA654, {0xA655, 0x0, 0x0}, 1},
- {0xA656, {0xA657, 0x0, 0x0}, 1},
- {0xA658, {0xA659, 0x0, 0x0}, 1},
- {0xA65A, {0xA65B, 0x0, 0x0}, 1},
- {0xA65C, {0xA65D, 0x0, 0x0}, 1},
- {0xA65E, {0xA65F, 0x0, 0x0}, 1},
- {0xA660, {0xA661, 0x0, 0x0}, 1},
- {0xA662, {0xA663, 0x0, 0x0}, 1},
- {0xA664, {0xA665, 0x0, 0x0}, 1},
- {0xA666, {0xA667, 0x0, 0x0}, 1},
- {0xA668, {0xA669, 0x0, 0x0}, 1},
- {0xA66A, {0xA66B, 0x0, 0x0}, 1},
- {0xA66C, {0xA66D, 0x0, 0x0}, 1},
- {0xA66F, {0x0, 0x0, 0x0}, 0},
- {0xA674, {0x0, 0x0, 0x0}, 0},
- {0xA675, {0x0, 0x0, 0x0}, 0},
- {0xA676, {0x0, 0x0, 0x0}, 0},
- {0xA677, {0x0, 0x0, 0x0}, 0},
- {0xA678, {0x0, 0x0, 0x0}, 0},
- {0xA679, {0x0, 0x0, 0x0}, 0},
- {0xA67A, {0x0, 0x0, 0x0}, 0},
- {0xA67B, {0x0, 0x0, 0x0}, 0},
- {0xA67C, {0x0, 0x0, 0x0}, 0},
- {0xA67D, {0x0, 0x0, 0x0}, 0},
- {0xA680, {0xA681, 0x0, 0x0}, 1},
- {0xA682, {0xA683, 0x0, 0x0}, 1},
- {0xA684, {0xA685, 0x0, 0x0}, 1},
- {0xA686, {0xA687, 0x0, 0x0}, 1},
- {0xA688, {0xA689, 0x0, 0x0}, 1},
- {0xA68A, {0xA68B, 0x0, 0x0}, 1},
- {0xA68C, {0xA68D, 0x0, 0x0}, 1},
- {0xA68E, {0xA68F, 0x0, 0x0}, 1},
- {0xA690, {0xA691, 0x0, 0x0}, 1},
- {0xA692, {0xA693, 0x0, 0x0}, 1},
- {0xA694, {0xA695, 0x0, 0x0}, 1},
- {0xA696, {0xA697, 0x0, 0x0}, 1},
- {0xA698, {0xA699, 0x0, 0x0}, 1},
- {0xA69A, {0xA69B, 0x0, 0x0}, 1},
- {0xA69E, {0x0, 0x0, 0x0}, 0},
- {0xA69F, {0x0, 0x0, 0x0}, 0},
- {0xA6F0, {0x0, 0x0, 0x0}, 0},
- {0xA6F1, {0x0, 0x0, 0x0}, 0},
- {0xA722, {0xA723, 0x0, 0x0}, 1},
- {0xA724, {0xA725, 0x0, 0x0}, 1},
- {0xA726, {0xA727, 0x0, 0x0}, 1},
- {0xA728, {0xA729, 0x0, 0x0}, 1},
- {0xA72A, {0xA72B, 0x0, 0x0}, 1},
- {0xA72C, {0xA72D, 0x0, 0x0}, 1},
- {0xA72E, {0xA72F, 0x0, 0x0}, 1},
- {0xA732, {0xA733, 0x0, 0x0}, 1},
- {0xA734, {0xA735, 0x0, 0x0}, 1},
- {0xA736, {0xA737, 0x0, 0x0}, 1},
- {0xA738, {0xA739, 0x0, 0x0}, 1},
- {0xA73A, {0xA73B, 0x0, 0x0}, 1},
- {0xA73C, {0xA73D, 0x0, 0x0}, 1},
- {0xA73E, {0xA73F, 0x0, 0x0}, 1},
- {0xA740, {0xA741, 0x0, 0x0}, 1},
- {0xA742, {0xA743, 0x0, 0x0}, 1},
- {0xA744, {0xA745, 0x0, 0x0}, 1},
- {0xA746, {0xA747, 0x0, 0x0}, 1},
- {0xA748, {0xA749, 0x0, 0x0}, 1},
- {0xA74A, {0xA74B, 0x0, 0x0}, 1},
- {0xA74C, {0xA74D, 0x0, 0x0}, 1},
- {0xA74E, {0xA74F, 0x0, 0x0}, 1},
- {0xA750, {0xA751, 0x0, 0x0}, 1},
- {0xA752, {0xA753, 0x0, 0x0}, 1},
- {0xA754, {0xA755, 0x0, 0x0}, 1},
- {0xA756, {0xA757, 0x0, 0x0}, 1},
- {0xA758, {0xA759, 0x0, 0x0}, 1},
- {0xA75A, {0xA75B, 0x0, 0x0}, 1},
- {0xA75C, {0xA75D, 0x0, 0x0}, 1},
- {0xA75E, {0xA75F, 0x0, 0x0}, 1},
- {0xA760, {0xA761, 0x0, 0x0}, 1},
- {0xA762, {0xA763, 0x0, 0x0}, 1},
- {0xA764, {0xA765, 0x0, 0x0}, 1},
- {0xA766, {0xA767, 0x0, 0x0}, 1},
- {0xA768, {0xA769, 0x0, 0x0}, 1},
- {0xA76A, {0xA76B, 0x0, 0x0}, 1},
- {0xA76C, {0xA76D, 0x0, 0x0}, 1},
- {0xA76E, {0xA76F, 0x0, 0x0}, 1},
- {0xA779, {0xA77A, 0x0, 0x0}, 1},
- {0xA77B, {0xA77C, 0x0, 0x0}, 1},
- {0xA77D, {0x1D79, 0x0, 0x0}, 1},
- {0xA77E, {0xA77F, 0x0, 0x0}, 1},
- {0xA780, {0xA781, 0x0, 0x0}, 1},
- {0xA782, {0xA783, 0x0, 0x0}, 1},
- {0xA784, {0xA785, 0x0, 0x0}, 1},
- {0xA786, {0xA787, 0x0, 0x0}, 1},
- {0xA78B, {0xA78C, 0x0, 0x0}, 1},
- {0xA78D, {0x265, 0x0, 0x0}, 1},
- {0xA790, {0xA791, 0x0, 0x0}, 1},
- {0xA792, {0xA793, 0x0, 0x0}, 1},
- {0xA796, {0xA797, 0x0, 0x0}, 1},
- {0xA798, {0xA799, 0x0, 0x0}, 1},
- {0xA79A, {0xA79B, 0x0, 0x0}, 1},
- {0xA79C, {0xA79D, 0x0, 0x0}, 1},
- {0xA79E, {0xA79F, 0x0, 0x0}, 1},
- {0xA7A0, {0xA7A1, 0x0, 0x0}, 1},
- {0xA7A2, {0xA7A3, 0x0, 0x0}, 1},
- {0xA7A4, {0xA7A5, 0x0, 0x0}, 1},
- {0xA7A6, {0xA7A7, 0x0, 0x0}, 1},
- {0xA7A8, {0xA7A9, 0x0, 0x0}, 1},
- {0xA7AA, {0x266, 0x0, 0x0}, 1},
- {0xA7AB, {0x25C, 0x0, 0x0}, 1},
- {0xA7AC, {0x261, 0x0, 0x0}, 1},
- {0xA7AD, {0x26C, 0x0, 0x0}, 1},
- {0xA7AE, {0x26A, 0x0, 0x0}, 1},
- {0xA7B0, {0x29E, 0x0, 0x0}, 1},
- {0xA7B1, {0x287, 0x0, 0x0}, 1},
- {0xA7B2, {0x29D, 0x0, 0x0}, 1},
- {0xA7B3, {0xAB53, 0x0, 0x0}, 1},
- {0xA7B4, {0xA7B5, 0x0, 0x0}, 1},
- {0xA7B6, {0xA7B7, 0x0, 0x0}, 1},
- {0xA7B8, {0xA7B9, 0x0, 0x0}, 1},
- {0xA7BA, {0xA7BB, 0x0, 0x0}, 1},
- {0xA7BC, {0xA7BD, 0x0, 0x0}, 1},
- {0xA7BE, {0xA7BF, 0x0, 0x0}, 1},
- {0xA7C2, {0xA7C3, 0x0, 0x0}, 1},
- {0xA7C4, {0xA794, 0x0, 0x0}, 1},
- {0xA7C5, {0x282, 0x0, 0x0}, 1},
- {0xA7C6, {0x1D8E, 0x0, 0x0}, 1},
- {0xA7C7, {0xA7C8, 0x0, 0x0}, 1},
- {0xA7C9, {0xA7CA, 0x0, 0x0}, 1},
- {0xA7F5, {0xA7F6, 0x0, 0x0}, 1},
- {0xA802, {0x0, 0x0, 0x0}, 0},
- {0xA806, {0x0, 0x0, 0x0}, 0},
- {0xA80B, {0x0, 0x0, 0x0}, 0},
- {0xA825, {0x0, 0x0, 0x0}, 0},
- {0xA826, {0x0, 0x0, 0x0}, 0},
- {0xA82C, {0x0, 0x0, 0x0}, 0},
- {0xA8C4, {0x0, 0x0, 0x0}, 0},
- {0xA8C5, {0x0, 0x0, 0x0}, 0},
- {0xA8E0, {0x0, 0x0, 0x0}, 0},
- {0xA8E1, {0x0, 0x0, 0x0}, 0},
- {0xA8E2, {0x0, 0x0, 0x0}, 0},
- {0xA8E3, {0x0, 0x0, 0x0}, 0},
- {0xA8E4, {0x0, 0x0, 0x0}, 0},
- {0xA8E5, {0x0, 0x0, 0x0}, 0},
- {0xA8E6, {0x0, 0x0, 0x0}, 0},
- {0xA8E7, {0x0, 0x0, 0x0}, 0},
- {0xA8E8, {0x0, 0x0, 0x0}, 0},
- {0xA8E9, {0x0, 0x0, 0x0}, 0},
- {0xA8EA, {0x0, 0x0, 0x0}, 0},
- {0xA8EB, {0x0, 0x0, 0x0}, 0},
- {0xA8EC, {0x0, 0x0, 0x0}, 0},
- {0xA8ED, {0x0, 0x0, 0x0}, 0},
- {0xA8EE, {0x0, 0x0, 0x0}, 0},
- {0xA8EF, {0x0, 0x0, 0x0}, 0},
- {0xA8F0, {0x0, 0x0, 0x0}, 0},
- {0xA8F1, {0x0, 0x0, 0x0}, 0},
- {0xA8FF, {0x0, 0x0, 0x0}, 0},
- {0xA926, {0x0, 0x0, 0x0}, 0},
- {0xA927, {0x0, 0x0, 0x0}, 0},
- {0xA928, {0x0, 0x0, 0x0}, 0},
- {0xA929, {0x0, 0x0, 0x0}, 0},
- {0xA92A, {0x0, 0x0, 0x0}, 0},
- {0xA92B, {0x0, 0x0, 0x0}, 0},
- {0xA92C, {0x0, 0x0, 0x0}, 0},
- {0xA92D, {0x0, 0x0, 0x0}, 0},
- {0xA947, {0x0, 0x0, 0x0}, 0},
- {0xA948, {0x0, 0x0, 0x0}, 0},
- {0xA949, {0x0, 0x0, 0x0}, 0},
- {0xA94A, {0x0, 0x0, 0x0}, 0},
- {0xA94B, {0x0, 0x0, 0x0}, 0},
- {0xA94C, {0x0, 0x0, 0x0}, 0},
- {0xA94D, {0x0, 0x0, 0x0}, 0},
- {0xA94E, {0x0, 0x0, 0x0}, 0},
- {0xA94F, {0x0, 0x0, 0x0}, 0},
- {0xA950, {0x0, 0x0, 0x0}, 0},
- {0xA951, {0x0, 0x0, 0x0}, 0},
- {0xA980, {0x0, 0x0, 0x0}, 0},
- {0xA981, {0x0, 0x0, 0x0}, 0},
- {0xA982, {0x0, 0x0, 0x0}, 0},
- {0xA9B3, {0x0, 0x0, 0x0}, 0},
- {0xA9B6, {0x0, 0x0, 0x0}, 0},
- {0xA9B7, {0x0, 0x0, 0x0}, 0},
- {0xA9B8, {0x0, 0x0, 0x0}, 0},
- {0xA9B9, {0x0, 0x0, 0x0}, 0},
- {0xA9BC, {0x0, 0x0, 0x0}, 0},
- {0xA9BD, {0x0, 0x0, 0x0}, 0},
- {0xA9E5, {0x0, 0x0, 0x0}, 0},
- {0xAA29, {0x0, 0x0, 0x0}, 0},
- {0xAA2A, {0x0, 0x0, 0x0}, 0},
- {0xAA2B, {0x0, 0x0, 0x0}, 0},
- {0xAA2C, {0x0, 0x0, 0x0}, 0},
- {0xAA2D, {0x0, 0x0, 0x0}, 0},
- {0xAA2E, {0x0, 0x0, 0x0}, 0},
- {0xAA31, {0x0, 0x0, 0x0}, 0},
- {0xAA32, {0x0, 0x0, 0x0}, 0},
- {0xAA35, {0x0, 0x0, 0x0}, 0},
- {0xAA36, {0x0, 0x0, 0x0}, 0},
- {0xAA43, {0x0, 0x0, 0x0}, 0},
- {0xAA4C, {0x0, 0x0, 0x0}, 0},
- {0xAA7C, {0x0, 0x0, 0x0}, 0},
- {0xAAB0, {0x0, 0x0, 0x0}, 0},
- {0xAAB2, {0x0, 0x0, 0x0}, 0},
- {0xAAB3, {0x0, 0x0, 0x0}, 0},
- {0xAAB4, {0x0, 0x0, 0x0}, 0},
- {0xAAB7, {0x0, 0x0, 0x0}, 0},
- {0xAAB8, {0x0, 0x0, 0x0}, 0},
- {0xAABE, {0x0, 0x0, 0x0}, 0},
- {0xAABF, {0x0, 0x0, 0x0}, 0},
- {0xAAC1, {0x0, 0x0, 0x0}, 0},
- {0xAAEC, {0x0, 0x0, 0x0}, 0},
- {0xAAED, {0x0, 0x0, 0x0}, 0},
- {0xAAF6, {0x0, 0x0, 0x0}, 0},
- {0xABE5, {0x0, 0x0, 0x0}, 0},
- {0xABE8, {0x0, 0x0, 0x0}, 0},
- {0xABED, {0x0, 0x0, 0x0}, 0},
- {0xF900, {0x8C48, 0x0, 0x0}, 1},
- {0xF901, {0x66F4, 0x0, 0x0}, 1},
- {0xF902, {0x8ECA, 0x0, 0x0}, 1},
- {0xF903, {0x8CC8, 0x0, 0x0}, 1},
- {0xF904, {0x6ED1, 0x0, 0x0}, 1},
- {0xF905, {0x4E32, 0x0, 0x0}, 1},
- {0xF906, {0x53E5, 0x0, 0x0}, 1},
- {0xF907, {0x9F9C, 0x0, 0x0}, 1},
- {0xF908, {0x9F9C, 0x0, 0x0}, 1},
- {0xF909, {0x5951, 0x0, 0x0}, 1},
- {0xF90A, {0x91D1, 0x0, 0x0}, 1},
- {0xF90B, {0x5587, 0x0, 0x0}, 1},
- {0xF90C, {0x5948, 0x0, 0x0}, 1},
- {0xF90D, {0x61F6, 0x0, 0x0}, 1},
- {0xF90E, {0x7669, 0x0, 0x0}, 1},
- {0xF90F, {0x7F85, 0x0, 0x0}, 1},
- {0xF910, {0x863F, 0x0, 0x0}, 1},
- {0xF911, {0x87BA, 0x0, 0x0}, 1},
- {0xF912, {0x88F8, 0x0, 0x0}, 1},
- {0xF913, {0x908F, 0x0, 0x0}, 1},
- {0xF914, {0x6A02, 0x0, 0x0}, 1},
- {0xF915, {0x6D1B, 0x0, 0x0}, 1},
- {0xF916, {0x70D9, 0x0, 0x0}, 1},
- {0xF917, {0x73DE, 0x0, 0x0}, 1},
- {0xF918, {0x843D, 0x0, 0x0}, 1},
- {0xF919, {0x916A, 0x0, 0x0}, 1},
- {0xF91A, {0x99F1, 0x0, 0x0}, 1},
- {0xF91B, {0x4E82, 0x0, 0x0}, 1},
- {0xF91C, {0x5375, 0x0, 0x0}, 1},
- {0xF91D, {0x6B04, 0x0, 0x0}, 1},
- {0xF91E, {0x721B, 0x0, 0x0}, 1},
- {0xF91F, {0x862D, 0x0, 0x0}, 1},
- {0xF920, {0x9E1E, 0x0, 0x0}, 1},
- {0xF921, {0x5D50, 0x0, 0x0}, 1},
- {0xF922, {0x6FEB, 0x0, 0x0}, 1},
- {0xF923, {0x85CD, 0x0, 0x0}, 1},
- {0xF924, {0x8964, 0x0, 0x0}, 1},
- {0xF925, {0x62C9, 0x0, 0x0}, 1},
- {0xF926, {0x81D8, 0x0, 0x0}, 1},
- {0xF927, {0x881F, 0x0, 0x0}, 1},
- {0xF928, {0x5ECA, 0x0, 0x0}, 1},
- {0xF929, {0x6717, 0x0, 0x0}, 1},
- {0xF92A, {0x6D6A, 0x0, 0x0}, 1},
- {0xF92B, {0x72FC, 0x0, 0x0}, 1},
- {0xF92C, {0x90CE, 0x0, 0x0}, 1},
- {0xF92D, {0x4F86, 0x0, 0x0}, 1},
- {0xF92E, {0x51B7, 0x0, 0x0}, 1},
- {0xF92F, {0x52DE, 0x0, 0x0}, 1},
- {0xF930, {0x64C4, 0x0, 0x0}, 1},
- {0xF931, {0x6AD3, 0x0, 0x0}, 1},
- {0xF932, {0x7210, 0x0, 0x0}, 1},
- {0xF933, {0x76E7, 0x0, 0x0}, 1},
- {0xF934, {0x8001, 0x0, 0x0}, 1},
- {0xF935, {0x8606, 0x0, 0x0}, 1},
- {0xF936, {0x865C, 0x0, 0x0}, 1},
- {0xF937, {0x8DEF, 0x0, 0x0}, 1},
- {0xF938, {0x9732, 0x0, 0x0}, 1},
- {0xF939, {0x9B6F, 0x0, 0x0}, 1},
- {0xF93A, {0x9DFA, 0x0, 0x0}, 1},
- {0xF93B, {0x788C, 0x0, 0x0}, 1},
- {0xF93C, {0x797F, 0x0, 0x0}, 1},
- {0xF93D, {0x7DA0, 0x0, 0x0}, 1},
- {0xF93E, {0x83C9, 0x0, 0x0}, 1},
- {0xF93F, {0x9304, 0x0, 0x0}, 1},
- {0xF940, {0x9E7F, 0x0, 0x0}, 1},
- {0xF941, {0x8AD6, 0x0, 0x0}, 1},
- {0xF942, {0x58DF, 0x0, 0x0}, 1},
- {0xF943, {0x5F04, 0x0, 0x0}, 1},
- {0xF944, {0x7C60, 0x0, 0x0}, 1},
- {0xF945, {0x807E, 0x0, 0x0}, 1},
- {0xF946, {0x7262, 0x0, 0x0}, 1},
- {0xF947, {0x78CA, 0x0, 0x0}, 1},
- {0xF948, {0x8CC2, 0x0, 0x0}, 1},
- {0xF949, {0x96F7, 0x0, 0x0}, 1},
- {0xF94A, {0x58D8, 0x0, 0x0}, 1},
- {0xF94B, {0x5C62, 0x0, 0x0}, 1},
- {0xF94C, {0x6A13, 0x0, 0x0}, 1},
- {0xF94D, {0x6DDA, 0x0, 0x0}, 1},
- {0xF94E, {0x6F0F, 0x0, 0x0}, 1},
- {0xF94F, {0x7D2F, 0x0, 0x0}, 1},
- {0xF950, {0x7E37, 0x0, 0x0}, 1},
- {0xF951, {0x964B, 0x0, 0x0}, 1},
- {0xF952, {0x52D2, 0x0, 0x0}, 1},
- {0xF953, {0x808B, 0x0, 0x0}, 1},
- {0xF954, {0x51DC, 0x0, 0x0}, 1},
- {0xF955, {0x51CC, 0x0, 0x0}, 1},
- {0xF956, {0x7A1C, 0x0, 0x0}, 1},
- {0xF957, {0x7DBE, 0x0, 0x0}, 1},
- {0xF958, {0x83F1, 0x0, 0x0}, 1},
- {0xF959, {0x9675, 0x0, 0x0}, 1},
- {0xF95A, {0x8B80, 0x0, 0x0}, 1},
- {0xF95B, {0x62CF, 0x0, 0x0}, 1},
- {0xF95C, {0x6A02, 0x0, 0x0}, 1},
- {0xF95D, {0x8AFE, 0x0, 0x0}, 1},
- {0xF95E, {0x4E39, 0x0, 0x0}, 1},
- {0xF95F, {0x5BE7, 0x0, 0x0}, 1},
- {0xF960, {0x6012, 0x0, 0x0}, 1},
- {0xF961, {0x7387, 0x0, 0x0}, 1},
- {0xF962, {0x7570, 0x0, 0x0}, 1},
- {0xF963, {0x5317, 0x0, 0x0}, 1},
- {0xF964, {0x78FB, 0x0, 0x0}, 1},
- {0xF965, {0x4FBF, 0x0, 0x0}, 1},
- {0xF966, {0x5FA9, 0x0, 0x0}, 1},
- {0xF967, {0x4E0D, 0x0, 0x0}, 1},
- {0xF968, {0x6CCC, 0x0, 0x0}, 1},
- {0xF969, {0x6578, 0x0, 0x0}, 1},
- {0xF96A, {0x7D22, 0x0, 0x0}, 1},
- {0xF96B, {0x53C3, 0x0, 0x0}, 1},
- {0xF96C, {0x585E, 0x0, 0x0}, 1},
- {0xF96D, {0x7701, 0x0, 0x0}, 1},
- {0xF96E, {0x8449, 0x0, 0x0}, 1},
- {0xF96F, {0x8AAA, 0x0, 0x0}, 1},
- {0xF970, {0x6BBA, 0x0, 0x0}, 1},
- {0xF971, {0x8FB0, 0x0, 0x0}, 1},
- {0xF972, {0x6C88, 0x0, 0x0}, 1},
- {0xF973, {0x62FE, 0x0, 0x0}, 1},
- {0xF974, {0x82E5, 0x0, 0x0}, 1},
- {0xF975, {0x63A0, 0x0, 0x0}, 1},
- {0xF976, {0x7565, 0x0, 0x0}, 1},
- {0xF977, {0x4EAE, 0x0, 0x0}, 1},
- {0xF978, {0x5169, 0x0, 0x0}, 1},
- {0xF979, {0x51C9, 0x0, 0x0}, 1},
- {0xF97A, {0x6881, 0x0, 0x0}, 1},
- {0xF97B, {0x7CE7, 0x0, 0x0}, 1},
- {0xF97C, {0x826F, 0x0, 0x0}, 1},
- {0xF97D, {0x8AD2, 0x0, 0x0}, 1},
- {0xF97E, {0x91CF, 0x0, 0x0}, 1},
- {0xF97F, {0x52F5, 0x0, 0x0}, 1},
- {0xF980, {0x5442, 0x0, 0x0}, 1},
- {0xF981, {0x5973, 0x0, 0x0}, 1},
- {0xF982, {0x5EEC, 0x0, 0x0}, 1},
- {0xF983, {0x65C5, 0x0, 0x0}, 1},
- {0xF984, {0x6FFE, 0x0, 0x0}, 1},
- {0xF985, {0x792A, 0x0, 0x0}, 1},
- {0xF986, {0x95AD, 0x0, 0x0}, 1},
- {0xF987, {0x9A6A, 0x0, 0x0}, 1},
- {0xF988, {0x9E97, 0x0, 0x0}, 1},
- {0xF989, {0x9ECE, 0x0, 0x0}, 1},
- {0xF98A, {0x529B, 0x0, 0x0}, 1},
- {0xF98B, {0x66C6, 0x0, 0x0}, 1},
- {0xF98C, {0x6B77, 0x0, 0x0}, 1},
- {0xF98D, {0x8F62, 0x0, 0x0}, 1},
- {0xF98E, {0x5E74, 0x0, 0x0}, 1},
- {0xF98F, {0x6190, 0x0, 0x0}, 1},
- {0xF990, {0x6200, 0x0, 0x0}, 1},
- {0xF991, {0x649A, 0x0, 0x0}, 1},
- {0xF992, {0x6F23, 0x0, 0x0}, 1},
- {0xF993, {0x7149, 0x0, 0x0}, 1},
- {0xF994, {0x7489, 0x0, 0x0}, 1},
- {0xF995, {0x79CA, 0x0, 0x0}, 1},
- {0xF996, {0x7DF4, 0x0, 0x0}, 1},
- {0xF997, {0x806F, 0x0, 0x0}, 1},
- {0xF998, {0x8F26, 0x0, 0x0}, 1},
- {0xF999, {0x84EE, 0x0, 0x0}, 1},
- {0xF99A, {0x9023, 0x0, 0x0}, 1},
- {0xF99B, {0x934A, 0x0, 0x0}, 1},
- {0xF99C, {0x5217, 0x0, 0x0}, 1},
- {0xF99D, {0x52A3, 0x0, 0x0}, 1},
- {0xF99E, {0x54BD, 0x0, 0x0}, 1},
- {0xF99F, {0x70C8, 0x0, 0x0}, 1},
- {0xF9A0, {0x88C2, 0x0, 0x0}, 1},
- {0xF9A1, {0x8AAA, 0x0, 0x0}, 1},
- {0xF9A2, {0x5EC9, 0x0, 0x0}, 1},
- {0xF9A3, {0x5FF5, 0x0, 0x0}, 1},
- {0xF9A4, {0x637B, 0x0, 0x0}, 1},
- {0xF9A5, {0x6BAE, 0x0, 0x0}, 1},
- {0xF9A6, {0x7C3E, 0x0, 0x0}, 1},
- {0xF9A7, {0x7375, 0x0, 0x0}, 1},
- {0xF9A8, {0x4EE4, 0x0, 0x0}, 1},
- {0xF9A9, {0x56F9, 0x0, 0x0}, 1},
- {0xF9AA, {0x5BE7, 0x0, 0x0}, 1},
- {0xF9AB, {0x5DBA, 0x0, 0x0}, 1},
- {0xF9AC, {0x601C, 0x0, 0x0}, 1},
- {0xF9AD, {0x73B2, 0x0, 0x0}, 1},
- {0xF9AE, {0x7469, 0x0, 0x0}, 1},
- {0xF9AF, {0x7F9A, 0x0, 0x0}, 1},
- {0xF9B0, {0x8046, 0x0, 0x0}, 1},
- {0xF9B1, {0x9234, 0x0, 0x0}, 1},
- {0xF9B2, {0x96F6, 0x0, 0x0}, 1},
- {0xF9B3, {0x9748, 0x0, 0x0}, 1},
- {0xF9B4, {0x9818, 0x0, 0x0}, 1},
- {0xF9B5, {0x4F8B, 0x0, 0x0}, 1},
- {0xF9B6, {0x79AE, 0x0, 0x0}, 1},
- {0xF9B7, {0x91B4, 0x0, 0x0}, 1},
- {0xF9B8, {0x96B8, 0x0, 0x0}, 1},
- {0xF9B9, {0x60E1, 0x0, 0x0}, 1},
- {0xF9BA, {0x4E86, 0x0, 0x0}, 1},
- {0xF9BB, {0x50DA, 0x0, 0x0}, 1},
- {0xF9BC, {0x5BEE, 0x0, 0x0}, 1},
- {0xF9BD, {0x5C3F, 0x0, 0x0}, 1},
- {0xF9BE, {0x6599, 0x0, 0x0}, 1},
- {0xF9BF, {0x6A02, 0x0, 0x0}, 1},
- {0xF9C0, {0x71CE, 0x0, 0x0}, 1},
- {0xF9C1, {0x7642, 0x0, 0x0}, 1},
- {0xF9C2, {0x84FC, 0x0, 0x0}, 1},
- {0xF9C3, {0x907C, 0x0, 0x0}, 1},
- {0xF9C4, {0x9F8D, 0x0, 0x0}, 1},
- {0xF9C5, {0x6688, 0x0, 0x0}, 1},
- {0xF9C6, {0x962E, 0x0, 0x0}, 1},
- {0xF9C7, {0x5289, 0x0, 0x0}, 1},
- {0xF9C8, {0x677B, 0x0, 0x0}, 1},
- {0xF9C9, {0x67F3, 0x0, 0x0}, 1},
- {0xF9CA, {0x6D41, 0x0, 0x0}, 1},
- {0xF9CB, {0x6E9C, 0x0, 0x0}, 1},
- {0xF9CC, {0x7409, 0x0, 0x0}, 1},
- {0xF9CD, {0x7559, 0x0, 0x0}, 1},
- {0xF9CE, {0x786B, 0x0, 0x0}, 1},
- {0xF9CF, {0x7D10, 0x0, 0x0}, 1},
- {0xF9D0, {0x985E, 0x0, 0x0}, 1},
- {0xF9D1, {0x516D, 0x0, 0x0}, 1},
- {0xF9D2, {0x622E, 0x0, 0x0}, 1},
- {0xF9D3, {0x9678, 0x0, 0x0}, 1},
- {0xF9D4, {0x502B, 0x0, 0x0}, 1},
- {0xF9D5, {0x5D19, 0x0, 0x0}, 1},
- {0xF9D6, {0x6DEA, 0x0, 0x0}, 1},
- {0xF9D7, {0x8F2A, 0x0, 0x0}, 1},
- {0xF9D8, {0x5F8B, 0x0, 0x0}, 1},
- {0xF9D9, {0x6144, 0x0, 0x0}, 1},
- {0xF9DA, {0x6817, 0x0, 0x0}, 1},
- {0xF9DB, {0x7387, 0x0, 0x0}, 1},
- {0xF9DC, {0x9686, 0x0, 0x0}, 1},
- {0xF9DD, {0x5229, 0x0, 0x0}, 1},
- {0xF9DE, {0x540F, 0x0, 0x0}, 1},
- {0xF9DF, {0x5C65, 0x0, 0x0}, 1},
- {0xF9E0, {0x6613, 0x0, 0x0}, 1},
- {0xF9E1, {0x674E, 0x0, 0x0}, 1},
- {0xF9E2, {0x68A8, 0x0, 0x0}, 1},
- {0xF9E3, {0x6CE5, 0x0, 0x0}, 1},
- {0xF9E4, {0x7406, 0x0, 0x0}, 1},
- {0xF9E5, {0x75E2, 0x0, 0x0}, 1},
- {0xF9E6, {0x7F79, 0x0, 0x0}, 1},
- {0xF9E7, {0x88CF, 0x0, 0x0}, 1},
- {0xF9E8, {0x88E1, 0x0, 0x0}, 1},
- {0xF9E9, {0x91CC, 0x0, 0x0}, 1},
- {0xF9EA, {0x96E2, 0x0, 0x0}, 1},
- {0xF9EB, {0x533F, 0x0, 0x0}, 1},
- {0xF9EC, {0x6EBA, 0x0, 0x0}, 1},
- {0xF9ED, {0x541D, 0x0, 0x0}, 1},
- {0xF9EE, {0x71D0, 0x0, 0x0}, 1},
- {0xF9EF, {0x7498, 0x0, 0x0}, 1},
- {0xF9F0, {0x85FA, 0x0, 0x0}, 1},
- {0xF9F1, {0x96A3, 0x0, 0x0}, 1},
- {0xF9F2, {0x9C57, 0x0, 0x0}, 1},
- {0xF9F3, {0x9E9F, 0x0, 0x0}, 1},
- {0xF9F4, {0x6797, 0x0, 0x0}, 1},
- {0xF9F5, {0x6DCB, 0x0, 0x0}, 1},
- {0xF9F6, {0x81E8, 0x0, 0x0}, 1},
- {0xF9F7, {0x7ACB, 0x0, 0x0}, 1},
- {0xF9F8, {0x7B20, 0x0, 0x0}, 1},
- {0xF9F9, {0x7C92, 0x0, 0x0}, 1},
- {0xF9FA, {0x72C0, 0x0, 0x0}, 1},
- {0xF9FB, {0x7099, 0x0, 0x0}, 1},
- {0xF9FC, {0x8B58, 0x0, 0x0}, 1},
- {0xF9FD, {0x4EC0, 0x0, 0x0}, 1},
- {0xF9FE, {0x8336, 0x0, 0x0}, 1},
- {0xF9FF, {0x523A, 0x0, 0x0}, 1},
- {0xFA00, {0x5207, 0x0, 0x0}, 1},
- {0xFA01, {0x5EA6, 0x0, 0x0}, 1},
- {0xFA02, {0x62D3, 0x0, 0x0}, 1},
- {0xFA03, {0x7CD6, 0x0, 0x0}, 1},
- {0xFA04, {0x5B85, 0x0, 0x0}, 1},
- {0xFA05, {0x6D1E, 0x0, 0x0}, 1},
- {0xFA06, {0x66B4, 0x0, 0x0}, 1},
- {0xFA07, {0x8F3B, 0x0, 0x0}, 1},
- {0xFA08, {0x884C, 0x0, 0x0}, 1},
- {0xFA09, {0x964D, 0x0, 0x0}, 1},
- {0xFA0A, {0x898B, 0x0, 0x0}, 1},
- {0xFA0B, {0x5ED3, 0x0, 0x0}, 1},
- {0xFA0C, {0x5140, 0x0, 0x0}, 1},
- {0xFA0D, {0x55C0, 0x0, 0x0}, 1},
- {0xFA10, {0x585A, 0x0, 0x0}, 1},
- {0xFA12, {0x6674, 0x0, 0x0}, 1},
- {0xFA15, {0x51DE, 0x0, 0x0}, 1},
- {0xFA16, {0x732A, 0x0, 0x0}, 1},
- {0xFA17, {0x76CA, 0x0, 0x0}, 1},
- {0xFA18, {0x793C, 0x0, 0x0}, 1},
- {0xFA19, {0x795E, 0x0, 0x0}, 1},
- {0xFA1A, {0x7965, 0x0, 0x0}, 1},
- {0xFA1B, {0x798F, 0x0, 0x0}, 1},
- {0xFA1C, {0x9756, 0x0, 0x0}, 1},
- {0xFA1D, {0x7CBE, 0x0, 0x0}, 1},
- {0xFA1E, {0x7FBD, 0x0, 0x0}, 1},
- {0xFA20, {0x8612, 0x0, 0x0}, 1},
- {0xFA22, {0x8AF8, 0x0, 0x0}, 1},
- {0xFA25, {0x9038, 0x0, 0x0}, 1},
- {0xFA26, {0x90FD, 0x0, 0x0}, 1},
- {0xFA2A, {0x98EF, 0x0, 0x0}, 1},
- {0xFA2B, {0x98FC, 0x0, 0x0}, 1},
- {0xFA2C, {0x9928, 0x0, 0x0}, 1},
- {0xFA2D, {0x9DB4, 0x0, 0x0}, 1},
- {0xFA2E, {0x90DE, 0x0, 0x0}, 1},
- {0xFA2F, {0x96B7, 0x0, 0x0}, 1},
- {0xFA30, {0x4FAE, 0x0, 0x0}, 1},
- {0xFA31, {0x50E7, 0x0, 0x0}, 1},
- {0xFA32, {0x514D, 0x0, 0x0}, 1},
- {0xFA33, {0x52C9, 0x0, 0x0}, 1},
- {0xFA34, {0x52E4, 0x0, 0x0}, 1},
- {0xFA35, {0x5351, 0x0, 0x0}, 1},
- {0xFA36, {0x559D, 0x0, 0x0}, 1},
- {0xFA37, {0x5606, 0x0, 0x0}, 1},
- {0xFA38, {0x5668, 0x0, 0x0}, 1},
- {0xFA39, {0x5840, 0x0, 0x0}, 1},
- {0xFA3A, {0x58A8, 0x0, 0x0}, 1},
- {0xFA3B, {0x5C64, 0x0, 0x0}, 1},
- {0xFA3C, {0x5C6E, 0x0, 0x0}, 1},
- {0xFA3D, {0x6094, 0x0, 0x0}, 1},
- {0xFA3E, {0x6168, 0x0, 0x0}, 1},
- {0xFA3F, {0x618E, 0x0, 0x0}, 1},
- {0xFA40, {0x61F2, 0x0, 0x0}, 1},
- {0xFA41, {0x654F, 0x0, 0x0}, 1},
- {0xFA42, {0x65E2, 0x0, 0x0}, 1},
- {0xFA43, {0x6691, 0x0, 0x0}, 1},
- {0xFA44, {0x6885, 0x0, 0x0}, 1},
- {0xFA45, {0x6D77, 0x0, 0x0}, 1},
- {0xFA46, {0x6E1A, 0x0, 0x0}, 1},
- {0xFA47, {0x6F22, 0x0, 0x0}, 1},
- {0xFA48, {0x716E, 0x0, 0x0}, 1},
- {0xFA49, {0x722B, 0x0, 0x0}, 1},
- {0xFA4A, {0x7422, 0x0, 0x0}, 1},
- {0xFA4B, {0x7891, 0x0, 0x0}, 1},
- {0xFA4C, {0x793E, 0x0, 0x0}, 1},
- {0xFA4D, {0x7949, 0x0, 0x0}, 1},
- {0xFA4E, {0x7948, 0x0, 0x0}, 1},
- {0xFA4F, {0x7950, 0x0, 0x0}, 1},
- {0xFA50, {0x7956, 0x0, 0x0}, 1},
- {0xFA51, {0x795D, 0x0, 0x0}, 1},
- {0xFA52, {0x798D, 0x0, 0x0}, 1},
- {0xFA53, {0x798E, 0x0, 0x0}, 1},
- {0xFA54, {0x7A40, 0x0, 0x0}, 1},
- {0xFA55, {0x7A81, 0x0, 0x0}, 1},
- {0xFA56, {0x7BC0, 0x0, 0x0}, 1},
- {0xFA57, {0x7DF4, 0x0, 0x0}, 1},
- {0xFA58, {0x7E09, 0x0, 0x0}, 1},
- {0xFA59, {0x7E41, 0x0, 0x0}, 1},
- {0xFA5A, {0x7F72, 0x0, 0x0}, 1},
- {0xFA5B, {0x8005, 0x0, 0x0}, 1},
- {0xFA5C, {0x81ED, 0x0, 0x0}, 1},
- {0xFA5D, {0x8279, 0x0, 0x0}, 1},
- {0xFA5E, {0x8279, 0x0, 0x0}, 1},
- {0xFA5F, {0x8457, 0x0, 0x0}, 1},
- {0xFA60, {0x8910, 0x0, 0x0}, 1},
- {0xFA61, {0x8996, 0x0, 0x0}, 1},
- {0xFA62, {0x8B01, 0x0, 0x0}, 1},
- {0xFA63, {0x8B39, 0x0, 0x0}, 1},
- {0xFA64, {0x8CD3, 0x0, 0x0}, 1},
- {0xFA65, {0x8D08, 0x0, 0x0}, 1},
- {0xFA66, {0x8FB6, 0x0, 0x0}, 1},
- {0xFA67, {0x9038, 0x0, 0x0}, 1},
- {0xFA68, {0x96E3, 0x0, 0x0}, 1},
- {0xFA69, {0x97FF, 0x0, 0x0}, 1},
- {0xFA6A, {0x983B, 0x0, 0x0}, 1},
- {0xFA6B, {0x6075, 0x0, 0x0}, 1},
- {0xFA6C, {0x242EE, 0x0, 0x0}, 1},
- {0xFA6D, {0x8218, 0x0, 0x0}, 1},
- {0xFA70, {0x4E26, 0x0, 0x0}, 1},
- {0xFA71, {0x51B5, 0x0, 0x0}, 1},
- {0xFA72, {0x5168, 0x0, 0x0}, 1},
- {0xFA73, {0x4F80, 0x0, 0x0}, 1},
- {0xFA74, {0x5145, 0x0, 0x0}, 1},
- {0xFA75, {0x5180, 0x0, 0x0}, 1},
- {0xFA76, {0x52C7, 0x0, 0x0}, 1},
- {0xFA77, {0x52FA, 0x0, 0x0}, 1},
- {0xFA78, {0x559D, 0x0, 0x0}, 1},
- {0xFA79, {0x5555, 0x0, 0x0}, 1},
- {0xFA7A, {0x5599, 0x0, 0x0}, 1},
- {0xFA7B, {0x55E2, 0x0, 0x0}, 1},
- {0xFA7C, {0x585A, 0x0, 0x0}, 1},
- {0xFA7D, {0x58B3, 0x0, 0x0}, 1},
- {0xFA7E, {0x5944, 0x0, 0x0}, 1},
- {0xFA7F, {0x5954, 0x0, 0x0}, 1},
- {0xFA80, {0x5A62, 0x0, 0x0}, 1},
- {0xFA81, {0x5B28, 0x0, 0x0}, 1},
- {0xFA82, {0x5ED2, 0x0, 0x0}, 1},
- {0xFA83, {0x5ED9, 0x0, 0x0}, 1},
- {0xFA84, {0x5F69, 0x0, 0x0}, 1},
- {0xFA85, {0x5FAD, 0x0, 0x0}, 1},
- {0xFA86, {0x60D8, 0x0, 0x0}, 1},
- {0xFA87, {0x614E, 0x0, 0x0}, 1},
- {0xFA88, {0x6108, 0x0, 0x0}, 1},
- {0xFA89, {0x618E, 0x0, 0x0}, 1},
- {0xFA8A, {0x6160, 0x0, 0x0}, 1},
- {0xFA8B, {0x61F2, 0x0, 0x0}, 1},
- {0xFA8C, {0x6234, 0x0, 0x0}, 1},
- {0xFA8D, {0x63C4, 0x0, 0x0}, 1},
- {0xFA8E, {0x641C, 0x0, 0x0}, 1},
- {0xFA8F, {0x6452, 0x0, 0x0}, 1},
- {0xFA90, {0x6556, 0x0, 0x0}, 1},
- {0xFA91, {0x6674, 0x0, 0x0}, 1},
- {0xFA92, {0x6717, 0x0, 0x0}, 1},
- {0xFA93, {0x671B, 0x0, 0x0}, 1},
- {0xFA94, {0x6756, 0x0, 0x0}, 1},
- {0xFA95, {0x6B79, 0x0, 0x0}, 1},
- {0xFA96, {0x6BBA, 0x0, 0x0}, 1},
- {0xFA97, {0x6D41, 0x0, 0x0}, 1},
- {0xFA98, {0x6EDB, 0x0, 0x0}, 1},
- {0xFA99, {0x6ECB, 0x0, 0x0}, 1},
- {0xFA9A, {0x6F22, 0x0, 0x0}, 1},
- {0xFA9B, {0x701E, 0x0, 0x0}, 1},
- {0xFA9C, {0x716E, 0x0, 0x0}, 1},
- {0xFA9D, {0x77A7, 0x0, 0x0}, 1},
- {0xFA9E, {0x7235, 0x0, 0x0}, 1},
- {0xFA9F, {0x72AF, 0x0, 0x0}, 1},
- {0xFAA0, {0x732A, 0x0, 0x0}, 1},
- {0xFAA1, {0x7471, 0x0, 0x0}, 1},
- {0xFAA2, {0x7506, 0x0, 0x0}, 1},
- {0xFAA3, {0x753B, 0x0, 0x0}, 1},
- {0xFAA4, {0x761D, 0x0, 0x0}, 1},
- {0xFAA5, {0x761F, 0x0, 0x0}, 1},
- {0xFAA6, {0x76CA, 0x0, 0x0}, 1},
- {0xFAA7, {0x76DB, 0x0, 0x0}, 1},
- {0xFAA8, {0x76F4, 0x0, 0x0}, 1},
- {0xFAA9, {0x774A, 0x0, 0x0}, 1},
- {0xFAAA, {0x7740, 0x0, 0x0}, 1},
- {0xFAAB, {0x78CC, 0x0, 0x0}, 1},
- {0xFAAC, {0x7AB1, 0x0, 0x0}, 1},
- {0xFAAD, {0x7BC0, 0x0, 0x0}, 1},
- {0xFAAE, {0x7C7B, 0x0, 0x0}, 1},
- {0xFAAF, {0x7D5B, 0x0, 0x0}, 1},
- {0xFAB0, {0x7DF4, 0x0, 0x0}, 1},
- {0xFAB1, {0x7F3E, 0x0, 0x0}, 1},
- {0xFAB2, {0x8005, 0x0, 0x0}, 1},
- {0xFAB3, {0x8352, 0x0, 0x0}, 1},
- {0xFAB4, {0x83EF, 0x0, 0x0}, 1},
- {0xFAB5, {0x8779, 0x0, 0x0}, 1},
- {0xFAB6, {0x8941, 0x0, 0x0}, 1},
- {0xFAB7, {0x8986, 0x0, 0x0}, 1},
- {0xFAB8, {0x8996, 0x0, 0x0}, 1},
- {0xFAB9, {0x8ABF, 0x0, 0x0}, 1},
- {0xFABA, {0x8AF8, 0x0, 0x0}, 1},
- {0xFABB, {0x8ACB, 0x0, 0x0}, 1},
- {0xFABC, {0x8B01, 0x0, 0x0}, 1},
- {0xFABD, {0x8AFE, 0x0, 0x0}, 1},
- {0xFABE, {0x8AED, 0x0, 0x0}, 1},
- {0xFABF, {0x8B39, 0x0, 0x0}, 1},
- {0xFAC0, {0x8B8A, 0x0, 0x0}, 1},
- {0xFAC1, {0x8D08, 0x0, 0x0}, 1},
- {0xFAC2, {0x8F38, 0x0, 0x0}, 1},
- {0xFAC3, {0x9072, 0x0, 0x0}, 1},
- {0xFAC4, {0x9199, 0x0, 0x0}, 1},
- {0xFAC5, {0x9276, 0x0, 0x0}, 1},
- {0xFAC6, {0x967C, 0x0, 0x0}, 1},
- {0xFAC7, {0x96E3, 0x0, 0x0}, 1},
- {0xFAC8, {0x9756, 0x0, 0x0}, 1},
- {0xFAC9, {0x97DB, 0x0, 0x0}, 1},
- {0xFACA, {0x97FF, 0x0, 0x0}, 1},
- {0xFACB, {0x980B, 0x0, 0x0}, 1},
- {0xFACC, {0x983B, 0x0, 0x0}, 1},
- {0xFACD, {0x9B12, 0x0, 0x0}, 1},
- {0xFACE, {0x9F9C, 0x0, 0x0}, 1},
- {0xFACF, {0x2284A, 0x0, 0x0}, 1},
- {0xFAD0, {0x22844, 0x0, 0x0}, 1},
- {0xFAD1, {0x233D5, 0x0, 0x0}, 1},
- {0xFAD2, {0x3B9D, 0x0, 0x0}, 1},
- {0xFAD3, {0x4018, 0x0, 0x0}, 1},
- {0xFAD4, {0x4039, 0x0, 0x0}, 1},
- {0xFAD5, {0x25249, 0x0, 0x0}, 1},
- {0xFAD6, {0x25CD0, 0x0, 0x0}, 1},
- {0xFAD7, {0x27ED3, 0x0, 0x0}, 1},
- {0xFAD8, {0x9F43, 0x0, 0x0}, 1},
- {0xFAD9, {0x9F8E, 0x0, 0x0}, 1},
- {0xFB1D, {0x5D9, 0x0, 0x0}, 1},
- {0xFB1E, {0x0, 0x0, 0x0}, 0},
- {0xFB1F, {0x5F2, 0x0, 0x0}, 1},
- {0xFB2A, {0x5E9, 0x0, 0x0}, 1},
- {0xFB2B, {0x5E9, 0x0, 0x0}, 1},
- {0xFB2C, {0x5E9, 0x0, 0x0}, 1},
- {0xFB2D, {0x5E9, 0x0, 0x0}, 1},
- {0xFB2E, {0x5D0, 0x0, 0x0}, 1},
- {0xFB2F, {0x5D0, 0x0, 0x0}, 1},
- {0xFB30, {0x5D0, 0x0, 0x0}, 1},
- {0xFB31, {0x5D1, 0x0, 0x0}, 1},
- {0xFB32, {0x5D2, 0x0, 0x0}, 1},
- {0xFB33, {0x5D3, 0x0, 0x0}, 1},
- {0xFB34, {0x5D4, 0x0, 0x0}, 1},
- {0xFB35, {0x5D5, 0x0, 0x0}, 1},
- {0xFB36, {0x5D6, 0x0, 0x0}, 1},
- {0xFB38, {0x5D8, 0x0, 0x0}, 1},
- {0xFB39, {0x5D9, 0x0, 0x0}, 1},
- {0xFB3A, {0x5DA, 0x0, 0x0}, 1},
- {0xFB3B, {0x5DB, 0x0, 0x0}, 1},
- {0xFB3C, {0x5DC, 0x0, 0x0}, 1},
- {0xFB3E, {0x5DE, 0x0, 0x0}, 1},
- {0xFB40, {0x5E0, 0x0, 0x0}, 1},
- {0xFB41, {0x5E1, 0x0, 0x0}, 1},
- {0xFB43, {0x5E3, 0x0, 0x0}, 1},
- {0xFB44, {0x5E4, 0x0, 0x0}, 1},
- {0xFB46, {0x5E6, 0x0, 0x0}, 1},
- {0xFB47, {0x5E7, 0x0, 0x0}, 1},
- {0xFB48, {0x5E8, 0x0, 0x0}, 1},
- {0xFB49, {0x5E9, 0x0, 0x0}, 1},
- {0xFB4A, {0x5EA, 0x0, 0x0}, 1},
- {0xFB4B, {0x5D5, 0x0, 0x0}, 1},
- {0xFB4C, {0x5D1, 0x0, 0x0}, 1},
- {0xFB4D, {0x5DB, 0x0, 0x0}, 1},
- {0xFB4E, {0x5E4, 0x0, 0x0}, 1},
- {0xFE00, {0x0, 0x0, 0x0}, 0},
- {0xFE01, {0x0, 0x0, 0x0}, 0},
- {0xFE02, {0x0, 0x0, 0x0}, 0},
- {0xFE03, {0x0, 0x0, 0x0}, 0},
- {0xFE04, {0x0, 0x0, 0x0}, 0},
- {0xFE05, {0x0, 0x0, 0x0}, 0},
- {0xFE06, {0x0, 0x0, 0x0}, 0},
- {0xFE07, {0x0, 0x0, 0x0}, 0},
- {0xFE08, {0x0, 0x0, 0x0}, 0},
- {0xFE09, {0x0, 0x0, 0x0}, 0},
- {0xFE0A, {0x0, 0x0, 0x0}, 0},
- {0xFE0B, {0x0, 0x0, 0x0}, 0},
- {0xFE0C, {0x0, 0x0, 0x0}, 0},
- {0xFE0D, {0x0, 0x0, 0x0}, 0},
- {0xFE0E, {0x0, 0x0, 0x0}, 0},
- {0xFE0F, {0x0, 0x0, 0x0}, 0},
- {0xFE20, {0x0, 0x0, 0x0}, 0},
- {0xFE21, {0x0, 0x0, 0x0}, 0},
- {0xFE22, {0x0, 0x0, 0x0}, 0},
- {0xFE23, {0x0, 0x0, 0x0}, 0},
- {0xFE24, {0x0, 0x0, 0x0}, 0},
- {0xFE25, {0x0, 0x0, 0x0}, 0},
- {0xFE26, {0x0, 0x0, 0x0}, 0},
- {0xFE27, {0x0, 0x0, 0x0}, 0},
- {0xFE28, {0x0, 0x0, 0x0}, 0},
- {0xFE29, {0x0, 0x0, 0x0}, 0},
- {0xFE2A, {0x0, 0x0, 0x0}, 0},
- {0xFE2B, {0x0, 0x0, 0x0}, 0},
- {0xFE2C, {0x0, 0x0, 0x0}, 0},
- {0xFE2D, {0x0, 0x0, 0x0}, 0},
- {0xFE2E, {0x0, 0x0, 0x0}, 0},
- {0xFE2F, {0x0, 0x0, 0x0}, 0},
- {0xFF21, {0xFF41, 0x0, 0x0}, 1},
- {0xFF22, {0xFF42, 0x0, 0x0}, 1},
- {0xFF23, {0xFF43, 0x0, 0x0}, 1},
- {0xFF24, {0xFF44, 0x0, 0x0}, 1},
- {0xFF25, {0xFF45, 0x0, 0x0}, 1},
- {0xFF26, {0xFF46, 0x0, 0x0}, 1},
- {0xFF27, {0xFF47, 0x0, 0x0}, 1},
- {0xFF28, {0xFF48, 0x0, 0x0}, 1},
- {0xFF29, {0xFF49, 0x0, 0x0}, 1},
- {0xFF2A, {0xFF4A, 0x0, 0x0}, 1},
- {0xFF2B, {0xFF4B, 0x0, 0x0}, 1},
- {0xFF2C, {0xFF4C, 0x0, 0x0}, 1},
- {0xFF2D, {0xFF4D, 0x0, 0x0}, 1},
- {0xFF2E, {0xFF4E, 0x0, 0x0}, 1},
- {0xFF2F, {0xFF4F, 0x0, 0x0}, 1},
- {0xFF30, {0xFF50, 0x0, 0x0}, 1},
- {0xFF31, {0xFF51, 0x0, 0x0}, 1},
- {0xFF32, {0xFF52, 0x0, 0x0}, 1},
- {0xFF33, {0xFF53, 0x0, 0x0}, 1},
- {0xFF34, {0xFF54, 0x0, 0x0}, 1},
- {0xFF35, {0xFF55, 0x0, 0x0}, 1},
- {0xFF36, {0xFF56, 0x0, 0x0}, 1},
- {0xFF37, {0xFF57, 0x0, 0x0}, 1},
- {0xFF38, {0xFF58, 0x0, 0x0}, 1},
- {0xFF39, {0xFF59, 0x0, 0x0}, 1},
- {0xFF3A, {0xFF5A, 0x0, 0x0}, 1},
- {0x101FD, {0x0, 0x0, 0x0}, 0},
- {0x102E0, {0x0, 0x0, 0x0}, 0},
- {0x10376, {0x0, 0x0, 0x0}, 0},
- {0x10377, {0x0, 0x0, 0x0}, 0},
- {0x10378, {0x0, 0x0, 0x0}, 0},
- {0x10379, {0x0, 0x0, 0x0}, 0},
- {0x1037A, {0x0, 0x0, 0x0}, 0},
- {0x10400, {0x10428, 0x0, 0x0}, 1},
- {0x10401, {0x10429, 0x0, 0x0}, 1},
- {0x10402, {0x1042A, 0x0, 0x0}, 1},
- {0x10403, {0x1042B, 0x0, 0x0}, 1},
- {0x10404, {0x1042C, 0x0, 0x0}, 1},
- {0x10405, {0x1042D, 0x0, 0x0}, 1},
- {0x10406, {0x1042E, 0x0, 0x0}, 1},
- {0x10407, {0x1042F, 0x0, 0x0}, 1},
- {0x10408, {0x10430, 0x0, 0x0}, 1},
- {0x10409, {0x10431, 0x0, 0x0}, 1},
- {0x1040A, {0x10432, 0x0, 0x0}, 1},
- {0x1040B, {0x10433, 0x0, 0x0}, 1},
- {0x1040C, {0x10434, 0x0, 0x0}, 1},
- {0x1040D, {0x10435, 0x0, 0x0}, 1},
- {0x1040E, {0x10436, 0x0, 0x0}, 1},
- {0x1040F, {0x10437, 0x0, 0x0}, 1},
- {0x10410, {0x10438, 0x0, 0x0}, 1},
- {0x10411, {0x10439, 0x0, 0x0}, 1},
- {0x10412, {0x1043A, 0x0, 0x0}, 1},
- {0x10413, {0x1043B, 0x0, 0x0}, 1},
- {0x10414, {0x1043C, 0x0, 0x0}, 1},
- {0x10415, {0x1043D, 0x0, 0x0}, 1},
- {0x10416, {0x1043E, 0x0, 0x0}, 1},
- {0x10417, {0x1043F, 0x0, 0x0}, 1},
- {0x10418, {0x10440, 0x0, 0x0}, 1},
- {0x10419, {0x10441, 0x0, 0x0}, 1},
- {0x1041A, {0x10442, 0x0, 0x0}, 1},
- {0x1041B, {0x10443, 0x0, 0x0}, 1},
- {0x1041C, {0x10444, 0x0, 0x0}, 1},
- {0x1041D, {0x10445, 0x0, 0x0}, 1},
- {0x1041E, {0x10446, 0x0, 0x0}, 1},
- {0x1041F, {0x10447, 0x0, 0x0}, 1},
- {0x10420, {0x10448, 0x0, 0x0}, 1},
- {0x10421, {0x10449, 0x0, 0x0}, 1},
- {0x10422, {0x1044A, 0x0, 0x0}, 1},
- {0x10423, {0x1044B, 0x0, 0x0}, 1},
- {0x10424, {0x1044C, 0x0, 0x0}, 1},
- {0x10425, {0x1044D, 0x0, 0x0}, 1},
- {0x10426, {0x1044E, 0x0, 0x0}, 1},
- {0x10427, {0x1044F, 0x0, 0x0}, 1},
- {0x104B0, {0x104D8, 0x0, 0x0}, 1},
- {0x104B1, {0x104D9, 0x0, 0x0}, 1},
- {0x104B2, {0x104DA, 0x0, 0x0}, 1},
- {0x104B3, {0x104DB, 0x0, 0x0}, 1},
- {0x104B4, {0x104DC, 0x0, 0x0}, 1},
- {0x104B5, {0x104DD, 0x0, 0x0}, 1},
- {0x104B6, {0x104DE, 0x0, 0x0}, 1},
- {0x104B7, {0x104DF, 0x0, 0x0}, 1},
- {0x104B8, {0x104E0, 0x0, 0x0}, 1},
- {0x104B9, {0x104E1, 0x0, 0x0}, 1},
- {0x104BA, {0x104E2, 0x0, 0x0}, 1},
- {0x104BB, {0x104E3, 0x0, 0x0}, 1},
- {0x104BC, {0x104E4, 0x0, 0x0}, 1},
- {0x104BD, {0x104E5, 0x0, 0x0}, 1},
- {0x104BE, {0x104E6, 0x0, 0x0}, 1},
- {0x104BF, {0x104E7, 0x0, 0x0}, 1},
- {0x104C0, {0x104E8, 0x0, 0x0}, 1},
- {0x104C1, {0x104E9, 0x0, 0x0}, 1},
- {0x104C2, {0x104EA, 0x0, 0x0}, 1},
- {0x104C3, {0x104EB, 0x0, 0x0}, 1},
- {0x104C4, {0x104EC, 0x0, 0x0}, 1},
- {0x104C5, {0x104ED, 0x0, 0x0}, 1},
- {0x104C6, {0x104EE, 0x0, 0x0}, 1},
- {0x104C7, {0x104EF, 0x0, 0x0}, 1},
- {0x104C8, {0x104F0, 0x0, 0x0}, 1},
- {0x104C9, {0x104F1, 0x0, 0x0}, 1},
- {0x104CA, {0x104F2, 0x0, 0x0}, 1},
- {0x104CB, {0x104F3, 0x0, 0x0}, 1},
- {0x104CC, {0x104F4, 0x0, 0x0}, 1},
- {0x104CD, {0x104F5, 0x0, 0x0}, 1},
- {0x104CE, {0x104F6, 0x0, 0x0}, 1},
- {0x104CF, {0x104F7, 0x0, 0x0}, 1},
- {0x104D0, {0x104F8, 0x0, 0x0}, 1},
- {0x104D1, {0x104F9, 0x0, 0x0}, 1},
- {0x104D2, {0x104FA, 0x0, 0x0}, 1},
- {0x104D3, {0x104FB, 0x0, 0x0}, 1},
- {0x10A01, {0x0, 0x0, 0x0}, 0},
- {0x10A02, {0x0, 0x0, 0x0}, 0},
- {0x10A03, {0x0, 0x0, 0x0}, 0},
- {0x10A05, {0x0, 0x0, 0x0}, 0},
- {0x10A06, {0x0, 0x0, 0x0}, 0},
- {0x10A0C, {0x0, 0x0, 0x0}, 0},
- {0x10A0D, {0x0, 0x0, 0x0}, 0},
- {0x10A0E, {0x0, 0x0, 0x0}, 0},
- {0x10A0F, {0x0, 0x0, 0x0}, 0},
- {0x10A38, {0x0, 0x0, 0x0}, 0},
- {0x10A39, {0x0, 0x0, 0x0}, 0},
- {0x10A3A, {0x0, 0x0, 0x0}, 0},
- {0x10A3F, {0x0, 0x0, 0x0}, 0},
- {0x10AE5, {0x0, 0x0, 0x0}, 0},
- {0x10AE6, {0x0, 0x0, 0x0}, 0},
- {0x10C80, {0x10CC0, 0x0, 0x0}, 1},
- {0x10C81, {0x10CC1, 0x0, 0x0}, 1},
- {0x10C82, {0x10CC2, 0x0, 0x0}, 1},
- {0x10C83, {0x10CC3, 0x0, 0x0}, 1},
- {0x10C84, {0x10CC4, 0x0, 0x0}, 1},
- {0x10C85, {0x10CC5, 0x0, 0x0}, 1},
- {0x10C86, {0x10CC6, 0x0, 0x0}, 1},
- {0x10C87, {0x10CC7, 0x0, 0x0}, 1},
- {0x10C88, {0x10CC8, 0x0, 0x0}, 1},
- {0x10C89, {0x10CC9, 0x0, 0x0}, 1},
- {0x10C8A, {0x10CCA, 0x0, 0x0}, 1},
- {0x10C8B, {0x10CCB, 0x0, 0x0}, 1},
- {0x10C8C, {0x10CCC, 0x0, 0x0}, 1},
- {0x10C8D, {0x10CCD, 0x0, 0x0}, 1},
- {0x10C8E, {0x10CCE, 0x0, 0x0}, 1},
- {0x10C8F, {0x10CCF, 0x0, 0x0}, 1},
- {0x10C90, {0x10CD0, 0x0, 0x0}, 1},
- {0x10C91, {0x10CD1, 0x0, 0x0}, 1},
- {0x10C92, {0x10CD2, 0x0, 0x0}, 1},
- {0x10C93, {0x10CD3, 0x0, 0x0}, 1},
- {0x10C94, {0x10CD4, 0x0, 0x0}, 1},
- {0x10C95, {0x10CD5, 0x0, 0x0}, 1},
- {0x10C96, {0x10CD6, 0x0, 0x0}, 1},
- {0x10C97, {0x10CD7, 0x0, 0x0}, 1},
- {0x10C98, {0x10CD8, 0x0, 0x0}, 1},
- {0x10C99, {0x10CD9, 0x0, 0x0}, 1},
- {0x10C9A, {0x10CDA, 0x0, 0x0}, 1},
- {0x10C9B, {0x10CDB, 0x0, 0x0}, 1},
- {0x10C9C, {0x10CDC, 0x0, 0x0}, 1},
- {0x10C9D, {0x10CDD, 0x0, 0x0}, 1},
- {0x10C9E, {0x10CDE, 0x0, 0x0}, 1},
- {0x10C9F, {0x10CDF, 0x0, 0x0}, 1},
- {0x10CA0, {0x10CE0, 0x0, 0x0}, 1},
- {0x10CA1, {0x10CE1, 0x0, 0x0}, 1},
- {0x10CA2, {0x10CE2, 0x0, 0x0}, 1},
- {0x10CA3, {0x10CE3, 0x0, 0x0}, 1},
- {0x10CA4, {0x10CE4, 0x0, 0x0}, 1},
- {0x10CA5, {0x10CE5, 0x0, 0x0}, 1},
- {0x10CA6, {0x10CE6, 0x0, 0x0}, 1},
- {0x10CA7, {0x10CE7, 0x0, 0x0}, 1},
- {0x10CA8, {0x10CE8, 0x0, 0x0}, 1},
- {0x10CA9, {0x10CE9, 0x0, 0x0}, 1},
- {0x10CAA, {0x10CEA, 0x0, 0x0}, 1},
- {0x10CAB, {0x10CEB, 0x0, 0x0}, 1},
- {0x10CAC, {0x10CEC, 0x0, 0x0}, 1},
- {0x10CAD, {0x10CED, 0x0, 0x0}, 1},
- {0x10CAE, {0x10CEE, 0x0, 0x0}, 1},
- {0x10CAF, {0x10CEF, 0x0, 0x0}, 1},
- {0x10CB0, {0x10CF0, 0x0, 0x0}, 1},
- {0x10CB1, {0x10CF1, 0x0, 0x0}, 1},
- {0x10CB2, {0x10CF2, 0x0, 0x0}, 1},
- {0x10D24, {0x0, 0x0, 0x0}, 0},
- {0x10D25, {0x0, 0x0, 0x0}, 0},
- {0x10D26, {0x0, 0x0, 0x0}, 0},
- {0x10D27, {0x0, 0x0, 0x0}, 0},
- {0x10EAB, {0x0, 0x0, 0x0}, 0},
- {0x10EAC, {0x0, 0x0, 0x0}, 0},
- {0x10F46, {0x0, 0x0, 0x0}, 0},
- {0x10F47, {0x0, 0x0, 0x0}, 0},
- {0x10F48, {0x0, 0x0, 0x0}, 0},
- {0x10F49, {0x0, 0x0, 0x0}, 0},
- {0x10F4A, {0x0, 0x0, 0x0}, 0},
- {0x10F4B, {0x0, 0x0, 0x0}, 0},
- {0x10F4C, {0x0, 0x0, 0x0}, 0},
- {0x10F4D, {0x0, 0x0, 0x0}, 0},
- {0x10F4E, {0x0, 0x0, 0x0}, 0},
- {0x10F4F, {0x0, 0x0, 0x0}, 0},
- {0x10F50, {0x0, 0x0, 0x0}, 0},
- {0x11001, {0x0, 0x0, 0x0}, 0},
- {0x11038, {0x0, 0x0, 0x0}, 0},
- {0x11039, {0x0, 0x0, 0x0}, 0},
- {0x1103A, {0x0, 0x0, 0x0}, 0},
- {0x1103B, {0x0, 0x0, 0x0}, 0},
- {0x1103C, {0x0, 0x0, 0x0}, 0},
- {0x1103D, {0x0, 0x0, 0x0}, 0},
- {0x1103E, {0x0, 0x0, 0x0}, 0},
- {0x1103F, {0x0, 0x0, 0x0}, 0},
- {0x11040, {0x0, 0x0, 0x0}, 0},
- {0x11041, {0x0, 0x0, 0x0}, 0},
- {0x11042, {0x0, 0x0, 0x0}, 0},
- {0x11043, {0x0, 0x0, 0x0}, 0},
- {0x11044, {0x0, 0x0, 0x0}, 0},
- {0x11045, {0x0, 0x0, 0x0}, 0},
- {0x11046, {0x0, 0x0, 0x0}, 0},
- {0x1107F, {0x0, 0x0, 0x0}, 0},
- {0x11080, {0x0, 0x0, 0x0}, 0},
- {0x11081, {0x0, 0x0, 0x0}, 0},
- {0x1109A, {0x11099, 0x0, 0x0}, 1},
- {0x1109C, {0x1109B, 0x0, 0x0}, 1},
- {0x110AB, {0x110A5, 0x0, 0x0}, 1},
- {0x110B3, {0x0, 0x0, 0x0}, 0},
- {0x110B4, {0x0, 0x0, 0x0}, 0},
- {0x110B5, {0x0, 0x0, 0x0}, 0},
- {0x110B6, {0x0, 0x0, 0x0}, 0},
- {0x110B9, {0x0, 0x0, 0x0}, 0},
- {0x110BA, {0x0, 0x0, 0x0}, 0},
- {0x11100, {0x0, 0x0, 0x0}, 0},
- {0x11101, {0x0, 0x0, 0x0}, 0},
- {0x11102, {0x0, 0x0, 0x0}, 0},
- {0x11127, {0x0, 0x0, 0x0}, 0},
- {0x11128, {0x0, 0x0, 0x0}, 0},
- {0x11129, {0x0, 0x0, 0x0}, 0},
- {0x1112A, {0x0, 0x0, 0x0}, 0},
- {0x1112B, {0x0, 0x0, 0x0}, 0},
- {0x1112D, {0x0, 0x0, 0x0}, 0},
- {0x1112E, {0x0, 0x0, 0x0}, 0},
- {0x1112F, {0x0, 0x0, 0x0}, 0},
- {0x11130, {0x0, 0x0, 0x0}, 0},
- {0x11131, {0x0, 0x0, 0x0}, 0},
- {0x11132, {0x0, 0x0, 0x0}, 0},
- {0x11133, {0x0, 0x0, 0x0}, 0},
- {0x11134, {0x0, 0x0, 0x0}, 0},
- {0x11173, {0x0, 0x0, 0x0}, 0},
- {0x11180, {0x0, 0x0, 0x0}, 0},
- {0x11181, {0x0, 0x0, 0x0}, 0},
- {0x111B6, {0x0, 0x0, 0x0}, 0},
- {0x111B7, {0x0, 0x0, 0x0}, 0},
- {0x111B8, {0x0, 0x0, 0x0}, 0},
- {0x111B9, {0x0, 0x0, 0x0}, 0},
- {0x111BA, {0x0, 0x0, 0x0}, 0},
- {0x111BB, {0x0, 0x0, 0x0}, 0},
- {0x111BC, {0x0, 0x0, 0x0}, 0},
- {0x111BD, {0x0, 0x0, 0x0}, 0},
- {0x111BE, {0x0, 0x0, 0x0}, 0},
- {0x111C9, {0x0, 0x0, 0x0}, 0},
- {0x111CA, {0x0, 0x0, 0x0}, 0},
- {0x111CB, {0x0, 0x0, 0x0}, 0},
- {0x111CC, {0x0, 0x0, 0x0}, 0},
- {0x111CF, {0x0, 0x0, 0x0}, 0},
- {0x1122F, {0x0, 0x0, 0x0}, 0},
- {0x11230, {0x0, 0x0, 0x0}, 0},
- {0x11231, {0x0, 0x0, 0x0}, 0},
- {0x11234, {0x0, 0x0, 0x0}, 0},
- {0x11236, {0x0, 0x0, 0x0}, 0},
- {0x11237, {0x0, 0x0, 0x0}, 0},
- {0x1123E, {0x0, 0x0, 0x0}, 0},
- {0x112DF, {0x0, 0x0, 0x0}, 0},
- {0x112E3, {0x0, 0x0, 0x0}, 0},
- {0x112E4, {0x0, 0x0, 0x0}, 0},
- {0x112E5, {0x0, 0x0, 0x0}, 0},
- {0x112E6, {0x0, 0x0, 0x0}, 0},
- {0x112E7, {0x0, 0x0, 0x0}, 0},
- {0x112E8, {0x0, 0x0, 0x0}, 0},
- {0x112E9, {0x0, 0x0, 0x0}, 0},
- {0x112EA, {0x0, 0x0, 0x0}, 0},
- {0x11300, {0x0, 0x0, 0x0}, 0},
- {0x11301, {0x0, 0x0, 0x0}, 0},
- {0x1133B, {0x0, 0x0, 0x0}, 0},
- {0x1133C, {0x0, 0x0, 0x0}, 0},
- {0x11340, {0x0, 0x0, 0x0}, 0},
- {0x1134B, {0x11347, 0x1133E, 0x0}, 2},
- {0x1134C, {0x11347, 0x11357, 0x0}, 2},
- {0x11366, {0x0, 0x0, 0x0}, 0},
- {0x11367, {0x0, 0x0, 0x0}, 0},
- {0x11368, {0x0, 0x0, 0x0}, 0},
- {0x11369, {0x0, 0x0, 0x0}, 0},
- {0x1136A, {0x0, 0x0, 0x0}, 0},
- {0x1136B, {0x0, 0x0, 0x0}, 0},
- {0x1136C, {0x0, 0x0, 0x0}, 0},
- {0x11370, {0x0, 0x0, 0x0}, 0},
- {0x11371, {0x0, 0x0, 0x0}, 0},
- {0x11372, {0x0, 0x0, 0x0}, 0},
- {0x11373, {0x0, 0x0, 0x0}, 0},
- {0x11374, {0x0, 0x0, 0x0}, 0},
- {0x11438, {0x0, 0x0, 0x0}, 0},
- {0x11439, {0x0, 0x0, 0x0}, 0},
- {0x1143A, {0x0, 0x0, 0x0}, 0},
- {0x1143B, {0x0, 0x0, 0x0}, 0},
- {0x1143C, {0x0, 0x0, 0x0}, 0},
- {0x1143D, {0x0, 0x0, 0x0}, 0},
- {0x1143E, {0x0, 0x0, 0x0}, 0},
- {0x1143F, {0x0, 0x0, 0x0}, 0},
- {0x11442, {0x0, 0x0, 0x0}, 0},
- {0x11443, {0x0, 0x0, 0x0}, 0},
- {0x11444, {0x0, 0x0, 0x0}, 0},
- {0x11446, {0x0, 0x0, 0x0}, 0},
- {0x1145E, {0x0, 0x0, 0x0}, 0},
- {0x114B3, {0x0, 0x0, 0x0}, 0},
- {0x114B4, {0x0, 0x0, 0x0}, 0},
- {0x114B5, {0x0, 0x0, 0x0}, 0},
- {0x114B6, {0x0, 0x0, 0x0}, 0},
- {0x114B7, {0x0, 0x0, 0x0}, 0},
- {0x114B8, {0x0, 0x0, 0x0}, 0},
- {0x114BA, {0x0, 0x0, 0x0}, 0},
- {0x114BB, {0x114B9, 0x0, 0x0}, 1},
- {0x114BC, {0x114B9, 0x114B0, 0x0}, 2},
- {0x114BE, {0x114B9, 0x114BD, 0x0}, 2},
- {0x114BF, {0x0, 0x0, 0x0}, 0},
- {0x114C0, {0x0, 0x0, 0x0}, 0},
- {0x114C2, {0x0, 0x0, 0x0}, 0},
- {0x114C3, {0x0, 0x0, 0x0}, 0},
- {0x115B2, {0x0, 0x0, 0x0}, 0},
- {0x115B3, {0x0, 0x0, 0x0}, 0},
- {0x115B4, {0x0, 0x0, 0x0}, 0},
- {0x115B5, {0x0, 0x0, 0x0}, 0},
- {0x115BA, {0x115B8, 0x115AF, 0x0}, 2},
- {0x115BB, {0x115B9, 0x115AF, 0x0}, 2},
- {0x115BC, {0x0, 0x0, 0x0}, 0},
- {0x115BD, {0x0, 0x0, 0x0}, 0},
- {0x115BF, {0x0, 0x0, 0x0}, 0},
- {0x115C0, {0x0, 0x0, 0x0}, 0},
- {0x115DC, {0x0, 0x0, 0x0}, 0},
- {0x115DD, {0x0, 0x0, 0x0}, 0},
- {0x11633, {0x0, 0x0, 0x0}, 0},
- {0x11634, {0x0, 0x0, 0x0}, 0},
- {0x11635, {0x0, 0x0, 0x0}, 0},
- {0x11636, {0x0, 0x0, 0x0}, 0},
- {0x11637, {0x0, 0x0, 0x0}, 0},
- {0x11638, {0x0, 0x0, 0x0}, 0},
- {0x11639, {0x0, 0x0, 0x0}, 0},
- {0x1163A, {0x0, 0x0, 0x0}, 0},
- {0x1163D, {0x0, 0x0, 0x0}, 0},
- {0x1163F, {0x0, 0x0, 0x0}, 0},
- {0x11640, {0x0, 0x0, 0x0}, 0},
- {0x116AB, {0x0, 0x0, 0x0}, 0},
- {0x116AD, {0x0, 0x0, 0x0}, 0},
- {0x116B0, {0x0, 0x0, 0x0}, 0},
- {0x116B1, {0x0, 0x0, 0x0}, 0},
- {0x116B2, {0x0, 0x0, 0x0}, 0},
- {0x116B3, {0x0, 0x0, 0x0}, 0},
- {0x116B4, {0x0, 0x0, 0x0}, 0},
- {0x116B5, {0x0, 0x0, 0x0}, 0},
- {0x116B7, {0x0, 0x0, 0x0}, 0},
- {0x1171D, {0x0, 0x0, 0x0}, 0},
- {0x1171E, {0x0, 0x0, 0x0}, 0},
- {0x1171F, {0x0, 0x0, 0x0}, 0},
- {0x11722, {0x0, 0x0, 0x0}, 0},
- {0x11723, {0x0, 0x0, 0x0}, 0},
- {0x11724, {0x0, 0x0, 0x0}, 0},
- {0x11725, {0x0, 0x0, 0x0}, 0},
- {0x11727, {0x0, 0x0, 0x0}, 0},
- {0x11728, {0x0, 0x0, 0x0}, 0},
- {0x11729, {0x0, 0x0, 0x0}, 0},
- {0x1172A, {0x0, 0x0, 0x0}, 0},
- {0x1172B, {0x0, 0x0, 0x0}, 0},
- {0x1182F, {0x0, 0x0, 0x0}, 0},
- {0x11830, {0x0, 0x0, 0x0}, 0},
- {0x11831, {0x0, 0x0, 0x0}, 0},
- {0x11832, {0x0, 0x0, 0x0}, 0},
- {0x11833, {0x0, 0x0, 0x0}, 0},
- {0x11834, {0x0, 0x0, 0x0}, 0},
- {0x11835, {0x0, 0x0, 0x0}, 0},
- {0x11836, {0x0, 0x0, 0x0}, 0},
- {0x11837, {0x0, 0x0, 0x0}, 0},
- {0x11839, {0x0, 0x0, 0x0}, 0},
- {0x1183A, {0x0, 0x0, 0x0}, 0},
- {0x118A0, {0x118C0, 0x0, 0x0}, 1},
- {0x118A1, {0x118C1, 0x0, 0x0}, 1},
- {0x118A2, {0x118C2, 0x0, 0x0}, 1},
- {0x118A3, {0x118C3, 0x0, 0x0}, 1},
- {0x118A4, {0x118C4, 0x0, 0x0}, 1},
- {0x118A5, {0x118C5, 0x0, 0x0}, 1},
- {0x118A6, {0x118C6, 0x0, 0x0}, 1},
- {0x118A7, {0x118C7, 0x0, 0x0}, 1},
- {0x118A8, {0x118C8, 0x0, 0x0}, 1},
- {0x118A9, {0x118C9, 0x0, 0x0}, 1},
- {0x118AA, {0x118CA, 0x0, 0x0}, 1},
- {0x118AB, {0x118CB, 0x0, 0x0}, 1},
- {0x118AC, {0x118CC, 0x0, 0x0}, 1},
- {0x118AD, {0x118CD, 0x0, 0x0}, 1},
- {0x118AE, {0x118CE, 0x0, 0x0}, 1},
- {0x118AF, {0x118CF, 0x0, 0x0}, 1},
- {0x118B0, {0x118D0, 0x0, 0x0}, 1},
- {0x118B1, {0x118D1, 0x0, 0x0}, 1},
- {0x118B2, {0x118D2, 0x0, 0x0}, 1},
- {0x118B3, {0x118D3, 0x0, 0x0}, 1},
- {0x118B4, {0x118D4, 0x0, 0x0}, 1},
- {0x118B5, {0x118D5, 0x0, 0x0}, 1},
- {0x118B6, {0x118D6, 0x0, 0x0}, 1},
- {0x118B7, {0x118D7, 0x0, 0x0}, 1},
- {0x118B8, {0x118D8, 0x0, 0x0}, 1},
- {0x118B9, {0x118D9, 0x0, 0x0}, 1},
- {0x118BA, {0x118DA, 0x0, 0x0}, 1},
- {0x118BB, {0x118DB, 0x0, 0x0}, 1},
- {0x118BC, {0x118DC, 0x0, 0x0}, 1},
- {0x118BD, {0x118DD, 0x0, 0x0}, 1},
- {0x118BE, {0x118DE, 0x0, 0x0}, 1},
- {0x118BF, {0x118DF, 0x0, 0x0}, 1},
- {0x11938, {0x11935, 0x11930, 0x0}, 2},
- {0x1193B, {0x0, 0x0, 0x0}, 0},
- {0x1193C, {0x0, 0x0, 0x0}, 0},
- {0x1193E, {0x0, 0x0, 0x0}, 0},
- {0x11943, {0x0, 0x0, 0x0}, 0},
- {0x119D4, {0x0, 0x0, 0x0}, 0},
- {0x119D5, {0x0, 0x0, 0x0}, 0},
- {0x119D6, {0x0, 0x0, 0x0}, 0},
- {0x119D7, {0x0, 0x0, 0x0}, 0},
- {0x119DA, {0x0, 0x0, 0x0}, 0},
- {0x119DB, {0x0, 0x0, 0x0}, 0},
- {0x119E0, {0x0, 0x0, 0x0}, 0},
- {0x11A01, {0x0, 0x0, 0x0}, 0},
- {0x11A02, {0x0, 0x0, 0x0}, 0},
- {0x11A03, {0x0, 0x0, 0x0}, 0},
- {0x11A04, {0x0, 0x0, 0x0}, 0},
- {0x11A05, {0x0, 0x0, 0x0}, 0},
- {0x11A06, {0x0, 0x0, 0x0}, 0},
- {0x11A07, {0x0, 0x0, 0x0}, 0},
- {0x11A08, {0x0, 0x0, 0x0}, 0},
- {0x11A09, {0x0, 0x0, 0x0}, 0},
- {0x11A0A, {0x0, 0x0, 0x0}, 0},
- {0x11A33, {0x0, 0x0, 0x0}, 0},
- {0x11A34, {0x0, 0x0, 0x0}, 0},
- {0x11A35, {0x0, 0x0, 0x0}, 0},
- {0x11A36, {0x0, 0x0, 0x0}, 0},
- {0x11A37, {0x0, 0x0, 0x0}, 0},
- {0x11A38, {0x0, 0x0, 0x0}, 0},
- {0x11A3B, {0x0, 0x0, 0x0}, 0},
- {0x11A3C, {0x0, 0x0, 0x0}, 0},
- {0x11A3D, {0x0, 0x0, 0x0}, 0},
- {0x11A3E, {0x0, 0x0, 0x0}, 0},
- {0x11A47, {0x0, 0x0, 0x0}, 0},
- {0x11A51, {0x0, 0x0, 0x0}, 0},
- {0x11A52, {0x0, 0x0, 0x0}, 0},
- {0x11A53, {0x0, 0x0, 0x0}, 0},
- {0x11A54, {0x0, 0x0, 0x0}, 0},
- {0x11A55, {0x0, 0x0, 0x0}, 0},
- {0x11A56, {0x0, 0x0, 0x0}, 0},
- {0x11A59, {0x0, 0x0, 0x0}, 0},
- {0x11A5A, {0x0, 0x0, 0x0}, 0},
- {0x11A5B, {0x0, 0x0, 0x0}, 0},
- {0x11A8A, {0x0, 0x0, 0x0}, 0},
- {0x11A8B, {0x0, 0x0, 0x0}, 0},
- {0x11A8C, {0x0, 0x0, 0x0}, 0},
- {0x11A8D, {0x0, 0x0, 0x0}, 0},
- {0x11A8E, {0x0, 0x0, 0x0}, 0},
- {0x11A8F, {0x0, 0x0, 0x0}, 0},
- {0x11A90, {0x0, 0x0, 0x0}, 0},
- {0x11A91, {0x0, 0x0, 0x0}, 0},
- {0x11A92, {0x0, 0x0, 0x0}, 0},
- {0x11A93, {0x0, 0x0, 0x0}, 0},
- {0x11A94, {0x0, 0x0, 0x0}, 0},
- {0x11A95, {0x0, 0x0, 0x0}, 0},
- {0x11A96, {0x0, 0x0, 0x0}, 0},
- {0x11A98, {0x0, 0x0, 0x0}, 0},
- {0x11A99, {0x0, 0x0, 0x0}, 0},
- {0x11C30, {0x0, 0x0, 0x0}, 0},
- {0x11C31, {0x0, 0x0, 0x0}, 0},
- {0x11C32, {0x0, 0x0, 0x0}, 0},
- {0x11C33, {0x0, 0x0, 0x0}, 0},
- {0x11C34, {0x0, 0x0, 0x0}, 0},
- {0x11C35, {0x0, 0x0, 0x0}, 0},
- {0x11C36, {0x0, 0x0, 0x0}, 0},
- {0x11C38, {0x0, 0x0, 0x0}, 0},
- {0x11C39, {0x0, 0x0, 0x0}, 0},
- {0x11C3A, {0x0, 0x0, 0x0}, 0},
- {0x11C3B, {0x0, 0x0, 0x0}, 0},
- {0x11C3C, {0x0, 0x0, 0x0}, 0},
- {0x11C3D, {0x0, 0x0, 0x0}, 0},
- {0x11C3F, {0x0, 0x0, 0x0}, 0},
- {0x11C92, {0x0, 0x0, 0x0}, 0},
- {0x11C93, {0x0, 0x0, 0x0}, 0},
- {0x11C94, {0x0, 0x0, 0x0}, 0},
- {0x11C95, {0x0, 0x0, 0x0}, 0},
- {0x11C96, {0x0, 0x0, 0x0}, 0},
- {0x11C97, {0x0, 0x0, 0x0}, 0},
- {0x11C98, {0x0, 0x0, 0x0}, 0},
- {0x11C99, {0x0, 0x0, 0x0}, 0},
- {0x11C9A, {0x0, 0x0, 0x0}, 0},
- {0x11C9B, {0x0, 0x0, 0x0}, 0},
- {0x11C9C, {0x0, 0x0, 0x0}, 0},
- {0x11C9D, {0x0, 0x0, 0x0}, 0},
- {0x11C9E, {0x0, 0x0, 0x0}, 0},
- {0x11C9F, {0x0, 0x0, 0x0}, 0},
- {0x11CA0, {0x0, 0x0, 0x0}, 0},
- {0x11CA1, {0x0, 0x0, 0x0}, 0},
- {0x11CA2, {0x0, 0x0, 0x0}, 0},
- {0x11CA3, {0x0, 0x0, 0x0}, 0},
- {0x11CA4, {0x0, 0x0, 0x0}, 0},
- {0x11CA5, {0x0, 0x0, 0x0}, 0},
- {0x11CA6, {0x0, 0x0, 0x0}, 0},
- {0x11CA7, {0x0, 0x0, 0x0}, 0},
- {0x11CAA, {0x0, 0x0, 0x0}, 0},
- {0x11CAB, {0x0, 0x0, 0x0}, 0},
- {0x11CAC, {0x0, 0x0, 0x0}, 0},
- {0x11CAD, {0x0, 0x0, 0x0}, 0},
- {0x11CAE, {0x0, 0x0, 0x0}, 0},
- {0x11CAF, {0x0, 0x0, 0x0}, 0},
- {0x11CB0, {0x0, 0x0, 0x0}, 0},
- {0x11CB2, {0x0, 0x0, 0x0}, 0},
- {0x11CB3, {0x0, 0x0, 0x0}, 0},
- {0x11CB5, {0x0, 0x0, 0x0}, 0},
- {0x11CB6, {0x0, 0x0, 0x0}, 0},
- {0x11D31, {0x0, 0x0, 0x0}, 0},
- {0x11D32, {0x0, 0x0, 0x0}, 0},
- {0x11D33, {0x0, 0x0, 0x0}, 0},
- {0x11D34, {0x0, 0x0, 0x0}, 0},
- {0x11D35, {0x0, 0x0, 0x0}, 0},
- {0x11D36, {0x0, 0x0, 0x0}, 0},
- {0x11D3A, {0x0, 0x0, 0x0}, 0},
- {0x11D3C, {0x0, 0x0, 0x0}, 0},
- {0x11D3D, {0x0, 0x0, 0x0}, 0},
- {0x11D3F, {0x0, 0x0, 0x0}, 0},
- {0x11D40, {0x0, 0x0, 0x0}, 0},
- {0x11D41, {0x0, 0x0, 0x0}, 0},
- {0x11D42, {0x0, 0x0, 0x0}, 0},
- {0x11D43, {0x0, 0x0, 0x0}, 0},
- {0x11D44, {0x0, 0x0, 0x0}, 0},
- {0x11D45, {0x0, 0x0, 0x0}, 0},
- {0x11D47, {0x0, 0x0, 0x0}, 0},
- {0x11D90, {0x0, 0x0, 0x0}, 0},
- {0x11D91, {0x0, 0x0, 0x0}, 0},
- {0x11D95, {0x0, 0x0, 0x0}, 0},
- {0x11D97, {0x0, 0x0, 0x0}, 0},
- {0x11EF3, {0x0, 0x0, 0x0}, 0},
- {0x11EF4, {0x0, 0x0, 0x0}, 0},
- {0x16AF0, {0x0, 0x0, 0x0}, 0},
- {0x16AF1, {0x0, 0x0, 0x0}, 0},
- {0x16AF2, {0x0, 0x0, 0x0}, 0},
- {0x16AF3, {0x0, 0x0, 0x0}, 0},
- {0x16AF4, {0x0, 0x0, 0x0}, 0},
- {0x16B30, {0x0, 0x0, 0x0}, 0},
- {0x16B31, {0x0, 0x0, 0x0}, 0},
- {0x16B32, {0x0, 0x0, 0x0}, 0},
- {0x16B33, {0x0, 0x0, 0x0}, 0},
- {0x16B34, {0x0, 0x0, 0x0}, 0},
- {0x16B35, {0x0, 0x0, 0x0}, 0},
- {0x16B36, {0x0, 0x0, 0x0}, 0},
- {0x16E40, {0x16E60, 0x0, 0x0}, 1},
- {0x16E41, {0x16E61, 0x0, 0x0}, 1},
- {0x16E42, {0x16E62, 0x0, 0x0}, 1},
- {0x16E43, {0x16E63, 0x0, 0x0}, 1},
- {0x16E44, {0x16E64, 0x0, 0x0}, 1},
- {0x16E45, {0x16E65, 0x0, 0x0}, 1},
- {0x16E46, {0x16E66, 0x0, 0x0}, 1},
- {0x16E47, {0x16E67, 0x0, 0x0}, 1},
- {0x16E48, {0x16E68, 0x0, 0x0}, 1},
- {0x16E49, {0x16E69, 0x0, 0x0}, 1},
- {0x16E4A, {0x16E6A, 0x0, 0x0}, 1},
- {0x16E4B, {0x16E6B, 0x0, 0x0}, 1},
- {0x16E4C, {0x16E6C, 0x0, 0x0}, 1},
- {0x16E4D, {0x16E6D, 0x0, 0x0}, 1},
- {0x16E4E, {0x16E6E, 0x0, 0x0}, 1},
- {0x16E4F, {0x16E6F, 0x0, 0x0}, 1},
- {0x16E50, {0x16E70, 0x0, 0x0}, 1},
- {0x16E51, {0x16E71, 0x0, 0x0}, 1},
- {0x16E52, {0x16E72, 0x0, 0x0}, 1},
- {0x16E53, {0x16E73, 0x0, 0x0}, 1},
- {0x16E54, {0x16E74, 0x0, 0x0}, 1},
- {0x16E55, {0x16E75, 0x0, 0x0}, 1},
- {0x16E56, {0x16E76, 0x0, 0x0}, 1},
- {0x16E57, {0x16E77, 0x0, 0x0}, 1},
- {0x16E58, {0x16E78, 0x0, 0x0}, 1},
- {0x16E59, {0x16E79, 0x0, 0x0}, 1},
- {0x16E5A, {0x16E7A, 0x0, 0x0}, 1},
- {0x16E5B, {0x16E7B, 0x0, 0x0}, 1},
- {0x16E5C, {0x16E7C, 0x0, 0x0}, 1},
- {0x16E5D, {0x16E7D, 0x0, 0x0}, 1},
- {0x16E5E, {0x16E7E, 0x0, 0x0}, 1},
- {0x16E5F, {0x16E7F, 0x0, 0x0}, 1},
- {0x16F4F, {0x0, 0x0, 0x0}, 0},
- {0x16F8F, {0x0, 0x0, 0x0}, 0},
- {0x16F90, {0x0, 0x0, 0x0}, 0},
- {0x16F91, {0x0, 0x0, 0x0}, 0},
- {0x16F92, {0x0, 0x0, 0x0}, 0},
- {0x16FE4, {0x0, 0x0, 0x0}, 0},
- {0x1BC9D, {0x0, 0x0, 0x0}, 0},
- {0x1BC9E, {0x0, 0x0, 0x0}, 0},
- {0x1D15E, {0x1D157, 0x1D165, 0x0}, 2},
- {0x1D15F, {0x1D158, 0x1D165, 0x0}, 2},
- {0x1D160, {0x1D158, 0x1D165, 0x1D16E}, 3},
- {0x1D161, {0x1D158, 0x1D165, 0x1D16F}, 3},
- {0x1D162, {0x1D158, 0x1D165, 0x1D170}, 3},
- {0x1D163, {0x1D158, 0x1D165, 0x1D171}, 3},
- {0x1D164, {0x1D158, 0x1D165, 0x1D172}, 3},
- {0x1D167, {0x0, 0x0, 0x0}, 0},
- {0x1D168, {0x0, 0x0, 0x0}, 0},
- {0x1D169, {0x0, 0x0, 0x0}, 0},
- {0x1D17B, {0x0, 0x0, 0x0}, 0},
- {0x1D17C, {0x0, 0x0, 0x0}, 0},
- {0x1D17D, {0x0, 0x0, 0x0}, 0},
- {0x1D17E, {0x0, 0x0, 0x0}, 0},
- {0x1D17F, {0x0, 0x0, 0x0}, 0},
- {0x1D180, {0x0, 0x0, 0x0}, 0},
- {0x1D181, {0x0, 0x0, 0x0}, 0},
- {0x1D182, {0x0, 0x0, 0x0}, 0},
- {0x1D185, {0x0, 0x0, 0x0}, 0},
- {0x1D186, {0x0, 0x0, 0x0}, 0},
- {0x1D187, {0x0, 0x0, 0x0}, 0},
- {0x1D188, {0x0, 0x0, 0x0}, 0},
- {0x1D189, {0x0, 0x0, 0x0}, 0},
- {0x1D18A, {0x0, 0x0, 0x0}, 0},
- {0x1D18B, {0x0, 0x0, 0x0}, 0},
- {0x1D1AA, {0x0, 0x0, 0x0}, 0},
- {0x1D1AB, {0x0, 0x0, 0x0}, 0},
- {0x1D1AC, {0x0, 0x0, 0x0}, 0},
- {0x1D1AD, {0x0, 0x0, 0x0}, 0},
- {0x1D1BB, {0x1D1B9, 0x1D165, 0x0}, 2},
- {0x1D1BC, {0x1D1BA, 0x1D165, 0x0}, 2},
- {0x1D1BD, {0x1D1B9, 0x1D165, 0x1D16E}, 3},
- {0x1D1BE, {0x1D1BA, 0x1D165, 0x1D16E}, 3},
- {0x1D1BF, {0x1D1B9, 0x1D165, 0x1D16F}, 3},
- {0x1D1C0, {0x1D1BA, 0x1D165, 0x1D16F}, 3},
- {0x1D242, {0x0, 0x0, 0x0}, 0},
- {0x1D243, {0x0, 0x0, 0x0}, 0},
- {0x1D244, {0x0, 0x0, 0x0}, 0},
- {0x1DA00, {0x0, 0x0, 0x0}, 0},
- {0x1DA01, {0x0, 0x0, 0x0}, 0},
- {0x1DA02, {0x0, 0x0, 0x0}, 0},
- {0x1DA03, {0x0, 0x0, 0x0}, 0},
- {0x1DA04, {0x0, 0x0, 0x0}, 0},
- {0x1DA05, {0x0, 0x0, 0x0}, 0},
- {0x1DA06, {0x0, 0x0, 0x0}, 0},
- {0x1DA07, {0x0, 0x0, 0x0}, 0},
- {0x1DA08, {0x0, 0x0, 0x0}, 0},
- {0x1DA09, {0x0, 0x0, 0x0}, 0},
- {0x1DA0A, {0x0, 0x0, 0x0}, 0},
- {0x1DA0B, {0x0, 0x0, 0x0}, 0},
- {0x1DA0C, {0x0, 0x0, 0x0}, 0},
- {0x1DA0D, {0x0, 0x0, 0x0}, 0},
- {0x1DA0E, {0x0, 0x0, 0x0}, 0},
- {0x1DA0F, {0x0, 0x0, 0x0}, 0},
- {0x1DA10, {0x0, 0x0, 0x0}, 0},
- {0x1DA11, {0x0, 0x0, 0x0}, 0},
- {0x1DA12, {0x0, 0x0, 0x0}, 0},
- {0x1DA13, {0x0, 0x0, 0x0}, 0},
- {0x1DA14, {0x0, 0x0, 0x0}, 0},
- {0x1DA15, {0x0, 0x0, 0x0}, 0},
- {0x1DA16, {0x0, 0x0, 0x0}, 0},
- {0x1DA17, {0x0, 0x0, 0x0}, 0},
- {0x1DA18, {0x0, 0x0, 0x0}, 0},
- {0x1DA19, {0x0, 0x0, 0x0}, 0},
- {0x1DA1A, {0x0, 0x0, 0x0}, 0},
- {0x1DA1B, {0x0, 0x0, 0x0}, 0},
- {0x1DA1C, {0x0, 0x0, 0x0}, 0},
- {0x1DA1D, {0x0, 0x0, 0x0}, 0},
- {0x1DA1E, {0x0, 0x0, 0x0}, 0},
- {0x1DA1F, {0x0, 0x0, 0x0}, 0},
- {0x1DA20, {0x0, 0x0, 0x0}, 0},
- {0x1DA21, {0x0, 0x0, 0x0}, 0},
- {0x1DA22, {0x0, 0x0, 0x0}, 0},
- {0x1DA23, {0x0, 0x0, 0x0}, 0},
- {0x1DA24, {0x0, 0x0, 0x0}, 0},
- {0x1DA25, {0x0, 0x0, 0x0}, 0},
- {0x1DA26, {0x0, 0x0, 0x0}, 0},
- {0x1DA27, {0x0, 0x0, 0x0}, 0},
- {0x1DA28, {0x0, 0x0, 0x0}, 0},
- {0x1DA29, {0x0, 0x0, 0x0}, 0},
- {0x1DA2A, {0x0, 0x0, 0x0}, 0},
- {0x1DA2B, {0x0, 0x0, 0x0}, 0},
- {0x1DA2C, {0x0, 0x0, 0x0}, 0},
- {0x1DA2D, {0x0, 0x0, 0x0}, 0},
- {0x1DA2E, {0x0, 0x0, 0x0}, 0},
- {0x1DA2F, {0x0, 0x0, 0x0}, 0},
- {0x1DA30, {0x0, 0x0, 0x0}, 0},
- {0x1DA31, {0x0, 0x0, 0x0}, 0},
- {0x1DA32, {0x0, 0x0, 0x0}, 0},
- {0x1DA33, {0x0, 0x0, 0x0}, 0},
- {0x1DA34, {0x0, 0x0, 0x0}, 0},
- {0x1DA35, {0x0, 0x0, 0x0}, 0},
- {0x1DA36, {0x0, 0x0, 0x0}, 0},
- {0x1DA3B, {0x0, 0x0, 0x0}, 0},
- {0x1DA3C, {0x0, 0x0, 0x0}, 0},
- {0x1DA3D, {0x0, 0x0, 0x0}, 0},
- {0x1DA3E, {0x0, 0x0, 0x0}, 0},
- {0x1DA3F, {0x0, 0x0, 0x0}, 0},
- {0x1DA40, {0x0, 0x0, 0x0}, 0},
- {0x1DA41, {0x0, 0x0, 0x0}, 0},
- {0x1DA42, {0x0, 0x0, 0x0}, 0},
- {0x1DA43, {0x0, 0x0, 0x0}, 0},
- {0x1DA44, {0x0, 0x0, 0x0}, 0},
- {0x1DA45, {0x0, 0x0, 0x0}, 0},
- {0x1DA46, {0x0, 0x0, 0x0}, 0},
- {0x1DA47, {0x0, 0x0, 0x0}, 0},
- {0x1DA48, {0x0, 0x0, 0x0}, 0},
- {0x1DA49, {0x0, 0x0, 0x0}, 0},
- {0x1DA4A, {0x0, 0x0, 0x0}, 0},
- {0x1DA4B, {0x0, 0x0, 0x0}, 0},
- {0x1DA4C, {0x0, 0x0, 0x0}, 0},
- {0x1DA4D, {0x0, 0x0, 0x0}, 0},
- {0x1DA4E, {0x0, 0x0, 0x0}, 0},
- {0x1DA4F, {0x0, 0x0, 0x0}, 0},
- {0x1DA50, {0x0, 0x0, 0x0}, 0},
- {0x1DA51, {0x0, 0x0, 0x0}, 0},
- {0x1DA52, {0x0, 0x0, 0x0}, 0},
- {0x1DA53, {0x0, 0x0, 0x0}, 0},
- {0x1DA54, {0x0, 0x0, 0x0}, 0},
- {0x1DA55, {0x0, 0x0, 0x0}, 0},
- {0x1DA56, {0x0, 0x0, 0x0}, 0},
- {0x1DA57, {0x0, 0x0, 0x0}, 0},
- {0x1DA58, {0x0, 0x0, 0x0}, 0},
- {0x1DA59, {0x0, 0x0, 0x0}, 0},
- {0x1DA5A, {0x0, 0x0, 0x0}, 0},
- {0x1DA5B, {0x0, 0x0, 0x0}, 0},
- {0x1DA5C, {0x0, 0x0, 0x0}, 0},
- {0x1DA5D, {0x0, 0x0, 0x0}, 0},
- {0x1DA5E, {0x0, 0x0, 0x0}, 0},
- {0x1DA5F, {0x0, 0x0, 0x0}, 0},
- {0x1DA60, {0x0, 0x0, 0x0}, 0},
- {0x1DA61, {0x0, 0x0, 0x0}, 0},
- {0x1DA62, {0x0, 0x0, 0x0}, 0},
- {0x1DA63, {0x0, 0x0, 0x0}, 0},
- {0x1DA64, {0x0, 0x0, 0x0}, 0},
- {0x1DA65, {0x0, 0x0, 0x0}, 0},
- {0x1DA66, {0x0, 0x0, 0x0}, 0},
- {0x1DA67, {0x0, 0x0, 0x0}, 0},
- {0x1DA68, {0x0, 0x0, 0x0}, 0},
- {0x1DA69, {0x0, 0x0, 0x0}, 0},
- {0x1DA6A, {0x0, 0x0, 0x0}, 0},
- {0x1DA6B, {0x0, 0x0, 0x0}, 0},
- {0x1DA6C, {0x0, 0x0, 0x0}, 0},
- {0x1DA75, {0x0, 0x0, 0x0}, 0},
- {0x1DA84, {0x0, 0x0, 0x0}, 0},
- {0x1DA9B, {0x0, 0x0, 0x0}, 0},
- {0x1DA9C, {0x0, 0x0, 0x0}, 0},
- {0x1DA9D, {0x0, 0x0, 0x0}, 0},
- {0x1DA9E, {0x0, 0x0, 0x0}, 0},
- {0x1DA9F, {0x0, 0x0, 0x0}, 0},
- {0x1DAA1, {0x0, 0x0, 0x0}, 0},
- {0x1DAA2, {0x0, 0x0, 0x0}, 0},
- {0x1DAA3, {0x0, 0x0, 0x0}, 0},
- {0x1DAA4, {0x0, 0x0, 0x0}, 0},
- {0x1DAA5, {0x0, 0x0, 0x0}, 0},
- {0x1DAA6, {0x0, 0x0, 0x0}, 0},
- {0x1DAA7, {0x0, 0x0, 0x0}, 0},
- {0x1DAA8, {0x0, 0x0, 0x0}, 0},
- {0x1DAA9, {0x0, 0x0, 0x0}, 0},
- {0x1DAAA, {0x0, 0x0, 0x0}, 0},
- {0x1DAAB, {0x0, 0x0, 0x0}, 0},
- {0x1DAAC, {0x0, 0x0, 0x0}, 0},
- {0x1DAAD, {0x0, 0x0, 0x0}, 0},
- {0x1DAAE, {0x0, 0x0, 0x0}, 0},
- {0x1DAAF, {0x0, 0x0, 0x0}, 0},
- {0x1E000, {0x0, 0x0, 0x0}, 0},
- {0x1E001, {0x0, 0x0, 0x0}, 0},
- {0x1E002, {0x0, 0x0, 0x0}, 0},
- {0x1E003, {0x0, 0x0, 0x0}, 0},
- {0x1E004, {0x0, 0x0, 0x0}, 0},
- {0x1E005, {0x0, 0x0, 0x0}, 0},
- {0x1E006, {0x0, 0x0, 0x0}, 0},
- {0x1E008, {0x0, 0x0, 0x0}, 0},
- {0x1E009, {0x0, 0x0, 0x0}, 0},
- {0x1E00A, {0x0, 0x0, 0x0}, 0},
- {0x1E00B, {0x0, 0x0, 0x0}, 0},
- {0x1E00C, {0x0, 0x0, 0x0}, 0},
- {0x1E00D, {0x0, 0x0, 0x0}, 0},
- {0x1E00E, {0x0, 0x0, 0x0}, 0},
- {0x1E00F, {0x0, 0x0, 0x0}, 0},
- {0x1E010, {0x0, 0x0, 0x0}, 0},
- {0x1E011, {0x0, 0x0, 0x0}, 0},
- {0x1E012, {0x0, 0x0, 0x0}, 0},
- {0x1E013, {0x0, 0x0, 0x0}, 0},
- {0x1E014, {0x0, 0x0, 0x0}, 0},
- {0x1E015, {0x0, 0x0, 0x0}, 0},
- {0x1E016, {0x0, 0x0, 0x0}, 0},
- {0x1E017, {0x0, 0x0, 0x0}, 0},
- {0x1E018, {0x0, 0x0, 0x0}, 0},
- {0x1E01B, {0x0, 0x0, 0x0}, 0},
- {0x1E01C, {0x0, 0x0, 0x0}, 0},
- {0x1E01D, {0x0, 0x0, 0x0}, 0},
- {0x1E01E, {0x0, 0x0, 0x0}, 0},
- {0x1E01F, {0x0, 0x0, 0x0}, 0},
- {0x1E020, {0x0, 0x0, 0x0}, 0},
- {0x1E021, {0x0, 0x0, 0x0}, 0},
- {0x1E023, {0x0, 0x0, 0x0}, 0},
- {0x1E024, {0x0, 0x0, 0x0}, 0},
- {0x1E026, {0x0, 0x0, 0x0}, 0},
- {0x1E027, {0x0, 0x0, 0x0}, 0},
- {0x1E028, {0x0, 0x0, 0x0}, 0},
- {0x1E029, {0x0, 0x0, 0x0}, 0},
- {0x1E02A, {0x0, 0x0, 0x0}, 0},
- {0x1E130, {0x0, 0x0, 0x0}, 0},
- {0x1E131, {0x0, 0x0, 0x0}, 0},
- {0x1E132, {0x0, 0x0, 0x0}, 0},
- {0x1E133, {0x0, 0x0, 0x0}, 0},
- {0x1E134, {0x0, 0x0, 0x0}, 0},
- {0x1E135, {0x0, 0x0, 0x0}, 0},
- {0x1E136, {0x0, 0x0, 0x0}, 0},
- {0x1E2EC, {0x0, 0x0, 0x0}, 0},
- {0x1E2ED, {0x0, 0x0, 0x0}, 0},
- {0x1E2EE, {0x0, 0x0, 0x0}, 0},
- {0x1E2EF, {0x0, 0x0, 0x0}, 0},
- {0x1E8D0, {0x0, 0x0, 0x0}, 0},
- {0x1E8D1, {0x0, 0x0, 0x0}, 0},
- {0x1E8D2, {0x0, 0x0, 0x0}, 0},
- {0x1E8D3, {0x0, 0x0, 0x0}, 0},
- {0x1E8D4, {0x0, 0x0, 0x0}, 0},
- {0x1E8D5, {0x0, 0x0, 0x0}, 0},
- {0x1E8D6, {0x0, 0x0, 0x0}, 0},
- {0x1E900, {0x1E922, 0x0, 0x0}, 1},
- {0x1E901, {0x1E923, 0x0, 0x0}, 1},
- {0x1E902, {0x1E924, 0x0, 0x0}, 1},
- {0x1E903, {0x1E925, 0x0, 0x0}, 1},
- {0x1E904, {0x1E926, 0x0, 0x0}, 1},
- {0x1E905, {0x1E927, 0x0, 0x0}, 1},
- {0x1E906, {0x1E928, 0x0, 0x0}, 1},
- {0x1E907, {0x1E929, 0x0, 0x0}, 1},
- {0x1E908, {0x1E92A, 0x0, 0x0}, 1},
- {0x1E909, {0x1E92B, 0x0, 0x0}, 1},
- {0x1E90A, {0x1E92C, 0x0, 0x0}, 1},
- {0x1E90B, {0x1E92D, 0x0, 0x0}, 1},
- {0x1E90C, {0x1E92E, 0x0, 0x0}, 1},
- {0x1E90D, {0x1E92F, 0x0, 0x0}, 1},
- {0x1E90E, {0x1E930, 0x0, 0x0}, 1},
- {0x1E90F, {0x1E931, 0x0, 0x0}, 1},
- {0x1E910, {0x1E932, 0x0, 0x0}, 1},
- {0x1E911, {0x1E933, 0x0, 0x0}, 1},
- {0x1E912, {0x1E934, 0x0, 0x0}, 1},
- {0x1E913, {0x1E935, 0x0, 0x0}, 1},
- {0x1E914, {0x1E936, 0x0, 0x0}, 1},
- {0x1E915, {0x1E937, 0x0, 0x0}, 1},
- {0x1E916, {0x1E938, 0x0, 0x0}, 1},
- {0x1E917, {0x1E939, 0x0, 0x0}, 1},
- {0x1E918, {0x1E93A, 0x0, 0x0}, 1},
- {0x1E919, {0x1E93B, 0x0, 0x0}, 1},
- {0x1E91A, {0x1E93C, 0x0, 0x0}, 1},
- {0x1E91B, {0x1E93D, 0x0, 0x0}, 1},
- {0x1E91C, {0x1E93E, 0x0, 0x0}, 1},
- {0x1E91D, {0x1E93F, 0x0, 0x0}, 1},
- {0x1E91E, {0x1E940, 0x0, 0x0}, 1},
- {0x1E91F, {0x1E941, 0x0, 0x0}, 1},
- {0x1E920, {0x1E942, 0x0, 0x0}, 1},
- {0x1E921, {0x1E943, 0x0, 0x0}, 1},
- {0x1E944, {0x0, 0x0, 0x0}, 0},
- {0x1E945, {0x0, 0x0, 0x0}, 0},
- {0x1E946, {0x0, 0x0, 0x0}, 0},
- {0x1E947, {0x0, 0x0, 0x0}, 0},
- {0x1E948, {0x0, 0x0, 0x0}, 0},
- {0x1E949, {0x0, 0x0, 0x0}, 0},
- {0x1E94A, {0x0, 0x0, 0x0}, 0},
- {0x2F800, {0x4E3D, 0x0, 0x0}, 1},
- {0x2F801, {0x4E38, 0x0, 0x0}, 1},
- {0x2F802, {0x4E41, 0x0, 0x0}, 1},
- {0x2F803, {0x20122, 0x0, 0x0}, 1},
- {0x2F804, {0x4F60, 0x0, 0x0}, 1},
- {0x2F805, {0x4FAE, 0x0, 0x0}, 1},
- {0x2F806, {0x4FBB, 0x0, 0x0}, 1},
- {0x2F807, {0x5002, 0x0, 0x0}, 1},
- {0x2F808, {0x507A, 0x0, 0x0}, 1},
- {0x2F809, {0x5099, 0x0, 0x0}, 1},
- {0x2F80A, {0x50E7, 0x0, 0x0}, 1},
- {0x2F80B, {0x50CF, 0x0, 0x0}, 1},
- {0x2F80C, {0x349E, 0x0, 0x0}, 1},
- {0x2F80D, {0x2063A, 0x0, 0x0}, 1},
- {0x2F80E, {0x514D, 0x0, 0x0}, 1},
- {0x2F80F, {0x5154, 0x0, 0x0}, 1},
- {0x2F810, {0x5164, 0x0, 0x0}, 1},
- {0x2F811, {0x5177, 0x0, 0x0}, 1},
- {0x2F812, {0x2051C, 0x0, 0x0}, 1},
- {0x2F813, {0x34B9, 0x0, 0x0}, 1},
- {0x2F814, {0x5167, 0x0, 0x0}, 1},
- {0x2F815, {0x518D, 0x0, 0x0}, 1},
- {0x2F816, {0x2054B, 0x0, 0x0}, 1},
- {0x2F817, {0x5197, 0x0, 0x0}, 1},
- {0x2F818, {0x51A4, 0x0, 0x0}, 1},
- {0x2F819, {0x4ECC, 0x0, 0x0}, 1},
- {0x2F81A, {0x51AC, 0x0, 0x0}, 1},
- {0x2F81B, {0x51B5, 0x0, 0x0}, 1},
- {0x2F81C, {0x291DF, 0x0, 0x0}, 1},
- {0x2F81D, {0x51F5, 0x0, 0x0}, 1},
- {0x2F81E, {0x5203, 0x0, 0x0}, 1},
- {0x2F81F, {0x34DF, 0x0, 0x0}, 1},
- {0x2F820, {0x523B, 0x0, 0x0}, 1},
- {0x2F821, {0x5246, 0x0, 0x0}, 1},
- {0x2F822, {0x5272, 0x0, 0x0}, 1},
- {0x2F823, {0x5277, 0x0, 0x0}, 1},
- {0x2F824, {0x3515, 0x0, 0x0}, 1},
- {0x2F825, {0x52C7, 0x0, 0x0}, 1},
- {0x2F826, {0x52C9, 0x0, 0x0}, 1},
- {0x2F827, {0x52E4, 0x0, 0x0}, 1},
- {0x2F828, {0x52FA, 0x0, 0x0}, 1},
- {0x2F829, {0x5305, 0x0, 0x0}, 1},
- {0x2F82A, {0x5306, 0x0, 0x0}, 1},
- {0x2F82B, {0x5317, 0x0, 0x0}, 1},
- {0x2F82C, {0x5349, 0x0, 0x0}, 1},
- {0x2F82D, {0x5351, 0x0, 0x0}, 1},
- {0x2F82E, {0x535A, 0x0, 0x0}, 1},
- {0x2F82F, {0x5373, 0x0, 0x0}, 1},
- {0x2F830, {0x537D, 0x0, 0x0}, 1},
- {0x2F831, {0x537F, 0x0, 0x0}, 1},
- {0x2F832, {0x537F, 0x0, 0x0}, 1},
- {0x2F833, {0x537F, 0x0, 0x0}, 1},
- {0x2F834, {0x20A2C, 0x0, 0x0}, 1},
- {0x2F835, {0x7070, 0x0, 0x0}, 1},
- {0x2F836, {0x53CA, 0x0, 0x0}, 1},
- {0x2F837, {0x53DF, 0x0, 0x0}, 1},
- {0x2F838, {0x20B63, 0x0, 0x0}, 1},
- {0x2F839, {0x53EB, 0x0, 0x0}, 1},
- {0x2F83A, {0x53F1, 0x0, 0x0}, 1},
- {0x2F83B, {0x5406, 0x0, 0x0}, 1},
- {0x2F83C, {0x549E, 0x0, 0x0}, 1},
- {0x2F83D, {0x5438, 0x0, 0x0}, 1},
- {0x2F83E, {0x5448, 0x0, 0x0}, 1},
- {0x2F83F, {0x5468, 0x0, 0x0}, 1},
- {0x2F840, {0x54A2, 0x0, 0x0}, 1},
- {0x2F841, {0x54F6, 0x0, 0x0}, 1},
- {0x2F842, {0x5510, 0x0, 0x0}, 1},
- {0x2F843, {0x5553, 0x0, 0x0}, 1},
- {0x2F844, {0x5563, 0x0, 0x0}, 1},
- {0x2F845, {0x5584, 0x0, 0x0}, 1},
- {0x2F846, {0x5584, 0x0, 0x0}, 1},
- {0x2F847, {0x5599, 0x0, 0x0}, 1},
- {0x2F848, {0x55AB, 0x0, 0x0}, 1},
- {0x2F849, {0x55B3, 0x0, 0x0}, 1},
- {0x2F84A, {0x55C2, 0x0, 0x0}, 1},
- {0x2F84B, {0x5716, 0x0, 0x0}, 1},
- {0x2F84C, {0x5606, 0x0, 0x0}, 1},
- {0x2F84D, {0x5717, 0x0, 0x0}, 1},
- {0x2F84E, {0x5651, 0x0, 0x0}, 1},
- {0x2F84F, {0x5674, 0x0, 0x0}, 1},
- {0x2F850, {0x5207, 0x0, 0x0}, 1},
- {0x2F851, {0x58EE, 0x0, 0x0}, 1},
- {0x2F852, {0x57CE, 0x0, 0x0}, 1},
- {0x2F853, {0x57F4, 0x0, 0x0}, 1},
- {0x2F854, {0x580D, 0x0, 0x0}, 1},
- {0x2F855, {0x578B, 0x0, 0x0}, 1},
- {0x2F856, {0x5832, 0x0, 0x0}, 1},
- {0x2F857, {0x5831, 0x0, 0x0}, 1},
- {0x2F858, {0x58AC, 0x0, 0x0}, 1},
- {0x2F859, {0x214E4, 0x0, 0x0}, 1},
- {0x2F85A, {0x58F2, 0x0, 0x0}, 1},
- {0x2F85B, {0x58F7, 0x0, 0x0}, 1},
- {0x2F85C, {0x5906, 0x0, 0x0}, 1},
- {0x2F85D, {0x591A, 0x0, 0x0}, 1},
- {0x2F85E, {0x5922, 0x0, 0x0}, 1},
- {0x2F85F, {0x5962, 0x0, 0x0}, 1},
- {0x2F860, {0x216A8, 0x0, 0x0}, 1},
- {0x2F861, {0x216EA, 0x0, 0x0}, 1},
- {0x2F862, {0x59EC, 0x0, 0x0}, 1},
- {0x2F863, {0x5A1B, 0x0, 0x0}, 1},
- {0x2F864, {0x5A27, 0x0, 0x0}, 1},
- {0x2F865, {0x59D8, 0x0, 0x0}, 1},
- {0x2F866, {0x5A66, 0x0, 0x0}, 1},
- {0x2F867, {0x36EE, 0x0, 0x0}, 1},
- {0x2F868, {0x36FC, 0x0, 0x0}, 1},
- {0x2F869, {0x5B08, 0x0, 0x0}, 1},
- {0x2F86A, {0x5B3E, 0x0, 0x0}, 1},
- {0x2F86B, {0x5B3E, 0x0, 0x0}, 1},
- {0x2F86C, {0x219C8, 0x0, 0x0}, 1},
- {0x2F86D, {0x5BC3, 0x0, 0x0}, 1},
- {0x2F86E, {0x5BD8, 0x0, 0x0}, 1},
- {0x2F86F, {0x5BE7, 0x0, 0x0}, 1},
- {0x2F870, {0x5BF3, 0x0, 0x0}, 1},
- {0x2F871, {0x21B18, 0x0, 0x0}, 1},
- {0x2F872, {0x5BFF, 0x0, 0x0}, 1},
- {0x2F873, {0x5C06, 0x0, 0x0}, 1},
- {0x2F874, {0x5F53, 0x0, 0x0}, 1},
- {0x2F875, {0x5C22, 0x0, 0x0}, 1},
- {0x2F876, {0x3781, 0x0, 0x0}, 1},
- {0x2F877, {0x5C60, 0x0, 0x0}, 1},
- {0x2F878, {0x5C6E, 0x0, 0x0}, 1},
- {0x2F879, {0x5CC0, 0x0, 0x0}, 1},
- {0x2F87A, {0x5C8D, 0x0, 0x0}, 1},
- {0x2F87B, {0x21DE4, 0x0, 0x0}, 1},
- {0x2F87C, {0x5D43, 0x0, 0x0}, 1},
- {0x2F87D, {0x21DE6, 0x0, 0x0}, 1},
- {0x2F87E, {0x5D6E, 0x0, 0x0}, 1},
- {0x2F87F, {0x5D6B, 0x0, 0x0}, 1},
- {0x2F880, {0x5D7C, 0x0, 0x0}, 1},
- {0x2F881, {0x5DE1, 0x0, 0x0}, 1},
- {0x2F882, {0x5DE2, 0x0, 0x0}, 1},
- {0x2F883, {0x382F, 0x0, 0x0}, 1},
- {0x2F884, {0x5DFD, 0x0, 0x0}, 1},
- {0x2F885, {0x5E28, 0x0, 0x0}, 1},
- {0x2F886, {0x5E3D, 0x0, 0x0}, 1},
- {0x2F887, {0x5E69, 0x0, 0x0}, 1},
- {0x2F888, {0x3862, 0x0, 0x0}, 1},
- {0x2F889, {0x22183, 0x0, 0x0}, 1},
- {0x2F88A, {0x387C, 0x0, 0x0}, 1},
- {0x2F88B, {0x5EB0, 0x0, 0x0}, 1},
- {0x2F88C, {0x5EB3, 0x0, 0x0}, 1},
- {0x2F88D, {0x5EB6, 0x0, 0x0}, 1},
- {0x2F88E, {0x5ECA, 0x0, 0x0}, 1},
- {0x2F88F, {0x2A392, 0x0, 0x0}, 1},
- {0x2F890, {0x5EFE, 0x0, 0x0}, 1},
- {0x2F891, {0x22331, 0x0, 0x0}, 1},
- {0x2F892, {0x22331, 0x0, 0x0}, 1},
- {0x2F893, {0x8201, 0x0, 0x0}, 1},
- {0x2F894, {0x5F22, 0x0, 0x0}, 1},
- {0x2F895, {0x5F22, 0x0, 0x0}, 1},
- {0x2F896, {0x38C7, 0x0, 0x0}, 1},
- {0x2F897, {0x232B8, 0x0, 0x0}, 1},
- {0x2F898, {0x261DA, 0x0, 0x0}, 1},
- {0x2F899, {0x5F62, 0x0, 0x0}, 1},
- {0x2F89A, {0x5F6B, 0x0, 0x0}, 1},
- {0x2F89B, {0x38E3, 0x0, 0x0}, 1},
- {0x2F89C, {0x5F9A, 0x0, 0x0}, 1},
- {0x2F89D, {0x5FCD, 0x0, 0x0}, 1},
- {0x2F89E, {0x5FD7, 0x0, 0x0}, 1},
- {0x2F89F, {0x5FF9, 0x0, 0x0}, 1},
- {0x2F8A0, {0x6081, 0x0, 0x0}, 1},
- {0x2F8A1, {0x393A, 0x0, 0x0}, 1},
- {0x2F8A2, {0x391C, 0x0, 0x0}, 1},
- {0x2F8A3, {0x6094, 0x0, 0x0}, 1},
- {0x2F8A4, {0x226D4, 0x0, 0x0}, 1},
- {0x2F8A5, {0x60C7, 0x0, 0x0}, 1},
- {0x2F8A6, {0x6148, 0x0, 0x0}, 1},
- {0x2F8A7, {0x614C, 0x0, 0x0}, 1},
- {0x2F8A8, {0x614E, 0x0, 0x0}, 1},
- {0x2F8A9, {0x614C, 0x0, 0x0}, 1},
- {0x2F8AA, {0x617A, 0x0, 0x0}, 1},
- {0x2F8AB, {0x618E, 0x0, 0x0}, 1},
- {0x2F8AC, {0x61B2, 0x0, 0x0}, 1},
- {0x2F8AD, {0x61A4, 0x0, 0x0}, 1},
- {0x2F8AE, {0x61AF, 0x0, 0x0}, 1},
- {0x2F8AF, {0x61DE, 0x0, 0x0}, 1},
- {0x2F8B0, {0x61F2, 0x0, 0x0}, 1},
- {0x2F8B1, {0x61F6, 0x0, 0x0}, 1},
- {0x2F8B2, {0x6210, 0x0, 0x0}, 1},
- {0x2F8B3, {0x621B, 0x0, 0x0}, 1},
- {0x2F8B4, {0x625D, 0x0, 0x0}, 1},
- {0x2F8B5, {0x62B1, 0x0, 0x0}, 1},
- {0x2F8B6, {0x62D4, 0x0, 0x0}, 1},
- {0x2F8B7, {0x6350, 0x0, 0x0}, 1},
- {0x2F8B8, {0x22B0C, 0x0, 0x0}, 1},
- {0x2F8B9, {0x633D, 0x0, 0x0}, 1},
- {0x2F8BA, {0x62FC, 0x0, 0x0}, 1},
- {0x2F8BB, {0x6368, 0x0, 0x0}, 1},
- {0x2F8BC, {0x6383, 0x0, 0x0}, 1},
- {0x2F8BD, {0x63E4, 0x0, 0x0}, 1},
- {0x2F8BE, {0x22BF1, 0x0, 0x0}, 1},
- {0x2F8BF, {0x6422, 0x0, 0x0}, 1},
- {0x2F8C0, {0x63C5, 0x0, 0x0}, 1},
- {0x2F8C1, {0x63A9, 0x0, 0x0}, 1},
- {0x2F8C2, {0x3A2E, 0x0, 0x0}, 1},
- {0x2F8C3, {0x6469, 0x0, 0x0}, 1},
- {0x2F8C4, {0x647E, 0x0, 0x0}, 1},
- {0x2F8C5, {0x649D, 0x0, 0x0}, 1},
- {0x2F8C6, {0x6477, 0x0, 0x0}, 1},
- {0x2F8C7, {0x3A6C, 0x0, 0x0}, 1},
- {0x2F8C8, {0x654F, 0x0, 0x0}, 1},
- {0x2F8C9, {0x656C, 0x0, 0x0}, 1},
- {0x2F8CA, {0x2300A, 0x0, 0x0}, 1},
- {0x2F8CB, {0x65E3, 0x0, 0x0}, 1},
- {0x2F8CC, {0x66F8, 0x0, 0x0}, 1},
- {0x2F8CD, {0x6649, 0x0, 0x0}, 1},
- {0x2F8CE, {0x3B19, 0x0, 0x0}, 1},
- {0x2F8CF, {0x6691, 0x0, 0x0}, 1},
- {0x2F8D0, {0x3B08, 0x0, 0x0}, 1},
- {0x2F8D1, {0x3AE4, 0x0, 0x0}, 1},
- {0x2F8D2, {0x5192, 0x0, 0x0}, 1},
- {0x2F8D3, {0x5195, 0x0, 0x0}, 1},
- {0x2F8D4, {0x6700, 0x0, 0x0}, 1},
- {0x2F8D5, {0x669C, 0x0, 0x0}, 1},
- {0x2F8D6, {0x80AD, 0x0, 0x0}, 1},
- {0x2F8D7, {0x43D9, 0x0, 0x0}, 1},
- {0x2F8D8, {0x6717, 0x0, 0x0}, 1},
- {0x2F8D9, {0x671B, 0x0, 0x0}, 1},
- {0x2F8DA, {0x6721, 0x0, 0x0}, 1},
- {0x2F8DB, {0x675E, 0x0, 0x0}, 1},
- {0x2F8DC, {0x6753, 0x0, 0x0}, 1},
- {0x2F8DD, {0x233C3, 0x0, 0x0}, 1},
- {0x2F8DE, {0x3B49, 0x0, 0x0}, 1},
- {0x2F8DF, {0x67FA, 0x0, 0x0}, 1},
- {0x2F8E0, {0x6785, 0x0, 0x0}, 1},
- {0x2F8E1, {0x6852, 0x0, 0x0}, 1},
- {0x2F8E2, {0x6885, 0x0, 0x0}, 1},
- {0x2F8E3, {0x2346D, 0x0, 0x0}, 1},
- {0x2F8E4, {0x688E, 0x0, 0x0}, 1},
- {0x2F8E5, {0x681F, 0x0, 0x0}, 1},
- {0x2F8E6, {0x6914, 0x0, 0x0}, 1},
- {0x2F8E7, {0x3B9D, 0x0, 0x0}, 1},
- {0x2F8E8, {0x6942, 0x0, 0x0}, 1},
- {0x2F8E9, {0x69A3, 0x0, 0x0}, 1},
- {0x2F8EA, {0x69EA, 0x0, 0x0}, 1},
- {0x2F8EB, {0x6AA8, 0x0, 0x0}, 1},
- {0x2F8EC, {0x236A3, 0x0, 0x0}, 1},
- {0x2F8ED, {0x6ADB, 0x0, 0x0}, 1},
- {0x2F8EE, {0x3C18, 0x0, 0x0}, 1},
- {0x2F8EF, {0x6B21, 0x0, 0x0}, 1},
- {0x2F8F0, {0x238A7, 0x0, 0x0}, 1},
- {0x2F8F1, {0x6B54, 0x0, 0x0}, 1},
- {0x2F8F2, {0x3C4E, 0x0, 0x0}, 1},
- {0x2F8F3, {0x6B72, 0x0, 0x0}, 1},
- {0x2F8F4, {0x6B9F, 0x0, 0x0}, 1},
- {0x2F8F5, {0x6BBA, 0x0, 0x0}, 1},
- {0x2F8F6, {0x6BBB, 0x0, 0x0}, 1},
- {0x2F8F7, {0x23A8D, 0x0, 0x0}, 1},
- {0x2F8F8, {0x21D0B, 0x0, 0x0}, 1},
- {0x2F8F9, {0x23AFA, 0x0, 0x0}, 1},
- {0x2F8FA, {0x6C4E, 0x0, 0x0}, 1},
- {0x2F8FB, {0x23CBC, 0x0, 0x0}, 1},
- {0x2F8FC, {0x6CBF, 0x0, 0x0}, 1},
- {0x2F8FD, {0x6CCD, 0x0, 0x0}, 1},
- {0x2F8FE, {0x6C67, 0x0, 0x0}, 1},
- {0x2F8FF, {0x6D16, 0x0, 0x0}, 1},
- {0x2F900, {0x6D3E, 0x0, 0x0}, 1},
- {0x2F901, {0x6D77, 0x0, 0x0}, 1},
- {0x2F902, {0x6D41, 0x0, 0x0}, 1},
- {0x2F903, {0x6D69, 0x0, 0x0}, 1},
- {0x2F904, {0x6D78, 0x0, 0x0}, 1},
- {0x2F905, {0x6D85, 0x0, 0x0}, 1},
- {0x2F906, {0x23D1E, 0x0, 0x0}, 1},
- {0x2F907, {0x6D34, 0x0, 0x0}, 1},
- {0x2F908, {0x6E2F, 0x0, 0x0}, 1},
- {0x2F909, {0x6E6E, 0x0, 0x0}, 1},
- {0x2F90A, {0x3D33, 0x0, 0x0}, 1},
- {0x2F90B, {0x6ECB, 0x0, 0x0}, 1},
- {0x2F90C, {0x6EC7, 0x0, 0x0}, 1},
- {0x2F90D, {0x23ED1, 0x0, 0x0}, 1},
- {0x2F90E, {0x6DF9, 0x0, 0x0}, 1},
- {0x2F90F, {0x6F6E, 0x0, 0x0}, 1},
- {0x2F910, {0x23F5E, 0x0, 0x0}, 1},
- {0x2F911, {0x23F8E, 0x0, 0x0}, 1},
- {0x2F912, {0x6FC6, 0x0, 0x0}, 1},
- {0x2F913, {0x7039, 0x0, 0x0}, 1},
- {0x2F914, {0x701E, 0x0, 0x0}, 1},
- {0x2F915, {0x701B, 0x0, 0x0}, 1},
- {0x2F916, {0x3D96, 0x0, 0x0}, 1},
- {0x2F917, {0x704A, 0x0, 0x0}, 1},
- {0x2F918, {0x707D, 0x0, 0x0}, 1},
- {0x2F919, {0x7077, 0x0, 0x0}, 1},
- {0x2F91A, {0x70AD, 0x0, 0x0}, 1},
- {0x2F91B, {0x20525, 0x0, 0x0}, 1},
- {0x2F91C, {0x7145, 0x0, 0x0}, 1},
- {0x2F91D, {0x24263, 0x0, 0x0}, 1},
- {0x2F91E, {0x719C, 0x0, 0x0}, 1},
- {0x2F91F, {0x243AB, 0x0, 0x0}, 1},
- {0x2F920, {0x7228, 0x0, 0x0}, 1},
- {0x2F921, {0x7235, 0x0, 0x0}, 1},
- {0x2F922, {0x7250, 0x0, 0x0}, 1},
- {0x2F923, {0x24608, 0x0, 0x0}, 1},
- {0x2F924, {0x7280, 0x0, 0x0}, 1},
- {0x2F925, {0x7295, 0x0, 0x0}, 1},
- {0x2F926, {0x24735, 0x0, 0x0}, 1},
- {0x2F927, {0x24814, 0x0, 0x0}, 1},
- {0x2F928, {0x737A, 0x0, 0x0}, 1},
- {0x2F929, {0x738B, 0x0, 0x0}, 1},
- {0x2F92A, {0x3EAC, 0x0, 0x0}, 1},
- {0x2F92B, {0x73A5, 0x0, 0x0}, 1},
- {0x2F92C, {0x3EB8, 0x0, 0x0}, 1},
- {0x2F92D, {0x3EB8, 0x0, 0x0}, 1},
- {0x2F92E, {0x7447, 0x0, 0x0}, 1},
- {0x2F92F, {0x745C, 0x0, 0x0}, 1},
- {0x2F930, {0x7471, 0x0, 0x0}, 1},
- {0x2F931, {0x7485, 0x0, 0x0}, 1},
- {0x2F932, {0x74CA, 0x0, 0x0}, 1},
- {0x2F933, {0x3F1B, 0x0, 0x0}, 1},
- {0x2F934, {0x7524, 0x0, 0x0}, 1},
- {0x2F935, {0x24C36, 0x0, 0x0}, 1},
- {0x2F936, {0x753E, 0x0, 0x0}, 1},
- {0x2F937, {0x24C92, 0x0, 0x0}, 1},
- {0x2F938, {0x7570, 0x0, 0x0}, 1},
- {0x2F939, {0x2219F, 0x0, 0x0}, 1},
- {0x2F93A, {0x7610, 0x0, 0x0}, 1},
- {0x2F93B, {0x24FA1, 0x0, 0x0}, 1},
- {0x2F93C, {0x24FB8, 0x0, 0x0}, 1},
- {0x2F93D, {0x25044, 0x0, 0x0}, 1},
- {0x2F93E, {0x3FFC, 0x0, 0x0}, 1},
- {0x2F93F, {0x4008, 0x0, 0x0}, 1},
- {0x2F940, {0x76F4, 0x0, 0x0}, 1},
- {0x2F941, {0x250F3, 0x0, 0x0}, 1},
- {0x2F942, {0x250F2, 0x0, 0x0}, 1},
- {0x2F943, {0x25119, 0x0, 0x0}, 1},
- {0x2F944, {0x25133, 0x0, 0x0}, 1},
- {0x2F945, {0x771E, 0x0, 0x0}, 1},
- {0x2F946, {0x771F, 0x0, 0x0}, 1},
- {0x2F947, {0x771F, 0x0, 0x0}, 1},
- {0x2F948, {0x774A, 0x0, 0x0}, 1},
- {0x2F949, {0x4039, 0x0, 0x0}, 1},
- {0x2F94A, {0x778B, 0x0, 0x0}, 1},
- {0x2F94B, {0x4046, 0x0, 0x0}, 1},
- {0x2F94C, {0x4096, 0x0, 0x0}, 1},
- {0x2F94D, {0x2541D, 0x0, 0x0}, 1},
- {0x2F94E, {0x784E, 0x0, 0x0}, 1},
- {0x2F94F, {0x788C, 0x0, 0x0}, 1},
- {0x2F950, {0x78CC, 0x0, 0x0}, 1},
- {0x2F951, {0x40E3, 0x0, 0x0}, 1},
- {0x2F952, {0x25626, 0x0, 0x0}, 1},
- {0x2F953, {0x7956, 0x0, 0x0}, 1},
- {0x2F954, {0x2569A, 0x0, 0x0}, 1},
- {0x2F955, {0x256C5, 0x0, 0x0}, 1},
- {0x2F956, {0x798F, 0x0, 0x0}, 1},
- {0x2F957, {0x79EB, 0x0, 0x0}, 1},
- {0x2F958, {0x412F, 0x0, 0x0}, 1},
- {0x2F959, {0x7A40, 0x0, 0x0}, 1},
- {0x2F95A, {0x7A4A, 0x0, 0x0}, 1},
- {0x2F95B, {0x7A4F, 0x0, 0x0}, 1},
- {0x2F95C, {0x2597C, 0x0, 0x0}, 1},
- {0x2F95D, {0x25AA7, 0x0, 0x0}, 1},
- {0x2F95E, {0x25AA7, 0x0, 0x0}, 1},
- {0x2F95F, {0x7AEE, 0x0, 0x0}, 1},
- {0x2F960, {0x4202, 0x0, 0x0}, 1},
- {0x2F961, {0x25BAB, 0x0, 0x0}, 1},
- {0x2F962, {0x7BC6, 0x0, 0x0}, 1},
- {0x2F963, {0x7BC9, 0x0, 0x0}, 1},
- {0x2F964, {0x4227, 0x0, 0x0}, 1},
- {0x2F965, {0x25C80, 0x0, 0x0}, 1},
- {0x2F966, {0x7CD2, 0x0, 0x0}, 1},
- {0x2F967, {0x42A0, 0x0, 0x0}, 1},
- {0x2F968, {0x7CE8, 0x0, 0x0}, 1},
- {0x2F969, {0x7CE3, 0x0, 0x0}, 1},
- {0x2F96A, {0x7D00, 0x0, 0x0}, 1},
- {0x2F96B, {0x25F86, 0x0, 0x0}, 1},
- {0x2F96C, {0x7D63, 0x0, 0x0}, 1},
- {0x2F96D, {0x4301, 0x0, 0x0}, 1},
- {0x2F96E, {0x7DC7, 0x0, 0x0}, 1},
- {0x2F96F, {0x7E02, 0x0, 0x0}, 1},
- {0x2F970, {0x7E45, 0x0, 0x0}, 1},
- {0x2F971, {0x4334, 0x0, 0x0}, 1},
- {0x2F972, {0x26228, 0x0, 0x0}, 1},
- {0x2F973, {0x26247, 0x0, 0x0}, 1},
- {0x2F974, {0x4359, 0x0, 0x0}, 1},
- {0x2F975, {0x262D9, 0x0, 0x0}, 1},
- {0x2F976, {0x7F7A, 0x0, 0x0}, 1},
- {0x2F977, {0x2633E, 0x0, 0x0}, 1},
- {0x2F978, {0x7F95, 0x0, 0x0}, 1},
- {0x2F979, {0x7FFA, 0x0, 0x0}, 1},
- {0x2F97A, {0x8005, 0x0, 0x0}, 1},
- {0x2F97B, {0x264DA, 0x0, 0x0}, 1},
- {0x2F97C, {0x26523, 0x0, 0x0}, 1},
- {0x2F97D, {0x8060, 0x0, 0x0}, 1},
- {0x2F97E, {0x265A8, 0x0, 0x0}, 1},
- {0x2F97F, {0x8070, 0x0, 0x0}, 1},
- {0x2F980, {0x2335F, 0x0, 0x0}, 1},
- {0x2F981, {0x43D5, 0x0, 0x0}, 1},
- {0x2F982, {0x80B2, 0x0, 0x0}, 1},
- {0x2F983, {0x8103, 0x0, 0x0}, 1},
- {0x2F984, {0x440B, 0x0, 0x0}, 1},
- {0x2F985, {0x813E, 0x0, 0x0}, 1},
- {0x2F986, {0x5AB5, 0x0, 0x0}, 1},
- {0x2F987, {0x267A7, 0x0, 0x0}, 1},
- {0x2F988, {0x267B5, 0x0, 0x0}, 1},
- {0x2F989, {0x23393, 0x0, 0x0}, 1},
- {0x2F98A, {0x2339C, 0x0, 0x0}, 1},
- {0x2F98B, {0x8201, 0x0, 0x0}, 1},
- {0x2F98C, {0x8204, 0x0, 0x0}, 1},
- {0x2F98D, {0x8F9E, 0x0, 0x0}, 1},
- {0x2F98E, {0x446B, 0x0, 0x0}, 1},
- {0x2F98F, {0x8291, 0x0, 0x0}, 1},
- {0x2F990, {0x828B, 0x0, 0x0}, 1},
- {0x2F991, {0x829D, 0x0, 0x0}, 1},
- {0x2F992, {0x52B3, 0x0, 0x0}, 1},
- {0x2F993, {0x82B1, 0x0, 0x0}, 1},
- {0x2F994, {0x82B3, 0x0, 0x0}, 1},
- {0x2F995, {0x82BD, 0x0, 0x0}, 1},
- {0x2F996, {0x82E6, 0x0, 0x0}, 1},
- {0x2F997, {0x26B3C, 0x0, 0x0}, 1},
- {0x2F998, {0x82E5, 0x0, 0x0}, 1},
- {0x2F999, {0x831D, 0x0, 0x0}, 1},
- {0x2F99A, {0x8363, 0x0, 0x0}, 1},
- {0x2F99B, {0x83AD, 0x0, 0x0}, 1},
- {0x2F99C, {0x8323, 0x0, 0x0}, 1},
- {0x2F99D, {0x83BD, 0x0, 0x0}, 1},
- {0x2F99E, {0x83E7, 0x0, 0x0}, 1},
- {0x2F99F, {0x8457, 0x0, 0x0}, 1},
- {0x2F9A0, {0x8353, 0x0, 0x0}, 1},
- {0x2F9A1, {0x83CA, 0x0, 0x0}, 1},
- {0x2F9A2, {0x83CC, 0x0, 0x0}, 1},
- {0x2F9A3, {0x83DC, 0x0, 0x0}, 1},
- {0x2F9A4, {0x26C36, 0x0, 0x0}, 1},
- {0x2F9A5, {0x26D6B, 0x0, 0x0}, 1},
- {0x2F9A6, {0x26CD5, 0x0, 0x0}, 1},
- {0x2F9A7, {0x452B, 0x0, 0x0}, 1},
- {0x2F9A8, {0x84F1, 0x0, 0x0}, 1},
- {0x2F9A9, {0x84F3, 0x0, 0x0}, 1},
- {0x2F9AA, {0x8516, 0x0, 0x0}, 1},
- {0x2F9AB, {0x273CA, 0x0, 0x0}, 1},
- {0x2F9AC, {0x8564, 0x0, 0x0}, 1},
- {0x2F9AD, {0x26F2C, 0x0, 0x0}, 1},
- {0x2F9AE, {0x455D, 0x0, 0x0}, 1},
- {0x2F9AF, {0x4561, 0x0, 0x0}, 1},
- {0x2F9B0, {0x26FB1, 0x0, 0x0}, 1},
- {0x2F9B1, {0x270D2, 0x0, 0x0}, 1},
- {0x2F9B2, {0x456B, 0x0, 0x0}, 1},
- {0x2F9B3, {0x8650, 0x0, 0x0}, 1},
- {0x2F9B4, {0x865C, 0x0, 0x0}, 1},
- {0x2F9B5, {0x8667, 0x0, 0x0}, 1},
- {0x2F9B6, {0x8669, 0x0, 0x0}, 1},
- {0x2F9B7, {0x86A9, 0x0, 0x0}, 1},
- {0x2F9B8, {0x8688, 0x0, 0x0}, 1},
- {0x2F9B9, {0x870E, 0x0, 0x0}, 1},
- {0x2F9BA, {0x86E2, 0x0, 0x0}, 1},
- {0x2F9BB, {0x8779, 0x0, 0x0}, 1},
- {0x2F9BC, {0x8728, 0x0, 0x0}, 1},
- {0x2F9BD, {0x876B, 0x0, 0x0}, 1},
- {0x2F9BE, {0x8786, 0x0, 0x0}, 1},
- {0x2F9BF, {0x45D7, 0x0, 0x0}, 1},
- {0x2F9C0, {0x87E1, 0x0, 0x0}, 1},
- {0x2F9C1, {0x8801, 0x0, 0x0}, 1},
- {0x2F9C2, {0x45F9, 0x0, 0x0}, 1},
- {0x2F9C3, {0x8860, 0x0, 0x0}, 1},
- {0x2F9C4, {0x8863, 0x0, 0x0}, 1},
- {0x2F9C5, {0x27667, 0x0, 0x0}, 1},
- {0x2F9C6, {0x88D7, 0x0, 0x0}, 1},
- {0x2F9C7, {0x88DE, 0x0, 0x0}, 1},
- {0x2F9C8, {0x4635, 0x0, 0x0}, 1},
- {0x2F9C9, {0x88FA, 0x0, 0x0}, 1},
- {0x2F9CA, {0x34BB, 0x0, 0x0}, 1},
- {0x2F9CB, {0x278AE, 0x0, 0x0}, 1},
- {0x2F9CC, {0x27966, 0x0, 0x0}, 1},
- {0x2F9CD, {0x46BE, 0x0, 0x0}, 1},
- {0x2F9CE, {0x46C7, 0x0, 0x0}, 1},
- {0x2F9CF, {0x8AA0, 0x0, 0x0}, 1},
- {0x2F9D0, {0x8AED, 0x0, 0x0}, 1},
- {0x2F9D1, {0x8B8A, 0x0, 0x0}, 1},
- {0x2F9D2, {0x8C55, 0x0, 0x0}, 1},
- {0x2F9D3, {0x27CA8, 0x0, 0x0}, 1},
- {0x2F9D4, {0x8CAB, 0x0, 0x0}, 1},
- {0x2F9D5, {0x8CC1, 0x0, 0x0}, 1},
- {0x2F9D6, {0x8D1B, 0x0, 0x0}, 1},
- {0x2F9D7, {0x8D77, 0x0, 0x0}, 1},
- {0x2F9D8, {0x27F2F, 0x0, 0x0}, 1},
- {0x2F9D9, {0x20804, 0x0, 0x0}, 1},
- {0x2F9DA, {0x8DCB, 0x0, 0x0}, 1},
- {0x2F9DB, {0x8DBC, 0x0, 0x0}, 1},
- {0x2F9DC, {0x8DF0, 0x0, 0x0}, 1},
- {0x2F9DD, {0x208DE, 0x0, 0x0}, 1},
- {0x2F9DE, {0x8ED4, 0x0, 0x0}, 1},
- {0x2F9DF, {0x8F38, 0x0, 0x0}, 1},
- {0x2F9E0, {0x285D2, 0x0, 0x0}, 1},
- {0x2F9E1, {0x285ED, 0x0, 0x0}, 1},
- {0x2F9E2, {0x9094, 0x0, 0x0}, 1},
- {0x2F9E3, {0x90F1, 0x0, 0x0}, 1},
- {0x2F9E4, {0x9111, 0x0, 0x0}, 1},
- {0x2F9E5, {0x2872E, 0x0, 0x0}, 1},
- {0x2F9E6, {0x911B, 0x0, 0x0}, 1},
- {0x2F9E7, {0x9238, 0x0, 0x0}, 1},
- {0x2F9E8, {0x92D7, 0x0, 0x0}, 1},
- {0x2F9E9, {0x92D8, 0x0, 0x0}, 1},
- {0x2F9EA, {0x927C, 0x0, 0x0}, 1},
- {0x2F9EB, {0x93F9, 0x0, 0x0}, 1},
- {0x2F9EC, {0x9415, 0x0, 0x0}, 1},
- {0x2F9ED, {0x28BFA, 0x0, 0x0}, 1},
- {0x2F9EE, {0x958B, 0x0, 0x0}, 1},
- {0x2F9EF, {0x4995, 0x0, 0x0}, 1},
- {0x2F9F0, {0x95B7, 0x0, 0x0}, 1},
- {0x2F9F1, {0x28D77, 0x0, 0x0}, 1},
- {0x2F9F2, {0x49E6, 0x0, 0x0}, 1},
- {0x2F9F3, {0x96C3, 0x0, 0x0}, 1},
- {0x2F9F4, {0x5DB2, 0x0, 0x0}, 1},
- {0x2F9F5, {0x9723, 0x0, 0x0}, 1},
- {0x2F9F6, {0x29145, 0x0, 0x0}, 1},
- {0x2F9F7, {0x2921A, 0x0, 0x0}, 1},
- {0x2F9F8, {0x4A6E, 0x0, 0x0}, 1},
- {0x2F9F9, {0x4A76, 0x0, 0x0}, 1},
- {0x2F9FA, {0x97E0, 0x0, 0x0}, 1},
- {0x2F9FB, {0x2940A, 0x0, 0x0}, 1},
- {0x2F9FC, {0x4AB2, 0x0, 0x0}, 1},
- {0x2F9FD, {0x29496, 0x0, 0x0}, 1},
- {0x2F9FE, {0x980B, 0x0, 0x0}, 1},
- {0x2F9FF, {0x980B, 0x0, 0x0}, 1},
- {0x2FA00, {0x9829, 0x0, 0x0}, 1},
- {0x2FA01, {0x295B6, 0x0, 0x0}, 1},
- {0x2FA02, {0x98E2, 0x0, 0x0}, 1},
- {0x2FA03, {0x4B33, 0x0, 0x0}, 1},
- {0x2FA04, {0x9929, 0x0, 0x0}, 1},
- {0x2FA05, {0x99A7, 0x0, 0x0}, 1},
- {0x2FA06, {0x99C2, 0x0, 0x0}, 1},
- {0x2FA07, {0x99FE, 0x0, 0x0}, 1},
- {0x2FA08, {0x4BCE, 0x0, 0x0}, 1},
- {0x2FA09, {0x29B30, 0x0, 0x0}, 1},
- {0x2FA0A, {0x9B12, 0x0, 0x0}, 1},
- {0x2FA0B, {0x9C40, 0x0, 0x0}, 1},
- {0x2FA0C, {0x9CFD, 0x0, 0x0}, 1},
- {0x2FA0D, {0x4CCE, 0x0, 0x0}, 1},
- {0x2FA0E, {0x4CED, 0x0, 0x0}, 1},
- {0x2FA0F, {0x9D67, 0x0, 0x0}, 1},
- {0x2FA10, {0x2A0CE, 0x0, 0x0}, 1},
- {0x2FA11, {0x4CF8, 0x0, 0x0}, 1},
- {0x2FA12, {0x2A105, 0x0, 0x0}, 1},
- {0x2FA13, {0x2A20E, 0x0, 0x0}, 1},
- {0x2FA14, {0x2A291, 0x0, 0x0}, 1},
- {0x2FA15, {0x9EBB, 0x0, 0x0}, 1},
- {0x2FA16, {0x4D56, 0x0, 0x0}, 1},
- {0x2FA17, {0x9EF9, 0x0, 0x0}, 1},
- {0x2FA18, {0x9EFE, 0x0, 0x0}, 1},
- {0x2FA19, {0x9F05, 0x0, 0x0}, 1},
- {0x2FA1A, {0x9F0F, 0x0, 0x0}, 1},
- {0x2FA1B, {0x9F16, 0x0, 0x0}, 1},
- {0x2FA1C, {0x9F3B, 0x0, 0x0}, 1},
- {0x2FA1D, {0x2A600, 0x0, 0x0}, 1},
- {0xE0100, {0x0, 0x0, 0x0}, 0},
- {0xE0101, {0x0, 0x0, 0x0}, 0},
- {0xE0102, {0x0, 0x0, 0x0}, 0},
- {0xE0103, {0x0, 0x0, 0x0}, 0},
- {0xE0104, {0x0, 0x0, 0x0}, 0},
- {0xE0105, {0x0, 0x0, 0x0}, 0},
- {0xE0106, {0x0, 0x0, 0x0}, 0},
- {0xE0107, {0x0, 0x0, 0x0}, 0},
- {0xE0108, {0x0, 0x0, 0x0}, 0},
- {0xE0109, {0x0, 0x0, 0x0}, 0},
- {0xE010A, {0x0, 0x0, 0x0}, 0},
- {0xE010B, {0x0, 0x0, 0x0}, 0},
- {0xE010C, {0x0, 0x0, 0x0}, 0},
- {0xE010D, {0x0, 0x0, 0x0}, 0},
- {0xE010E, {0x0, 0x0, 0x0}, 0},
- {0xE010F, {0x0, 0x0, 0x0}, 0},
- {0xE0110, {0x0, 0x0, 0x0}, 0},
- {0xE0111, {0x0, 0x0, 0x0}, 0},
- {0xE0112, {0x0, 0x0, 0x0}, 0},
- {0xE0113, {0x0, 0x0, 0x0}, 0},
- {0xE0114, {0x0, 0x0, 0x0}, 0},
- {0xE0115, {0x0, 0x0, 0x0}, 0},
- {0xE0116, {0x0, 0x0, 0x0}, 0},
- {0xE0117, {0x0, 0x0, 0x0}, 0},
- {0xE0118, {0x0, 0x0, 0x0}, 0},
- {0xE0119, {0x0, 0x0, 0x0}, 0},
- {0xE011A, {0x0, 0x0, 0x0}, 0},
- {0xE011B, {0x0, 0x0, 0x0}, 0},
- {0xE011C, {0x0, 0x0, 0x0}, 0},
- {0xE011D, {0x0, 0x0, 0x0}, 0},
- {0xE011E, {0x0, 0x0, 0x0}, 0},
- {0xE011F, {0x0, 0x0, 0x0}, 0},
- {0xE0120, {0x0, 0x0, 0x0}, 0},
- {0xE0121, {0x0, 0x0, 0x0}, 0},
- {0xE0122, {0x0, 0x0, 0x0}, 0},
- {0xE0123, {0x0, 0x0, 0x0}, 0},
- {0xE0124, {0x0, 0x0, 0x0}, 0},
- {0xE0125, {0x0, 0x0, 0x0}, 0},
- {0xE0126, {0x0, 0x0, 0x0}, 0},
- {0xE0127, {0x0, 0x0, 0x0}, 0},
- {0xE0128, {0x0, 0x0, 0x0}, 0},
- {0xE0129, {0x0, 0x0, 0x0}, 0},
- {0xE012A, {0x0, 0x0, 0x0}, 0},
- {0xE012B, {0x0, 0x0, 0x0}, 0},
- {0xE012C, {0x0, 0x0, 0x0}, 0},
- {0xE012D, {0x0, 0x0, 0x0}, 0},
- {0xE012E, {0x0, 0x0, 0x0}, 0},
- {0xE012F, {0x0, 0x0, 0x0}, 0},
- {0xE0130, {0x0, 0x0, 0x0}, 0},
- {0xE0131, {0x0, 0x0, 0x0}, 0},
- {0xE0132, {0x0, 0x0, 0x0}, 0},
- {0xE0133, {0x0, 0x0, 0x0}, 0},
- {0xE0134, {0x0, 0x0, 0x0}, 0},
- {0xE0135, {0x0, 0x0, 0x0}, 0},
- {0xE0136, {0x0, 0x0, 0x0}, 0},
- {0xE0137, {0x0, 0x0, 0x0}, 0},
- {0xE0138, {0x0, 0x0, 0x0}, 0},
- {0xE0139, {0x0, 0x0, 0x0}, 0},
- {0xE013A, {0x0, 0x0, 0x0}, 0},
- {0xE013B, {0x0, 0x0, 0x0}, 0},
- {0xE013C, {0x0, 0x0, 0x0}, 0},
- {0xE013D, {0x0, 0x0, 0x0}, 0},
- {0xE013E, {0x0, 0x0, 0x0}, 0},
- {0xE013F, {0x0, 0x0, 0x0}, 0},
- {0xE0140, {0x0, 0x0, 0x0}, 0},
- {0xE0141, {0x0, 0x0, 0x0}, 0},
- {0xE0142, {0x0, 0x0, 0x0}, 0},
- {0xE0143, {0x0, 0x0, 0x0}, 0},
- {0xE0144, {0x0, 0x0, 0x0}, 0},
- {0xE0145, {0x0, 0x0, 0x0}, 0},
- {0xE0146, {0x0, 0x0, 0x0}, 0},
- {0xE0147, {0x0, 0x0, 0x0}, 0},
- {0xE0148, {0x0, 0x0, 0x0}, 0},
- {0xE0149, {0x0, 0x0, 0x0}, 0},
- {0xE014A, {0x0, 0x0, 0x0}, 0},
- {0xE014B, {0x0, 0x0, 0x0}, 0},
- {0xE014C, {0x0, 0x0, 0x0}, 0},
- {0xE014D, {0x0, 0x0, 0x0}, 0},
- {0xE014E, {0x0, 0x0, 0x0}, 0},
- {0xE014F, {0x0, 0x0, 0x0}, 0},
- {0xE0150, {0x0, 0x0, 0x0}, 0},
- {0xE0151, {0x0, 0x0, 0x0}, 0},
- {0xE0152, {0x0, 0x0, 0x0}, 0},
- {0xE0153, {0x0, 0x0, 0x0}, 0},
- {0xE0154, {0x0, 0x0, 0x0}, 0},
- {0xE0155, {0x0, 0x0, 0x0}, 0},
- {0xE0156, {0x0, 0x0, 0x0}, 0},
- {0xE0157, {0x0, 0x0, 0x0}, 0},
- {0xE0158, {0x0, 0x0, 0x0}, 0},
- {0xE0159, {0x0, 0x0, 0x0}, 0},
- {0xE015A, {0x0, 0x0, 0x0}, 0},
- {0xE015B, {0x0, 0x0, 0x0}, 0},
- {0xE015C, {0x0, 0x0, 0x0}, 0},
- {0xE015D, {0x0, 0x0, 0x0}, 0},
- {0xE015E, {0x0, 0x0, 0x0}, 0},
- {0xE015F, {0x0, 0x0, 0x0}, 0},
- {0xE0160, {0x0, 0x0, 0x0}, 0},
- {0xE0161, {0x0, 0x0, 0x0}, 0},
- {0xE0162, {0x0, 0x0, 0x0}, 0},
- {0xE0163, {0x0, 0x0, 0x0}, 0},
- {0xE0164, {0x0, 0x0, 0x0}, 0},
- {0xE0165, {0x0, 0x0, 0x0}, 0},
- {0xE0166, {0x0, 0x0, 0x0}, 0},
- {0xE0167, {0x0, 0x0, 0x0}, 0},
- {0xE0168, {0x0, 0x0, 0x0}, 0},
- {0xE0169, {0x0, 0x0, 0x0}, 0},
- {0xE016A, {0x0, 0x0, 0x0}, 0},
- {0xE016B, {0x0, 0x0, 0x0}, 0},
- {0xE016C, {0x0, 0x0, 0x0}, 0},
- {0xE016D, {0x0, 0x0, 0x0}, 0},
- {0xE016E, {0x0, 0x0, 0x0}, 0},
- {0xE016F, {0x0, 0x0, 0x0}, 0},
- {0xE0170, {0x0, 0x0, 0x0}, 0},
- {0xE0171, {0x0, 0x0, 0x0}, 0},
- {0xE0172, {0x0, 0x0, 0x0}, 0},
- {0xE0173, {0x0, 0x0, 0x0}, 0},
- {0xE0174, {0x0, 0x0, 0x0}, 0},
- {0xE0175, {0x0, 0x0, 0x0}, 0},
- {0xE0176, {0x0, 0x0, 0x0}, 0},
- {0xE0177, {0x0, 0x0, 0x0}, 0},
- {0xE0178, {0x0, 0x0, 0x0}, 0},
- {0xE0179, {0x0, 0x0, 0x0}, 0},
- {0xE017A, {0x0, 0x0, 0x0}, 0},
- {0xE017B, {0x0, 0x0, 0x0}, 0},
- {0xE017C, {0x0, 0x0, 0x0}, 0},
- {0xE017D, {0x0, 0x0, 0x0}, 0},
- {0xE017E, {0x0, 0x0, 0x0}, 0},
- {0xE017F, {0x0, 0x0, 0x0}, 0},
- {0xE0180, {0x0, 0x0, 0x0}, 0},
- {0xE0181, {0x0, 0x0, 0x0}, 0},
- {0xE0182, {0x0, 0x0, 0x0}, 0},
- {0xE0183, {0x0, 0x0, 0x0}, 0},
- {0xE0184, {0x0, 0x0, 0x0}, 0},
- {0xE0185, {0x0, 0x0, 0x0}, 0},
- {0xE0186, {0x0, 0x0, 0x0}, 0},
- {0xE0187, {0x0, 0x0, 0x0}, 0},
- {0xE0188, {0x0, 0x0, 0x0}, 0},
- {0xE0189, {0x0, 0x0, 0x0}, 0},
- {0xE018A, {0x0, 0x0, 0x0}, 0},
- {0xE018B, {0x0, 0x0, 0x0}, 0},
- {0xE018C, {0x0, 0x0, 0x0}, 0},
- {0xE018D, {0x0, 0x0, 0x0}, 0},
- {0xE018E, {0x0, 0x0, 0x0}, 0},
- {0xE018F, {0x0, 0x0, 0x0}, 0},
- {0xE0190, {0x0, 0x0, 0x0}, 0},
- {0xE0191, {0x0, 0x0, 0x0}, 0},
- {0xE0192, {0x0, 0x0, 0x0}, 0},
- {0xE0193, {0x0, 0x0, 0x0}, 0},
- {0xE0194, {0x0, 0x0, 0x0}, 0},
- {0xE0195, {0x0, 0x0, 0x0}, 0},
- {0xE0196, {0x0, 0x0, 0x0}, 0},
- {0xE0197, {0x0, 0x0, 0x0}, 0},
- {0xE0198, {0x0, 0x0, 0x0}, 0},
- {0xE0199, {0x0, 0x0, 0x0}, 0},
- {0xE019A, {0x0, 0x0, 0x0}, 0},
- {0xE019B, {0x0, 0x0, 0x0}, 0},
- {0xE019C, {0x0, 0x0, 0x0}, 0},
- {0xE019D, {0x0, 0x0, 0x0}, 0},
- {0xE019E, {0x0, 0x0, 0x0}, 0},
- {0xE019F, {0x0, 0x0, 0x0}, 0},
- {0xE01A0, {0x0, 0x0, 0x0}, 0},
- {0xE01A1, {0x0, 0x0, 0x0}, 0},
- {0xE01A2, {0x0, 0x0, 0x0}, 0},
- {0xE01A3, {0x0, 0x0, 0x0}, 0},
- {0xE01A4, {0x0, 0x0, 0x0}, 0},
- {0xE01A5, {0x0, 0x0, 0x0}, 0},
- {0xE01A6, {0x0, 0x0, 0x0}, 0},
- {0xE01A7, {0x0, 0x0, 0x0}, 0},
- {0xE01A8, {0x0, 0x0, 0x0}, 0},
- {0xE01A9, {0x0, 0x0, 0x0}, 0},
- {0xE01AA, {0x0, 0x0, 0x0}, 0},
- {0xE01AB, {0x0, 0x0, 0x0}, 0},
- {0xE01AC, {0x0, 0x0, 0x0}, 0},
- {0xE01AD, {0x0, 0x0, 0x0}, 0},
- {0xE01AE, {0x0, 0x0, 0x0}, 0},
- {0xE01AF, {0x0, 0x0, 0x0}, 0},
- {0xE01B0, {0x0, 0x0, 0x0}, 0},
- {0xE01B1, {0x0, 0x0, 0x0}, 0},
- {0xE01B2, {0x0, 0x0, 0x0}, 0},
- {0xE01B3, {0x0, 0x0, 0x0}, 0},
- {0xE01B4, {0x0, 0x0, 0x0}, 0},
- {0xE01B5, {0x0, 0x0, 0x0}, 0},
- {0xE01B6, {0x0, 0x0, 0x0}, 0},
- {0xE01B7, {0x0, 0x0, 0x0}, 0},
- {0xE01B8, {0x0, 0x0, 0x0}, 0},
- {0xE01B9, {0x0, 0x0, 0x0}, 0},
- {0xE01BA, {0x0, 0x0, 0x0}, 0},
- {0xE01BB, {0x0, 0x0, 0x0}, 0},
- {0xE01BC, {0x0, 0x0, 0x0}, 0},
- {0xE01BD, {0x0, 0x0, 0x0}, 0},
- {0xE01BE, {0x0, 0x0, 0x0}, 0},
- {0xE01BF, {0x0, 0x0, 0x0}, 0},
- {0xE01C0, {0x0, 0x0, 0x0}, 0},
- {0xE01C1, {0x0, 0x0, 0x0}, 0},
- {0xE01C2, {0x0, 0x0, 0x0}, 0},
- {0xE01C3, {0x0, 0x0, 0x0}, 0},
- {0xE01C4, {0x0, 0x0, 0x0}, 0},
- {0xE01C5, {0x0, 0x0, 0x0}, 0},
- {0xE01C6, {0x0, 0x0, 0x0}, 0},
- {0xE01C7, {0x0, 0x0, 0x0}, 0},
- {0xE01C8, {0x0, 0x0, 0x0}, 0},
- {0xE01C9, {0x0, 0x0, 0x0}, 0},
- {0xE01CA, {0x0, 0x0, 0x0}, 0},
- {0xE01CB, {0x0, 0x0, 0x0}, 0},
- {0xE01CC, {0x0, 0x0, 0x0}, 0},
- {0xE01CD, {0x0, 0x0, 0x0}, 0},
- {0xE01CE, {0x0, 0x0, 0x0}, 0},
- {0xE01CF, {0x0, 0x0, 0x0}, 0},
- {0xE01D0, {0x0, 0x0, 0x0}, 0},
- {0xE01D1, {0x0, 0x0, 0x0}, 0},
- {0xE01D2, {0x0, 0x0, 0x0}, 0},
- {0xE01D3, {0x0, 0x0, 0x0}, 0},
- {0xE01D4, {0x0, 0x0, 0x0}, 0},
- {0xE01D5, {0x0, 0x0, 0x0}, 0},
- {0xE01D6, {0x0, 0x0, 0x0}, 0},
- {0xE01D7, {0x0, 0x0, 0x0}, 0},
- {0xE01D8, {0x0, 0x0, 0x0}, 0},
- {0xE01D9, {0x0, 0x0, 0x0}, 0},
- {0xE01DA, {0x0, 0x0, 0x0}, 0},
- {0xE01DB, {0x0, 0x0, 0x0}, 0},
- {0xE01DC, {0x0, 0x0, 0x0}, 0},
- {0xE01DD, {0x0, 0x0, 0x0}, 0},
- {0xE01DE, {0x0, 0x0, 0x0}, 0},
- {0xE01DF, {0x0, 0x0, 0x0}, 0},
- {0xE01E0, {0x0, 0x0, 0x0}, 0},
- {0xE01E1, {0x0, 0x0, 0x0}, 0},
- {0xE01E2, {0x0, 0x0, 0x0}, 0},
- {0xE01E3, {0x0, 0x0, 0x0}, 0},
- {0xE01E4, {0x0, 0x0, 0x0}, 0},
- {0xE01E5, {0x0, 0x0, 0x0}, 0},
- {0xE01E6, {0x0, 0x0, 0x0}, 0},
- {0xE01E7, {0x0, 0x0, 0x0}, 0},
- {0xE01E8, {0x0, 0x0, 0x0}, 0},
- {0xE01E9, {0x0, 0x0, 0x0}, 0},
- {0xE01EA, {0x0, 0x0, 0x0}, 0},
- {0xE01EB, {0x0, 0x0, 0x0}, 0},
- {0xE01EC, {0x0, 0x0, 0x0}, 0},
- {0xE01ED, {0x0, 0x0, 0x0}, 0},
- {0xE01EE, {0x0, 0x0, 0x0}, 0},
- {0xE01EF, {0x0, 0x0, 0x0}, 0},
-};
-enum { WP_FOLD_N = (int)(sizeof(WP_FOLD) / sizeof(WP_FOLD[0])) };
-
-#endif /* WORDPIECE_UNICODE_H */
diff --git a/src/tokenizer_wordpiece.c b/src/tokenizer_wordpiece.c
deleted file mode 100644
index d8acbbe..0000000
--- a/src/tokenizer_wordpiece.c
+++ /dev/null
@@ -1,678 +0,0 @@
-/*
- * wordpiece_tokenizer.c - BERT-family WordPiece tokenizer.
- *
- * Pipeline (matching transformers' BertTokenizer):
- * BasicTokenizer: clean control/whitespace -> space CJK ideographs ->
- * whitespace split -> (lowercase + strip accents) -> split on punctuation
- * WordPiece: greedy longest-match-first against vocab.txt, continuation
- * pieces marked "##", whole token -> [UNK] when any piece fails to match.
- *
- * Unicode handling is full and exact against transformers, driven by generated
- * tables (wordpiece_unicode.h): punctuation and control by Unicode category,
- * whitespace by the complete Zs set, the CJK split by the BERT ideograph ranges,
- * and the do_lower_case fold by a lowercase+NFD+strip-marks table (with Hangul
- * decomposed by formula). Cased and uncased multilingual encoders (e.g.
- * bert-base-multilingual-cased / -uncased) match Hugging Face exactly, including
- * Greek, Cyrillic, Vietnamese and Korean. The fold is per-codepoint, matching
- * transformers, which lowercases character by character (no context-sensitive
- * Greek final sigma).
- */
-
-#include "tokenizer_wordpiece.h"
-
-#include "tokenizer_unicode.h"
-
-#include <stdint.h>
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-/* ===================== string -> int open-addressing map ===================== */
-
-typedef struct {
- const char *key; /* points into tok->id_to_token; not owned */
- int val;
-} wp_entry_t;
-
-static uint64_t fnv1a_hash(const char *s) {
- uint64_t h = 1469598103934665603ULL;
- for (const unsigned char *p = (const unsigned char *)s; *p; p++) {
- h ^= (uint64_t)*p;
- h *= 1099511628211ULL;
- }
- return h;
-}
-
-static int next_pow2(int x) {
- int p = 1;
- while (p < x)
- p <<= 1;
- return p;
-}
-
-static void map_put(wp_entry_t *map, int cap, const char *key, int val) {
- uint64_t mask = (uint64_t)cap - 1;
- int pos = (int)(fnv1a_hash(key) & mask);
- while (map[pos].key) {
- if (!strcmp(map[pos].key, key)) {
- map[pos].val = val; /* keep the first (lowest) id, like a dict insert */
- return;
- }
- pos = (int)((pos + 1) & (int)mask);
- }
- map[pos].key = key;
- map[pos].val = val;
-}
-
-static int map_get(const wp_entry_t *map, int cap, const char *key) {
- if (!map)
- return -1;
- uint64_t mask = (uint64_t)cap - 1;
- int pos = (int)(fnv1a_hash(key) & mask);
- while (map[pos].key) {
- if (!strcmp(map[pos].key, key))
- return map[pos].val;
- pos = (int)((pos + 1) & (int)mask);
- }
- return -1;
-}
-
-/* ============================== UTF-8 helpers ================================ */
-
-/* Decode one codepoint. Returns bytes consumed (>=1); on a malformed lead byte
- * emits U+FFFD and consumes one byte so iteration always advances. */
-static int utf8_decode(const unsigned char *p, const unsigned char *end, uint32_t *cp) {
- unsigned char c = p[0];
- if (c < 0x80) {
- *cp = c;
- return 1;
- }
- int n;
- uint32_t v;
- if ((c & 0xE0) == 0xC0) {
- n = 2;
- v = c & 0x1F;
- } else if ((c & 0xF0) == 0xE0) {
- n = 3;
- v = c & 0x0F;
- } else if ((c & 0xF8) == 0xF0) {
- n = 4;
- v = c & 0x07;
- } else {
- *cp = 0xFFFD;
- return 1;
- }
- if (p + n > end) {
- *cp = 0xFFFD;
- return 1;
- }
- for (int i = 1; i < n; i++) {
- if ((p[i] & 0xC0) != 0x80) {
- *cp = 0xFFFD;
- return 1;
- }
- v = (v << 6) | (p[i] & 0x3F);
- }
- *cp = v;
- return n;
-}
-
-static int utf8_encode(uint32_t cp, char out[4]) {
- if (cp < 0x80) {
- out[0] = (char)cp;
- return 1;
- }
- if (cp < 0x800) {
- out[0] = (char)(0xC0 | (cp >> 6));
- out[1] = (char)(0x80 | (cp & 0x3F));
- return 2;
- }
- if (cp < 0x10000) {
- out[0] = (char)(0xE0 | (cp >> 12));
- out[1] = (char)(0x80 | ((cp >> 6) & 0x3F));
- out[2] = (char)(0x80 | (cp & 0x3F));
- return 3;
- }
- out[0] = (char)(0xF0 | (cp >> 18));
- out[1] = (char)(0x80 | ((cp >> 12) & 0x3F));
- out[2] = (char)(0x80 | ((cp >> 6) & 0x3F));
- out[3] = (char)(0x80 | (cp & 0x3F));
- return 4;
-}
-
-/* ========================== Unicode classification ========================== */
-
-static int wp_is_whitespace(uint32_t cp) {
- if (cp == ' ' || cp == '\t' || cp == '\n' || cp == '\r')
- return 1;
- return cp == 0x00A0 || cp == 0x1680 || (cp >= 0x2000 && cp <= 0x200A) || cp == 0x2028 ||
- cp == 0x2029 || cp == 0x202F || cp == 0x205F || cp == 0x3000;
-}
-
-/* Binary search a sorted, non-overlapping [lo, hi] range table. */
-static int wp_in_ranges(uint32_t cp, const wp_range_t *r, int n) {
- int lo = 0, hi = n - 1;
- while (lo <= hi) {
- int mid = (lo + hi) >> 1;
- if (cp < r[mid].lo)
- hi = mid - 1;
- else if (cp > r[mid].hi)
- lo = mid + 1;
- else
- return 1;
- }
- return 0;
-}
-
-/* transformers' _is_control: tab/newline/carriage-return are whitespace, not
- * control (they fall through to wp_is_whitespace); everything else is Unicode
- * category Cc, Cf, or Co. */
-static int wp_is_control(uint32_t cp) {
- if (cp == '\t' || cp == '\n' || cp == '\r')
- return 0;
- return wp_in_ranges(cp, WP_CONTROL_RANGES, WP_CONTROL_RANGES_N);
-}
-
-/* transformers' _is_punctuation: the ASCII non-alphanumeric set plus every
- * Unicode category P* codepoint, both folded into the generated table. */
-static int wp_is_punct(uint32_t cp) { return wp_in_ranges(cp, WP_PUNCT_RANGES, WP_PUNCT_RANGES_N); }
-
-static int wp_is_cjk(uint32_t cp) {
- return (cp >= 0x4E00 && cp <= 0x9FFF) || (cp >= 0x3400 && cp <= 0x4DBF) ||
- (cp >= 0x20000 && cp <= 0x2A6DF) || (cp >= 0x2A700 && cp <= 0x2B73F) ||
- (cp >= 0x2B740 && cp <= 0x2B81F) || (cp >= 0x2B820 && cp <= 0x2CEAF) ||
- (cp >= 0xF900 && cp <= 0xFAFF) || (cp >= 0x2F800 && cp <= 0x2FA1F);
-}
-
-/* Hangul syllable decomposition constants (the standard algorithm). Hangul
- * syllables are not cased, so their do_lower_case fold is the canonical NFD into
- * leading (L), vowel (V), and optional trailing (T) jamo - none combining marks
- * - which keeps them out of the generated WP_FOLD table. */
-enum {
- WP_HANGUL_S = 0xAC00,
- WP_HANGUL_L = 0x1100,
- WP_HANGUL_V = 0x1161,
- WP_HANGUL_T = 0x11A7,
- WP_HANGUL_TCOUNT = 28,
- WP_HANGUL_NCOUNT = 588,
- WP_HANGUL_SCOUNT = 11172
-};
-
-/* Apply the do_lower_case fold (Python str.lower then NFD then drop combining
- * marks) to one codepoint, writing 0..3 output codepoints to out and the count
- * to *n_out. 0 outputs means the codepoint is a dropped mark. Hangul decomposes
- * by formula; everything else is the generated WP_FOLD table or an identity. */
-static void wp_fold(uint32_t cp, uint32_t out[3], int *n_out) {
- if (cp >= WP_HANGUL_S && cp < WP_HANGUL_S + WP_HANGUL_SCOUNT) {
- int s = (int)(cp - WP_HANGUL_S);
- out[0] = (uint32_t)(WP_HANGUL_L + s / WP_HANGUL_NCOUNT);
- out[1] = (uint32_t)(WP_HANGUL_V + (s % WP_HANGUL_NCOUNT) / WP_HANGUL_TCOUNT);
- int t = s % WP_HANGUL_TCOUNT;
- if (t == 0) {
- *n_out = 2;
- } else {
- out[2] = (uint32_t)(WP_HANGUL_T + t);
- *n_out = 3;
- }
- return;
- }
- int lo = 0, hi = WP_FOLD_N - 1;
- while (lo <= hi) {
- int mid = (lo + hi) >> 1;
- if (cp < WP_FOLD[mid].cp) {
- hi = mid - 1;
- } else if (cp > WP_FOLD[mid].cp) {
- lo = mid + 1;
- } else {
- int n = WP_FOLD[mid].n;
- for (int i = 0; i < n; i++)
- out[i] = WP_FOLD[mid].out[i];
- *n_out = n;
- return;
- }
- }
- out[0] = cp;
- *n_out = 1;
-}
-
-/* =============================== workspace ================================== */
-
-struct wordpiece_workspace {
- uint32_t *cps; /* cleaned + CJK-spaced codepoints of the whole input */
- int cps_cap;
- uint32_t *norm; /* one word's normalized codepoints */
- int norm_cap;
- char *wbuf; /* one sub-word's UTF-8 bytes */
- int wbuf_cap;
- int *boff; /* byte offset of each codepoint within wbuf (+1 sentinel) */
- int boff_cap;
- char *look; /* "##" + candidate substring, for vocab lookup */
- int look_cap;
- int *ids; /* output ids */
- int ids_cap;
-};
-
-static int grow_u32(uint32_t **buf, int *cap, int need) {
- if (*cap >= need)
- return 0;
- int n = *cap ? *cap : 64;
- while (n < need)
- n *= 2;
- uint32_t *p = (uint32_t *)realloc(*buf, (size_t)n * sizeof(uint32_t));
- if (!p)
- return -1;
- *buf = p;
- *cap = n;
- return 0;
-}
-
-static int grow_int(int **buf, int *cap, int need) {
- if (*cap >= need)
- return 0;
- int n = *cap ? *cap : 64;
- while (n < need)
- n *= 2;
- int *p = (int *)realloc(*buf, (size_t)n * sizeof(int));
- if (!p)
- return -1;
- *buf = p;
- *cap = n;
- return 0;
-}
-
-static int grow_char(char **buf, int *cap, int need) {
- if (*cap >= need)
- return 0;
- int n = *cap ? *cap : 256;
- while (n < need)
- n *= 2;
- char *p = (char *)realloc(*buf, (size_t)n);
- if (!p)
- return -1;
- *buf = p;
- *cap = n;
- return 0;
-}
-
-wordpiece_workspace_t *wordpiece_workspace_new(void) {
- return (wordpiece_workspace_t *)calloc(1, sizeof(wordpiece_workspace_t));
-}
-
-void wordpiece_workspace_free(wordpiece_workspace_t *ws) {
- if (!ws)
- return;
- free(ws->cps);
- free(ws->norm);
- free(ws->wbuf);
- free(ws->boff);
- free(ws->look);
- free(ws->ids);
- free(ws);
-}
-
-/* ============================== WordPiece core ============================== */
-
-/* Append the WordPiece ids of one BasicTokenizer sub-word (codepoints
- * word[0..n)) to ws->ids[*n_out]. The whole sub-word maps to [UNK] when it is
- * too long or any greedy piece misses the vocab. Returns 0 / -1 on OOM. */
-static int wordpiece_subword(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const uint32_t *word,
- int n,
- int *n_out) {
- if (n == 0)
- return 0;
- if (n > tok->max_chars_per_word) {
- if (grow_int(&ws->ids, &ws->ids_cap, *n_out + 1) != 0)
- return -1;
- ws->ids[(*n_out)++] = tok->unk_id;
- return 0;
- }
-
- /* Materialize the sub-word as UTF-8 with per-codepoint byte offsets. */
- if (grow_int(&ws->boff, &ws->boff_cap, n + 1) != 0 ||
- grow_char(&ws->wbuf, &ws->wbuf_cap, n * 4 + 1) != 0)
- return -1;
- int bytes = 0;
- for (int i = 0; i < n; i++) {
- ws->boff[i] = bytes;
- bytes += utf8_encode(word[i], ws->wbuf + bytes);
- }
- ws->boff[n] = bytes;
-
- int start_count = *n_out;
- int start = 0;
- int bad = 0;
- while (start < n) {
- int end = n;
- int matched = -1;
- int matched_end = start;
- while (start < end) {
- int seg = ws->boff[end] - ws->boff[start];
- int lp = 0;
- if (grow_char(&ws->look, &ws->look_cap, seg + 3) != 0)
- return -1;
- if (start > 0) {
- ws->look[lp++] = '#';
- ws->look[lp++] = '#';
- }
- memcpy(ws->look + lp, ws->wbuf + ws->boff[start], (size_t)seg);
- ws->look[lp + seg] = '\0';
- int id = map_get((const wp_entry_t *)tok->vocab_map, tok->vocab_map_cap, ws->look);
- if (id >= 0) {
- matched = id;
- matched_end = end;
- break;
- }
- end--;
- }
- if (matched < 0) {
- bad = 1;
- break;
- }
- if (grow_int(&ws->ids, &ws->ids_cap, *n_out + 1) != 0)
- return -1;
- ws->ids[(*n_out)++] = matched;
- start = matched_end;
- }
- if (bad) {
- *n_out = start_count; /* discard partial pieces; emit one [UNK] */
- if (grow_int(&ws->ids, &ws->ids_cap, *n_out + 1) != 0)
- return -1;
- ws->ids[(*n_out)++] = tok->unk_id;
- }
- return 0;
-}
-
-/* Run the full BasicTokenizer + WordPiece pipeline into ws->ids. Returns the
- * token count, or -1 on allocation failure. */
-static int
-wordpiece_run(const wordpiece_tokenizer_t *tok, wordpiece_workspace_t *ws, const char *text) {
- /* clean_text + tokenize_chinese_chars, in one pass over the input. */
- const unsigned char *p = (const unsigned char *)text;
- const unsigned char *end = p + strlen(text);
- int n_cps = 0;
- while (p < end) {
- uint32_t cp;
- p += utf8_decode(p, end, &cp);
- if (cp == 0 || cp == 0xFFFD || wp_is_control(cp))
- continue;
- if (wp_is_whitespace(cp)) {
- if (grow_u32(&ws->cps, &ws->cps_cap, n_cps + 1) != 0)
- return -1;
- ws->cps[n_cps++] = ' ';
- continue;
- }
- int extra = wp_is_cjk(cp) ? 3 : 1;
- if (grow_u32(&ws->cps, &ws->cps_cap, n_cps + extra) != 0)
- return -1;
- if (extra == 3) {
- ws->cps[n_cps++] = ' ';
- ws->cps[n_cps++] = cp;
- ws->cps[n_cps++] = ' ';
- } else {
- ws->cps[n_cps++] = cp;
- }
- }
-
- int n_out = 0;
- int i = 0;
- while (i < n_cps) {
- if (ws->cps[i] == ' ') {
- i++;
- continue;
- }
- int wstart = i;
- while (i < n_cps && ws->cps[i] != ' ')
- i++;
- int wlen = i - wstart;
-
- /* lowercase + strip accents; the fold can drop a codepoint (combining
- * mark) or expand it to up to 3 (Hangul, a few ligatures). */
- int m = 0;
- if (grow_u32(&ws->norm, &ws->norm_cap, tok->do_lower_case ? wlen * 3 : wlen) != 0)
- return -1;
- for (int k = 0; k < wlen; k++) {
- uint32_t cp = ws->cps[wstart + k];
- if (tok->do_lower_case) {
- uint32_t out[3];
- int no = 0;
- wp_fold(cp, out, &no);
- for (int j = 0; j < no; j++)
- ws->norm[m++] = out[j];
- } else {
- ws->norm[m++] = cp;
- }
- }
-
- /* split_on_punc, then WordPiece each piece. */
- int run = 0;
- for (int k = 0; k < m; k++) {
- if (wp_is_punct(ws->norm[k])) {
- if (run > 0 && wordpiece_subword(tok, ws, ws->norm + (k - run), run, &n_out) != 0)
- return -1;
- run = 0;
- if (wordpiece_subword(tok, ws, ws->norm + k, 1, &n_out) != 0)
- return -1;
- } else {
- run++;
- }
- }
- if (run > 0 && wordpiece_subword(tok, ws, ws->norm + (m - run), run, &n_out) != 0)
- return -1;
- }
- return n_out;
-}
-
-/* ================================ loading =================================== */
-
-static char *read_file(const char *path, long *out_len) {
- FILE *f = fopen(path, "rb");
- if (!f)
- return NULL;
- if (fseek(f, 0, SEEK_END) != 0) {
- fclose(f);
- return NULL;
- }
- long len = ftell(f);
- if (len < 0 || fseek(f, 0, SEEK_SET) != 0) {
- fclose(f);
- return NULL;
- }
- char *buf = (char *)malloc((size_t)len + 1);
- if (!buf) {
- fclose(f);
- return NULL;
- }
- size_t rd = fread(buf, 1, (size_t)len, f);
- fclose(f);
- if (rd != (size_t)len) {
- free(buf);
- return NULL;
- }
- buf[len] = '\0';
- if (out_len)
- *out_len = len;
- return buf;
-}
-
-/* do_lower_case from tokenizer_config.json: default true (BERT uncased). A
- * lightweight scan is enough - we only need one boolean and avoid a JSON dep
- * here, mirroring how the config layer reads small flags. */
-static int read_do_lower_case(const char *model_dir) {
- char path[2048];
- snprintf(path, sizeof(path), "%s/tokenizer_config.json", model_dir);
- char *buf = read_file(path, NULL);
- if (!buf)
- return 1;
- int result = 1;
- const char *k = strstr(buf, "\"do_lower_case\"");
- if (k) {
- const char *c = strchr(k, ':');
- if (c) {
- c++;
- while (*c == ' ' || *c == '\t' || *c == '\n' || *c == '\r')
- c++;
- if (!strncmp(c, "false", 5))
- result = 0;
- }
- }
- free(buf);
- return result;
-}
-
-wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir) {
- if (!model_dir)
- return NULL;
- char path[2048];
- snprintf(path, sizeof(path), "%s/vocab.txt", model_dir);
- long len = 0;
- char *buf = read_file(path, &len);
- if (!buf) {
- fprintf(stderr, "wordpiece: cannot read %s\n", path);
- return NULL;
- }
-
- /* Count lines (one token each); a single trailing newline adds no token. */
- int count = 0;
- for (long i = 0; i < len; i++)
- if (buf[i] == '\n')
- count++;
- if (len > 0 && buf[len - 1] != '\n')
- count++;
-
- wordpiece_tokenizer_t *tok = (wordpiece_tokenizer_t *)calloc(1, sizeof(*tok));
- char **id_to_token = (char **)calloc((size_t)(count > 0 ? count : 1), sizeof(char *));
- if (!tok || !id_to_token) {
- free(buf);
- free(tok);
- free(id_to_token);
- return NULL;
- }
-
- int n = 0;
- char *line = buf;
- for (long i = 0; i <= len && n < count; i++) {
- if (i == len || buf[i] == '\n') {
- char *e = buf + i;
- if (e > line && e[-1] == '\r')
- e[-1] = '\0';
- else
- *e = '\0';
- id_to_token[n] = strdup(line);
- if (!id_to_token[n]) {
- for (int j = 0; j < n; j++)
- free(id_to_token[j]);
- free(id_to_token);
- free(tok);
- free(buf);
- return NULL;
- }
- n++;
- line = buf + i + 1;
- }
- }
- free(buf);
-
- tok->id_to_token = id_to_token;
- tok->vocab_size = n;
- tok->do_lower_case = read_do_lower_case(model_dir);
- tok->max_chars_per_word = 100;
-
- int cap = next_pow2(n * 2 < 16 ? 16 : n * 2);
- wp_entry_t *map = (wp_entry_t *)calloc((size_t)cap, sizeof(wp_entry_t));
- if (!map) {
- wordpiece_tokenizer_free(tok);
- return NULL;
- }
- for (int i = 0; i < n; i++)
- map_put(map, cap, id_to_token[i], i);
- tok->vocab_map = map;
- tok->vocab_map_cap = cap;
-
- tok->unk_id = map_get(map, cap, "[UNK]");
- if (tok->unk_id < 0) {
- fprintf(stderr, "wordpiece: vocab.txt has no [UNK] token\n");
- wordpiece_tokenizer_free(tok);
- return NULL;
- }
- return tok;
-}
-
-/* ============================== public API ================================= */
-
-const char *wordpiece_tokenizer_decode(const wordpiece_tokenizer_t *tok, int id) {
- if (!tok || id < 0 || id >= tok->vocab_size)
- return NULL;
- return tok->id_to_token[id];
-}
-
-int wordpiece_tokenizer_token_id(const wordpiece_tokenizer_t *tok, const char *token) {
- if (!tok || !token)
- return -1;
- return map_get((const wp_entry_t *)tok->vocab_map, tok->vocab_map_cap, token);
-}
-
-int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- if (!tok || !ws || !text)
- return -1;
- int n = wordpiece_run(tok, ws, text);
- if (n < 0)
- return -1;
- if (out_n_tokens)
- *out_n_tokens = n;
- if (!out_ids || out_cap < n)
- return out_ids ? -2 : -1;
- memcpy(out_ids, ws->ids, (size_t)n * sizeof(int));
- return 0;
-}
-
-int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const char *text,
- int *out_n_tokens) {
- if (out_n_tokens)
- *out_n_tokens = 0;
- if (!tok || !ws || !text)
- return NULL;
- int n = wordpiece_run(tok, ws, text);
- if (n < 0)
- return NULL;
- int *ids = (int *)malloc((size_t)(n > 0 ? n : 1) * sizeof(int));
- if (!ids)
- return NULL;
- memcpy(ids, ws->ids, (size_t)n * sizeof(int));
- if (out_n_tokens)
- *out_n_tokens = n;
- return ids;
-}
-
-int *wordpiece_tokenizer_encode(const wordpiece_tokenizer_t *tok, const char *text, int *out_n) {
- if (out_n)
- *out_n = 0;
- wordpiece_workspace_t *ws = wordpiece_workspace_new();
- if (!ws)
- return NULL;
- int *ids = wordpiece_tokenizer_encode_with_workspace(tok, ws, text, out_n);
- wordpiece_workspace_free(ws);
- return ids;
-}
-
-void wordpiece_tokenizer_free(wordpiece_tokenizer_t *tok) {
- if (!tok)
- return;
- if (tok->id_to_token) {
- for (int i = 0; i < tok->vocab_size; i++)
- free(tok->id_to_token[i]);
- free(tok->id_to_token);
- }
- free(tok->vocab_map);
- free(tok);
-}
diff --git a/src/tokenizer_wordpiece.h b/src/tokenizer_wordpiece.h
deleted file mode 100644
index 59cb2bb..0000000
--- a/src/tokenizer_wordpiece.h
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * wordpiece_tokenizer.h - BERT-family WordPiece tokenizer.
- *
- * Implements the BERT tokenization pipeline: a BasicTokenizer (clean, optional
- * lowercase + accent strip, CJK spacing, whitespace and punctuation splitting)
- * followed by greedy longest-match WordPiece against a `vocab.txt`. It encodes
- * text only; the frontend wraps the result with [CLS]/[SEP], exactly as the
- * byte-level BPE backend leaves terminal/separator tokens to the frontend.
- *
- * The public surface mirrors embed_tokenizer.h (load / encode with a reusable
- * workspace / decode / resolve a named special token / free) so both can sit
- * behind one tokenizer interface later.
- */
-
-#ifndef WORDPIECE_TOKENIZER_H
-#define WORDPIECE_TOKENIZER_H
-
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-typedef struct {
- char **id_to_token; /* [vocab_size] vocab.txt entries, line order = id */
- int vocab_size;
- int unk_id; /* [UNK] id, resolved at load */
- int do_lower_case; /* lowercase + strip accents before WordPiece */
- int max_chars_per_word; /* words longer than this map to [UNK] (BERT: 100) */
-
- /* Opaque token-string -> id hash map. */
- void *vocab_map;
- int vocab_map_cap;
-} wordpiece_tokenizer_t;
-
-typedef struct wordpiece_workspace wordpiece_workspace_t;
-
-/* Load from <model_dir>/vocab.txt, reading do_lower_case from
- * <model_dir>/tokenizer_config.json (default: lowercase). Returns NULL on
- * error (missing vocab.txt, missing [UNK], allocation failure). */
-EMBED_API wordpiece_tokenizer_t *wordpiece_tokenizer_load(const char *model_dir);
-
-/* Decode a single id to its vocab token string (internal pointer), or NULL. */
-EMBED_API const char *wordpiece_tokenizer_decode(const wordpiece_tokenizer_t *tok, int id);
-
-/* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */
-EMBED_API int wordpiece_tokenizer_token_id(const wordpiece_tokenizer_t *tok, const char *token);
-
-/* Encode UTF-8 text into a malloc'd id array; sets *out_n_tokens. NULL on
- * error (and *out_n_tokens = 0). */
-EMBED_API int *
-wordpiece_tokenizer_encode(const wordpiece_tokenizer_t *tok, const char *text, int *out_n_tokens);
-
-/* Reusable scratch for allocation-light repeated encoding. */
-EMBED_API wordpiece_workspace_t *wordpiece_workspace_new(void);
-EMBED_API void wordpiece_workspace_free(wordpiece_workspace_t *ws);
-
-/* Encode into caller-provided out_ids[0..out_cap-1]. Returns 0 on success, -1
- * on error, -2 if out_cap is too small. *out_n_tokens (when non-NULL) is set to
- * the number of tokens needed/emitted in all cases. */
-EMBED_API int wordpiece_tokenizer_encode_into(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens);
-
-/* Same tokenization as wordpiece_tokenizer_encode(), reusing ws internally and
- * performing only the final returned int[] allocation. */
-EMBED_API int *wordpiece_tokenizer_encode_with_workspace(const wordpiece_tokenizer_t *tok,
- wordpiece_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
-
-EMBED_API void wordpiece_tokenizer_free(wordpiece_tokenizer_t *tok);
-
-#endif /* WORDPIECE_TOKENIZER_H */
diff --git a/src/vec.c b/src/vec.c
deleted file mode 100644
index cea1576..0000000
--- a/src/vec.c
+++ /dev/null
@@ -1,53 +0,0 @@
-#include "embed.h"
-
-#include <math.h>
-
-/* ========================================================================
- * Vector helpers
- *
- * Standalone post-processing on embedding vectors: callers (CLI, server) use
- * these to L2-normalize outputs and to score pooled embeddings with cosine
- * similarity. They touch no model or tensor internals, only the public API.
- * ======================================================================== */
-
-int embed_l2_normalize(float *vec, int dim) {
- if (!vec || dim <= 0)
- return -1;
-
- float norm_sq = 0.0f;
- for (int i = 0; i < dim; i++)
- norm_sq += vec[i] * vec[i];
-
- if (!(norm_sq > 0.0f) || !isfinite(norm_sq))
- return -1;
-
- float inv_norm = 1.0f / sqrtf(norm_sq);
- for (int i = 0; i < dim; i++)
- vec[i] *= inv_norm;
- return 0;
-}
-
-float embed_cosine_similarity(const float *a, const float *b, int dim) {
- if (!a || !b || dim <= 0)
- return 0.0f;
-
- float dot = 0.0f;
- float norm_a_sq = 0.0f;
- float norm_b_sq = 0.0f;
- for (int i = 0; i < dim; i++) {
- float av = a[i];
- float bv = b[i];
- dot += av * bv;
- norm_a_sq += av * av;
- norm_b_sq += bv * bv;
- }
-
- if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) || !isfinite(norm_a_sq) ||
- !isfinite(norm_b_sq) || !isfinite(dot))
- return 0.0f;
-
- double denom = sqrt((double)norm_a_sq) * sqrt((double)norm_b_sq);
- if (!(denom > 0.0) || !isfinite(denom))
- return 0.0f;
- return (float)((double)dot / denom);
-}