aboutsummaryrefslogtreecommitdiff
path: root/embed.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-15 08:40:11 +0200
commitc4a1c97a09da6d8d88c2ac834bf6234506fc41bc (patch)
tree81ce5675ce886aad0309a41f4d6a696826e0a061 /embed.c
parentb377ba8656bfb807ed8dda6c9f26942b50ce626d (diff)
downloadffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.tar.gz
ffwd-c4a1c97a09da6d8d88c2ac834bf6234506fc41bc.zip
chore: run clang-format
Diffstat (limited to 'embed.c')
-rw-r--r--embed.c1048
1 files changed, 532 insertions, 516 deletions
diff --git a/embed.c b/embed.c
index bd72846..d9e7936 100644
--- a/embed.c
+++ b/embed.c
@@ -31,32 +31,32 @@ struct embed_late_workspace {
int embed_verbose = 0;
int qwen_verbose = 0;
-#define EMBED_BF16_QKV_FUSE_MAX_SEQ 16
-#define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4
-#define EMBED_MIN_WORKSPACE_SEQ_CAP 16
+#define EMBED_BF16_QKV_FUSE_MAX_SEQ 16
+#define EMBED_BF16_PAIR_FUSE_MAX_SEQ 4
+#define EMBED_MIN_WORKSPACE_SEQ_CAP 16
#define EMBED_WORKSPACE_SEQ_GRANULARITY 16
/* ========================================================================
* Size helpers
* ======================================================================== */
-static int mul_size(size_t a, size_t b, size_t *out)
-{
- if (a != 0 && b > SIZE_MAX / a) return -1;
+static int mul_size(size_t a, size_t b, size_t *out) {
+ if (a != 0 && b > SIZE_MAX / a)
+ return -1;
*out = a * b;
return 0;
}
-static int add_size(size_t a, size_t b, size_t *out)
-{
- if (b > SIZE_MAX - a) return -1;
+static int add_size(size_t a, size_t b, size_t *out) {
+ if (b > SIZE_MAX - a)
+ return -1;
*out = a + b;
return 0;
}
-static int grow_cap(int current, int needed, int *out)
-{
- if (needed <= 0) return -1;
+static int grow_cap(int current, int needed, int *out) {
+ if (needed <= 0)
+ return -1;
int cap = needed;
if (cap < EMBED_MIN_WORKSPACE_SEQ_CAP)
@@ -65,7 +65,8 @@ static int grow_cap(int current, int needed, int *out)
int rem = cap % EMBED_WORKSPACE_SEQ_GRANULARITY;
if (rem != 0) {
int add = EMBED_WORKSPACE_SEQ_GRANULARITY - rem;
- if (cap > INT_MAX - add) return -1;
+ if (cap > INT_MAX - add)
+ return -1;
cap += add;
}
@@ -76,54 +77,57 @@ static int grow_cap(int current, int needed, int *out)
return 0;
}
-static int realloc_floats(float **ptr, size_t count)
-{
+static int realloc_floats(float **ptr, size_t count) {
size_t bytes;
- if (mul_size(count, sizeof(float), &bytes) != 0) return -1;
+ if (mul_size(count, sizeof(float), &bytes) != 0)
+ return -1;
void *p = realloc(*ptr, bytes);
- if (!p && bytes != 0) return -1;
+ if (!p && bytes != 0)
+ return -1;
*ptr = (float *)p;
return 0;
}
-static float *malloc_floats(size_t count)
-{
+static float *malloc_floats(size_t count) {
size_t bytes;
- if (mul_size(count, sizeof(float), &bytes) != 0) return NULL;
+ if (mul_size(count, sizeof(float), &bytes) != 0)
+ return NULL;
return (float *)malloc(bytes);
}
-static int model_dir_has_file(const char *model_dir, const char *rel)
-{
+static int model_dir_has_file(const char *model_dir, const char *rel) {
char path[1024];
int n = snprintf(path, sizeof(path), "%s/%s", model_dir, rel);
- if (n < 0 || (size_t)n >= sizeof(path)) return 0;
+ if (n < 0 || (size_t)n >= sizeof(path))
+ return 0;
FILE *f = fopen(path, "rb");
- if (!f) return 0;
+ if (!f)
+ return 0;
fclose(f);
return 1;
}
-static int model_dir_has_late_projection(const char *model_dir)
-{
+static int model_dir_has_late_projection(const char *model_dir) {
return model_dir_has_file(model_dir, "1_Dense/config.json") &&
model_dir_has_file(model_dir, "1_Dense/model.safetensors");
}
-static int realloc_floats_2d(float **ptr, int rows, int cols)
-{
- if (rows < 0 || cols < 0) return -1;
+static int realloc_floats_2d(float **ptr, int rows, int cols) {
+ if (rows < 0 || cols < 0)
+ return -1;
size_t count;
- if (mul_size((size_t)rows, (size_t)cols, &count) != 0) return -1;
+ if (mul_size((size_t)rows, (size_t)cols, &count) != 0)
+ return -1;
return realloc_floats(ptr, count);
}
-static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens)
-{
- if (!ws || !ws->model || n_tokens <= 0) return -1;
- if (n_tokens <= ws->states_seq_cap) return 0;
+static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens) {
+ if (!ws || !ws->model || n_tokens <= 0)
+ return -1;
+ if (n_tokens <= ws->states_seq_cap)
+ return 0;
int hidden = ws->model->base->config.hidden_size;
if (realloc_floats_2d(&ws->states, n_tokens, hidden) != 0)
return -1;
@@ -131,13 +135,14 @@ static int ensure_late_state_buffer(embed_late_workspace_t *ws, int n_tokens)
return 0;
}
-static int realloc_ints(int **ptr, size_t count)
-{
+static int realloc_ints(int **ptr, size_t count) {
size_t bytes;
- if (mul_size(count, sizeof(int), &bytes) != 0) return -1;
+ if (mul_size(count, sizeof(int), &bytes) != 0)
+ return -1;
void *p = realloc(*ptr, bytes);
- if (!p && bytes != 0) return -1;
+ if (!p && bytes != 0)
+ return -1;
*ptr = (int *)p;
return 0;
}
@@ -146,15 +151,14 @@ static int realloc_ints(int **ptr, size_t count)
* Minimal JSON helpers for config.json parsing
* ======================================================================== */
-static const char *skip_ws(const char *p)
-{
- while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t') p++;
+static const char *skip_ws(const char *p) {
+ while (*p == ' ' || *p == '\n' || *p == '\r' || *p == '\t')
+ p++;
return p;
}
/* Find "key": <value> in a flat JSON object. Returns pointer to value start. */
-static const char *json_find_key(const char *json, const char *key)
-{
+static const char *json_find_key(const char *json, const char *key) {
size_t klen = strlen(key);
const char *p = json;
while ((p = strstr(p, key)) != NULL) {
@@ -163,7 +167,8 @@ static const char *json_find_key(const char *json, const char *key)
const char *after = p + klen;
if (*after == '"') {
after = skip_ws(after + 1);
- if (*after == ':') return skip_ws(after + 1);
+ if (*after == ':')
+ return skip_ws(after + 1);
}
}
p += klen;
@@ -171,32 +176,30 @@ static const char *json_find_key(const char *json, const char *key)
return NULL;
}
-static int json_get_int(const char *json, const char *key, int fallback)
-{
+static int json_get_int(const char *json, const char *key, int fallback) {
const char *v = json_find_key(json, key);
- if (!v) return fallback;
+ if (!v)
+ return fallback;
return atoi(v);
}
-static double json_get_double(const char *json, const char *key, double fallback)
-{
+static double json_get_double(const char *json, const char *key, double fallback) {
const char *v = json_find_key(json, key);
- if (!v) return fallback;
+ if (!v)
+ return fallback;
return atof(v);
}
-static int json_string_equals(const char *json, const char *key,
- const char *expected)
-{
+static int json_string_equals(const char *json, const char *key, const char *expected) {
const char *v = json_find_key(json, key);
- if (!v || *v != '"') return 0;
+ if (!v || *v != '"')
+ return 0;
v++;
size_t len = strlen(expected);
return strncmp(v, expected, len) == 0 && v[len] == '"';
}
-static int parse_config(embed_config_t *cfg, const char *model_dir)
-{
+static int parse_config(embed_config_t *cfg, const char *model_dir) {
char path[1024];
int n = snprintf(path, sizeof(path), "%s/config.json", model_dir);
if (n < 0 || (size_t)n >= sizeof(path)) {
@@ -221,22 +224,24 @@ static int parse_config(embed_config_t *cfg, const char *model_dir)
}
char *buf = (char *)malloc((size_t)sz + 1);
if (!buf || fread(buf, 1, (size_t)sz, f) != (size_t)sz) {
- fclose(f); free(buf); return -1;
+ fclose(f);
+ free(buf);
+ return -1;
}
fclose(f);
buf[sz] = '\0';
- cfg->hidden_size = json_get_int(buf, "hidden_size", 0);
- cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0);
- cfg->n_heads = json_get_int(buf, "num_attention_heads", 0);
- cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0);
- cfg->head_dim = json_get_int(buf, "head_dim", EMBED_HEAD_DIM);
+ cfg->hidden_size = json_get_int(buf, "hidden_size", 0);
+ cfg->n_layers = json_get_int(buf, "num_hidden_layers", 0);
+ cfg->n_heads = json_get_int(buf, "num_attention_heads", 0);
+ cfg->n_kv_heads = json_get_int(buf, "num_key_value_heads", 0);
+ cfg->head_dim = json_get_int(buf, "head_dim", EMBED_HEAD_DIM);
cfg->intermediate_size = json_get_int(buf, "intermediate_size", 0);
- cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE);
- cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6);
- cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0);
- cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
- cfg->pooling_mode = EMBED_POOL_MEAN;
+ cfg->vocab_size = json_get_int(buf, "vocab_size", EMBED_VOCAB_SIZE);
+ cfg->rms_norm_eps = (float)json_get_double(buf, "rms_norm_eps", 1e-6);
+ cfg->rope_theta = (float)json_get_double(buf, "rope_theta", 1000000.0);
+ cfg->attention_mode = EMBED_ATTENTION_BIDIRECTIONAL;
+ cfg->pooling_mode = EMBED_POOL_MEAN;
cfg->normalize_embeddings = 0;
cfg->append_terminal_token = 0;
@@ -249,45 +254,40 @@ static int parse_config(embed_config_t *cfg, const char *model_dir)
cfg->q_dim = 0;
cfg->kv_dim = 0;
- if (cfg->n_heads > 0 && cfg->head_dim > 0 &&
- cfg->n_heads <= INT_MAX / cfg->head_dim)
+ if (cfg->n_heads > 0 && cfg->head_dim > 0 && cfg->n_heads <= INT_MAX / cfg->head_dim)
cfg->q_dim = cfg->n_heads * cfg->head_dim;
- if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 &&
- cfg->n_kv_heads <= INT_MAX / cfg->head_dim)
+ if (cfg->n_kv_heads > 0 && cfg->head_dim > 0 && cfg->n_kv_heads <= INT_MAX / cfg->head_dim)
cfg->kv_dim = cfg->n_kv_heads * cfg->head_dim;
free(buf);
/* Sanity checks */
- if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 ||
- cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 ||
- cfg->head_dim <= 0 || cfg->intermediate_size <= 0 ||
- cfg->vocab_size <= 0 || cfg->q_dim <= 0 || cfg->kv_dim <= 0 ||
- (cfg->head_dim & 1) || cfg->n_heads % cfg->n_kv_heads != 0 ||
- !isfinite(cfg->rms_norm_eps) || cfg->rms_norm_eps <= 0.0f ||
- !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
- fprintf(stderr, "embed_model_load: invalid config in %s "
+ if (cfg->hidden_size <= 0 || cfg->n_layers <= 0 || cfg->n_heads <= 0 || cfg->n_kv_heads <= 0 ||
+ cfg->head_dim <= 0 || cfg->intermediate_size <= 0 || cfg->vocab_size <= 0 ||
+ cfg->q_dim <= 0 || cfg->kv_dim <= 0 || (cfg->head_dim & 1) ||
+ cfg->n_heads % cfg->n_kv_heads != 0 || !isfinite(cfg->rms_norm_eps) ||
+ cfg->rms_norm_eps <= 0.0f || !isfinite(cfg->rope_theta) || cfg->rope_theta <= 0.0f) {
+ fprintf(stderr,
+ "embed_model_load: invalid config in %s "
"(hidden=%d, layers=%d, heads=%d/%d, head_dim=%d, inter=%d)\n",
- path, cfg->hidden_size, cfg->n_layers, cfg->n_heads,
- cfg->n_kv_heads, cfg->head_dim, cfg->intermediate_size);
+ path, cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads, cfg->head_dim,
+ cfg->intermediate_size);
return -1;
}
if (cfg->n_layers > EMBED_MAX_LAYERS) {
- fprintf(stderr, "embed_model_load: too many layers (%d > %d)\n",
- cfg->n_layers, EMBED_MAX_LAYERS);
+ fprintf(stderr, "embed_model_load: too many layers (%d > %d)\n", cfg->n_layers,
+ EMBED_MAX_LAYERS);
return -1;
}
if (embed_verbose >= 1)
- fprintf(stderr, "config: hidden=%d, layers=%d, heads=%d/%d, "
+ fprintf(stderr,
+ "config: hidden=%d, layers=%d, heads=%d/%d, "
"inter=%d, head_dim=%d, attention=%s, pooling=%s\n",
- cfg->hidden_size, cfg->n_layers,
- cfg->n_heads, cfg->n_kv_heads,
+ cfg->hidden_size, cfg->n_layers, cfg->n_heads, cfg->n_kv_heads,
cfg->intermediate_size, cfg->head_dim,
- cfg->attention_mode == EMBED_ATTENTION_CAUSAL
- ? "causal" : "bidirectional",
- cfg->pooling_mode == EMBED_POOL_LAST_TOKEN
- ? "last-token" : "mean");
+ cfg->attention_mode == EMBED_ATTENTION_CAUSAL ? "causal" : "bidirectional",
+ cfg->pooling_mode == EMBED_POOL_LAST_TOKEN ? "last-token" : "mean");
return 0;
}
@@ -296,48 +296,57 @@ static int parse_config(embed_config_t *cfg, const char *model_dir)
* Weight loading (direct mmap pointers into safetensors)
* ======================================================================== */
-static size_t dtype_size(safetensor_dtype_t dtype)
-{
+static size_t dtype_size(safetensor_dtype_t dtype) {
switch (dtype) {
- case DTYPE_F32: return sizeof(float);
- case DTYPE_BF16: return sizeof(uint16_t);
- default: return 0;
+ case DTYPE_F32:
+ return sizeof(float);
+ case DTYPE_BF16:
+ return sizeof(uint16_t);
+ default:
+ return 0;
}
}
-static const char *dtype_name(safetensor_dtype_t dtype)
-{
+static const char *dtype_name(safetensor_dtype_t dtype) {
switch (dtype) {
- case DTYPE_F32: return "F32";
- case DTYPE_F16: return "F16";
- case DTYPE_BF16: return "BF16";
- case DTYPE_I32: return "I32";
- case DTYPE_I64: return "I64";
- case DTYPE_BOOL: return "BOOL";
- default: return "UNKNOWN";
+ case DTYPE_F32:
+ return "F32";
+ case DTYPE_F16:
+ return "F16";
+ case DTYPE_BF16:
+ return "BF16";
+ case DTYPE_I32:
+ return "I32";
+ case DTYPE_I64:
+ return "I64";
+ case DTYPE_BOOL:
+ return "BOOL";
+ default:
+ return "UNKNOWN";
}
}
static int tensor_has_supported_shape(const safetensors_file_t *sf,
- const safetensor_t *t, const char *name,
- const int64_t *shape, int ndim,
- int bf16_ok)
-{
+ const safetensor_t *t,
+ const char *name,
+ const int64_t *shape,
+ int ndim,
+ int bf16_ok) {
if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) {
- fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n",
- name, dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32");
+ fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name,
+ dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32");
return 0;
}
if (t->ndim != ndim) {
- fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n",
- name, t->ndim, ndim);
+ fprintf(stderr, "embed: bad rank for %s: got %d, expected %d\n", name, t->ndim, ndim);
return 0;
}
size_t numel = 1;
for (int i = 0; i < ndim; i++) {
if (t->shape[i] != shape[i]) {
- fprintf(stderr, "embed: bad shape for %s at dim %d: "
+ fprintf(stderr,
+ "embed: bad shape for %s at dim %d: "
"got %lld, expected %lld\n",
name, i, (long long)t->shape[i], (long long)shape[i]);
return 0;
@@ -351,8 +360,7 @@ static int tensor_has_supported_shape(const safetensors_file_t *sf,
size_t elem_size = dtype_size(t->dtype);
if (elem_size == 0) {
- fprintf(stderr, "embed: unsupported dtype for %s: got %s\n",
- name, dtype_name(t->dtype));
+ fprintf(stderr, "embed: unsupported dtype for %s: got %s\n", name, dtype_name(t->dtype));
return 0;
}
@@ -362,14 +370,13 @@ static int tensor_has_supported_shape(const safetensors_file_t *sf,
return 0;
}
if (t->data_size != bytes) {
- fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n",
- name, t->data_size, bytes);
+ fprintf(stderr, "embed: bad data size for %s: got %zu, expected %zu\n", name, t->data_size,
+ bytes);
return 0;
}
size_t data_start = 8 + sf->header_size;
- if (data_start > sf->file_size ||
- t->data_offset > sf->file_size - data_start ||
+ if (data_start > sf->file_size || t->data_offset > sf->file_size - data_start ||
t->data_size > sf->file_size - data_start - t->data_offset) {
fprintf(stderr, "embed: tensor data out of bounds for %s\n", name);
return 0;
@@ -377,13 +384,11 @@ static int tensor_has_supported_shape(const safetensors_file_t *sf,
return 1;
}
-static embed_weight_ref_t load_weight_direct(multi_safetensors_t *ms,
- const char *name,
- const int64_t *shape, int ndim)
-{
- embed_weight_ref_t ref = { DTYPE_UNKNOWN, NULL };
+static embed_weight_ref_t
+load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) {
+ embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL};
safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
fprintf(stderr, "embed: tensor not found: %s\n", name);
return ref;
@@ -396,11 +401,10 @@ static embed_weight_ref_t load_weight_direct(multi_safetensors_t *ms,
return ref;
}
-static float *load_norm_f32(multi_safetensors_t *ms, const char *name,
- const int64_t *shape, int ndim)
-{
+static float *
+load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) {
safetensors_file_t *sf = NULL;
- const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
+ const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
fprintf(stderr, "embed: tensor not found: %s\n", name);
return NULL;
@@ -410,17 +414,14 @@ static float *load_norm_f32(multi_safetensors_t *ms, const char *name,
return safetensors_get_f32(sf, t);
}
-static void bf16_row_to_f32(float *dst, const uint16_t *src, int n)
-{
+static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) {
for (int i = 0; i < n; i++) {
uint32_t u = ((uint32_t)src[i]) << 16;
memcpy(dst + i, &u, sizeof(float));
}
}
-static void copy_weight_row(float *dst, const embed_weight_ref_t *w,
- size_t row, int row_width)
-{
+static void copy_weight_row(float *dst, const embed_weight_ref_t *w, size_t row, int row_width) {
if (w->dtype == DTYPE_F32) {
const float *src = (const float *)w->data + row * (size_t)row_width;
memcpy(dst, src, (size_t)row_width * sizeof(float));
@@ -430,23 +431,26 @@ static void copy_weight_row(float *dst, const embed_weight_ref_t *w,
}
}
-static int ensure_bf16_widen(embed_workspace_t *ws, size_t count)
-{
- if (ws->bf16_widen_count >= count) return 0;
+static int ensure_bf16_widen(embed_workspace_t *ws, size_t count) {
+ if (ws->bf16_widen_count >= count)
+ return 0;
float *p = (float *)realloc(ws->bf16_widen, count * sizeof(float));
- if (!p) return -1;
+ if (!p)
+ return -1;
ws->bf16_widen = p;
ws->bf16_widen_count = count;
return 0;
}
-static void linear_nobias_weight(embed_workspace_t *ws, float *y,
- const float *x, const embed_weight_ref_t *w,
- int seq_len, int in_dim, int out_dim)
-{
+static void linear_nobias_weight(embed_workspace_t *ws,
+ float *y,
+ const float *x,
+ const embed_weight_ref_t *w,
+ int seq_len,
+ int in_dim,
+ int out_dim) {
if (w->dtype == DTYPE_F32) {
- qwen_linear_nobias(y, x, (const float *)w->data,
- seq_len, in_dim, out_dim);
+ qwen_linear_nobias(y, x, (const float *)w->data, seq_len, in_dim, out_dim);
return;
}
if (seq_len > 16) {
@@ -454,31 +458,31 @@ static void linear_nobias_weight(embed_workspace_t *ws, float *y,
* BLAS-backed F32 path; below that the fused BF16 row kernels win. */
size_t count = (size_t)out_dim * (size_t)in_dim;
if (ensure_bf16_widen(ws, count) == 0) {
- qwen_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data,
- count);
+ qwen_bf16_to_f32_buf(ws->bf16_widen, (const uint16_t *)w->data, count);
qwen_linear_nobias(y, x, ws->bf16_widen, seq_len, in_dim, out_dim);
return;
}
}
- qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data,
- seq_len, in_dim, out_dim);
+ qwen_linear_nobias_bf16(y, x, (const uint16_t *)w->data, seq_len, in_dim, out_dim);
}
-static void linear_qkv_weight(embed_workspace_t *ws, float *q, float *k,
- float *v, const float *x,
+static void linear_qkv_weight(embed_workspace_t *ws,
+ float *q,
+ float *k,
+ float *v,
+ const float *x,
const embed_weight_ref_t *wq,
const embed_weight_ref_t *wk,
const embed_weight_ref_t *wv,
- int seq_len, int in_dim, int q_dim, int kv_dim)
-{
- if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 &&
- wv->dtype == DTYPE_BF16 &&
+ int seq_len,
+ int in_dim,
+ int q_dim,
+ int kv_dim) {
+ if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && wv->dtype == DTYPE_BF16 &&
seq_len <= EMBED_BF16_QKV_FUSE_MAX_SEQ) {
- qwen_linear_nobias_bf16_qkv(q, k, v, x,
- (const uint16_t *)wq->data,
- (const uint16_t *)wk->data,
- (const uint16_t *)wv->data,
- seq_len, in_dim, q_dim, kv_dim);
+ qwen_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data,
+ (const uint16_t *)wk->data, (const uint16_t *)wv->data, seq_len,
+ in_dim, q_dim, kv_dim);
return;
}
@@ -487,18 +491,20 @@ static void linear_qkv_weight(embed_workspace_t *ws, float *q, float *k,
linear_nobias_weight(ws, v, x, wv, seq_len, in_dim, kv_dim);
}
-static void linear_pair_weight(embed_workspace_t *ws, float *a, float *b,
+static void linear_pair_weight(embed_workspace_t *ws,
+ float *a,
+ float *b,
const float *x,
const embed_weight_ref_t *wa,
const embed_weight_ref_t *wb,
- int seq_len, int in_dim, int a_dim, int b_dim)
-{
+ int seq_len,
+ int in_dim,
+ int a_dim,
+ int b_dim) {
if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 &&
seq_len <= EMBED_BF16_PAIR_FUSE_MAX_SEQ) {
- qwen_linear_nobias_bf16_pair(a, b, x,
- (const uint16_t *)wa->data,
- (const uint16_t *)wb->data,
- seq_len, in_dim, a_dim, b_dim);
+ qwen_linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data,
+ (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim);
return;
}
@@ -510,26 +516,29 @@ static void linear_pair_weight(embed_workspace_t *ws, float *a, float *b,
* Working buffer management
* ======================================================================== */
-static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq)
-{
- if (seq <= ws->buf_seq_cap) return 0;
+static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int seq) {
+ if (seq <= ws->buf_seq_cap)
+ return 0;
int cap;
- if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0) return -1;
+ if (grow_cap(ws->buf_seq_cap, seq, &cap) != 0)
+ return -1;
-#define R(ptr, n) do { \
- if (realloc_floats_2d(&(ptr), cap, (n)) != 0) return -1; \
-} while (0)
+#define R(ptr, n) \
+ do { \
+ if (realloc_floats_2d(&(ptr), cap, (n)) != 0) \
+ return -1; \
+ } while (0)
- R(ws->x, c->hidden_size);
- R(ws->x_norm, c->hidden_size);
- R(ws->q, c->q_dim);
- R(ws->k, c->kv_dim);
- R(ws->v, c->kv_dim);
+ R(ws->x, c->hidden_size);
+ R(ws->x_norm, c->hidden_size);
+ R(ws->q, c->q_dim);
+ R(ws->k, c->kv_dim);
+ R(ws->v, c->kv_dim);
R(ws->attn_out, c->q_dim);
R(ws->proj_out, c->hidden_size);
R(ws->ffn_gate, c->intermediate_size);
- R(ws->ffn_up, c->intermediate_size);
+ R(ws->ffn_up, c->intermediate_size);
#undef R
@@ -537,15 +546,14 @@ static int ensure_buffers(embed_workspace_t *ws, const embed_config_t *c, int se
return 0;
}
-static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets,
- int batch)
-{
- size_t bytes =
- qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
- if (bytes <= ws->attn_scores_bytes) return;
+static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets, int batch) {
+ size_t bytes = qwen_bidirectional_gqa_attention_packed_scratch_bytes(offsets, batch);
+ if (bytes <= ws->attn_scores_bytes)
+ return;
void *p = realloc(ws->attn_scores, bytes);
- if (!p) return;
+ if (!p)
+ return;
ws->attn_scores = (float *)p;
ws->attn_scores_bytes = bytes;
}
@@ -554,20 +562,20 @@ static void ensure_attention_scores(embed_workspace_t *ws, const int *offsets,
* RoPE cache
* ======================================================================== */
-static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg,
- int n_pos)
-{
- if (n_pos <= ws->rope_cache_cap) return 0;
+static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg, int n_pos) {
+ if (n_pos <= ws->rope_cache_cap)
+ return 0;
int cap = ws->rope_cache_cap > 0 ? ws->rope_cache_cap : 512;
while (cap < n_pos) {
- if (cap > INT_MAX / 2) return -1;
+ if (cap > INT_MAX / 2)
+ return -1;
cap *= 2;
}
- int head_dim = cfg->head_dim;
- int half = head_dim / 2;
- float theta = cfg->rope_theta;
+ int head_dim = cfg->head_dim;
+ int half = head_dim / 2;
+ float theta = cfg->rope_theta;
size_t count;
if (mul_size((size_t)cap, (size_t)head_dim, &count) != 0)
@@ -585,8 +593,10 @@ static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg,
float inv = 1.0f / powf(theta, (float)(2 * d) / (float)head_dim);
float angle = fp * inv;
float cv = cosf(angle), sv = sinf(angle);
- c[d] = cv; c[half + d] = cv;
- s[d] = sv; s[half + d] = sv;
+ c[d] = cv;
+ c[half + d] = cv;
+ s[d] = sv;
+ s[half + d] = sv;
}
}
@@ -594,12 +604,13 @@ static int ensure_rope_cache(embed_workspace_t *ws, const embed_config_t *cfg,
return 0;
}
-static int ensure_offsets(embed_workspace_t *ws, int batch)
-{
- if (!ws || batch <= 0 || batch == INT_MAX) return -1;
+static int ensure_offsets(embed_workspace_t *ws, int batch) {
+ if (!ws || batch <= 0 || batch == INT_MAX)
+ return -1;
int needed = batch + 1;
- if (needed <= ws->offsets_cap) return 0;
+ if (needed <= ws->offsets_cap)
+ return 0;
if (realloc_ints(&ws->offsets, (size_t)needed) != 0)
return -1;
@@ -611,19 +622,18 @@ static int ensure_offsets(embed_workspace_t *ws, int batch)
* Model / workspace load and free
* ======================================================================== */
-static embed_model_t *model_load_range_ex(const char *model_dir,
- int layer_start, int layer_end,
- int allow_late)
-{
+static embed_model_t *
+model_load_range_ex(const char *model_dir, int layer_start, int layer_end, int allow_late) {
if (!allow_late && model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "embed_model_load: late-interaction models require "
- "token-level MaxSim support and are not valid pooled "
- "embedding models yet\n");
+ "token-level MaxSim support and are not valid pooled "
+ "embedding models yet\n");
return NULL;
}
embed_model_t *model = (embed_model_t *)calloc(1, sizeof(embed_model_t));
- if (!model) return NULL;
+ if (!model)
+ return NULL;
/* Parse config.json */
if (parse_config(&model->config, model_dir) != 0) {
@@ -634,19 +644,19 @@ static embed_model_t *model_load_range_ex(const char *model_dir,
const embed_config_t *cfg = &model->config;
if (layer_end == -1)
layer_end = cfg->n_layers;
- if (layer_start < 0 || layer_start >= layer_end ||
- layer_end > cfg->n_layers) {
- fprintf(stderr, "embed_model_load: invalid layer range [%d, %d) "
- "for %d-layer model\n", layer_start, layer_end, cfg->n_layers);
+ if (layer_start < 0 || layer_start >= layer_end || layer_end > cfg->n_layers) {
+ fprintf(stderr,
+ "embed_model_load: invalid layer range [%d, %d) "
+ "for %d-layer model\n",
+ layer_start, layer_end, cfg->n_layers);
free(model);
return NULL;
}
model->layer_start = layer_start;
model->layer_end = layer_end;
- model->attention =
- cfg->attention_mode == EMBED_ATTENTION_CAUSAL
- ? qwen_causal_gqa_attention_packed_with_scratch
- : qwen_bidirectional_gqa_attention_packed_with_scratch;
+ model->attention = cfg->attention_mode == EMBED_ATTENTION_CAUSAL
+ ? qwen_causal_gqa_attention_packed_with_scratch
+ : qwen_bidirectional_gqa_attention_packed_with_scratch;
/* Open safetensors (handles single file or multi-shard) */
multi_safetensors_t *ms = multi_safetensors_open(model_dir);
@@ -663,57 +673,53 @@ static embed_model_t *model_load_range_ex(const char *model_dir,
/* Only the first stage needs token embeddings. */
embed_weights_t *w = &model->weights;
if (layer_start == 0) {
- const int64_t embed_shape[2] = { cfg->vocab_size, cfg->hidden_size };
- w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight",
- embed_shape, 2);
- if (!w->embed_tokens.data) goto fail;
+ const int64_t embed_shape[2] = {cfg->vocab_size, cfg->hidden_size};
+ w->embed_tokens = load_weight_direct(ms, "embed_tokens.weight", embed_shape, 2);
+ if (!w->embed_tokens.data)
+ goto fail;
}
/* Allocate layer array */
w->layers = (embed_layer_t *)calloc(cfg->n_layers, sizeof(embed_layer_t));
- if (!w->layers) goto fail;
+ if (!w->layers)
+ goto fail;
/* Per-layer weights */
char name[256];
for (int i = layer_start; i < layer_end; i++) {
embed_layer_t *l = &w->layers[i];
-#define LOAD_NORM(field, fmt, d0) do { \
- const int64_t expect[1] = { (d0) }; \
- snprintf(name, sizeof(name), fmt, i); \
- l->field = load_norm_f32(ms, name, expect, 1); \
- if (!l->field) goto fail; \
-} while (0)
+#define LOAD_NORM(field, fmt, d0) \
+ do { \
+ const int64_t expect[1] = {(d0)}; \
+ snprintf(name, sizeof(name), fmt, i); \
+ l->field = load_norm_f32(ms, name, expect, 1); \
+ if (!l->field) \
+ goto fail; \
+ } while (0)
-#define LOAD2(field, fmt, d0, d1) do { \
- const int64_t expect[2] = { (d0), (d1) }; \
- snprintf(name, sizeof(name), fmt, i); \
- l->field = load_weight_direct(ms, name, expect, 2); \
- if (!l->field.data) goto fail; \
-} while (0)
+#define LOAD2(field, fmt, d0, d1) \
+ do { \
+ const int64_t expect[2] = {(d0), (d1)}; \
+ snprintf(name, sizeof(name), fmt, i); \
+ l->field = load_weight_direct(ms, name, expect, 2); \
+ if (!l->field.data) \
+ goto fail; \
+ } while (0)
- LOAD2(wq, "layers.%d.self_attn.q_proj.weight",
- cfg->q_dim, cfg->hidden_size);
- LOAD2(wk, "layers.%d.self_attn.k_proj.weight",
- cfg->kv_dim, cfg->hidden_size);
- LOAD2(wv, "layers.%d.self_attn.v_proj.weight",
- cfg->kv_dim, cfg->hidden_size);
- LOAD2(wo, "layers.%d.self_attn.o_proj.weight",
- cfg->hidden_size, cfg->q_dim);
- LOAD_NORM(q_norm, "layers.%d.self_attn.q_norm.weight",
- cfg->head_dim);
- LOAD_NORM(k_norm, "layers.%d.self_attn.k_norm.weight",
- cfg->head_dim);
- LOAD_NORM(input_norm, "layers.%d.input_layernorm.weight",
- cfg->hidden_size);
- LOAD_NORM(post_attn_norm, "layers.%d.post_attention_layernorm.weight",
- cfg->hidden_size);
- LOAD2(gate_proj, "layers.%d.mlp.gate_proj.weight",
- cfg->intermediate_size, cfg->hidden_size);
- LOAD2(up_proj, "layers.%d.mlp.up_proj.weight",
- cfg->intermediate_size, cfg->hidden_size);
- LOAD2(down_proj, "layers.%d.mlp.down_proj.weight",
- cfg->hidden_size, cfg->intermediate_size);
+ LOAD2(wq, "layers.%d.self_attn.q_proj.weight", cfg->q_dim, cfg->hidden_size);
+ LOAD2(wk, "layers.%d.self_attn.k_proj.weight", cfg->kv_dim, cfg->hidden_size);
+ LOAD2(wv, "layers.%d.self_attn.v_proj.weight", cfg->kv_dim, cfg->hidden_size);
+ LOAD2(wo, "layers.%d.self_attn.o_proj.weight", cfg->hidden_size, cfg->q_dim);
+ LOAD_NORM(q_norm, "layers.%d.self_attn.q_norm.weight", cfg->head_dim);
+ LOAD_NORM(k_norm, "layers.%d.self_attn.k_norm.weight", cfg->head_dim);
+ LOAD_NORM(input_norm, "layers.%d.input_layernorm.weight", cfg->hidden_size);
+ LOAD_NORM(post_attn_norm, "layers.%d.post_attention_layernorm.weight", cfg->hidden_size);
+ LOAD2(gate_proj, "layers.%d.mlp.gate_proj.weight", cfg->intermediate_size,
+ cfg->hidden_size);
+ LOAD2(up_proj, "layers.%d.mlp.up_proj.weight", cfg->intermediate_size, cfg->hidden_size);
+ LOAD2(down_proj, "layers.%d.mlp.down_proj.weight", cfg->hidden_size,
+ cfg->intermediate_size);
#undef LOAD2
#undef LOAD_NORM
@@ -724,13 +730,15 @@ static embed_model_t *model_load_range_ex(const char *model_dir,
/* Only the final stage applies the output RMSNorm. */
if (layer_end == cfg->n_layers) {
- const int64_t norm_shape[1] = { cfg->hidden_size };
+ const int64_t norm_shape[1] = {cfg->hidden_size};
w->norm = load_norm_f32(ms, "norm.weight", norm_shape, 1);
- if (!w->norm) goto fail;
+ if (!w->norm)
+ goto fail;
}
if (embed_verbose >= 1)
- fprintf(stderr, "embed_model_load: layers [%d, %d) loaded "
+ fprintf(stderr,
+ "embed_model_load: layers [%d, %d) loaded "
"(%d-dim embeddings)\n",
layer_start, layer_end, cfg->hidden_size);
@@ -741,14 +749,13 @@ fail:
return NULL;
}
-embed_model_t *embed_model_load(const char *model_dir)
-{
+embed_model_t *embed_model_load(const char *model_dir) {
return model_load_range_ex(model_dir, 0, -1, 0);
}
-void embed_model_free(embed_model_t *model)
-{
- if (!model) return;
+void embed_model_free(embed_model_t *model) {
+ if (!model)
+ return;
if (model->weights.layers) {
for (int i = 0; i < model->config.n_layers; i++) {
embed_layer_t *l = &model->weights.layers[i];
@@ -765,38 +772,43 @@ void embed_model_free(embed_model_t *model)
free(model);
}
-embed_workspace_t *embed_workspace_new(const embed_model_t *model)
-{
- if (!model) return NULL;
+embed_workspace_t *embed_workspace_new(const embed_model_t *model) {
+ if (!model)
+ return NULL;
embed_workspace_t *ws = (embed_workspace_t *)calloc(1, sizeof(embed_workspace_t));
- if (!ws) return NULL;
+ if (!ws)
+ return NULL;
ws->model = model;
return ws;
}
-void embed_workspace_free(embed_workspace_t *ws)
-{
- if (!ws) return;
- free(ws->x); free(ws->x_norm);
- free(ws->q); free(ws->k); free(ws->v);
- free(ws->attn_out); free(ws->proj_out);
- free(ws->ffn_gate); free(ws->ffn_up);
+void embed_workspace_free(embed_workspace_t *ws) {
+ if (!ws)
+ return;
+ free(ws->x);
+ free(ws->x_norm);
+ free(ws->q);
+ free(ws->k);
+ free(ws->v);
+ free(ws->attn_out);
+ free(ws->proj_out);
+ free(ws->ffn_gate);
+ free(ws->ffn_up);
free(ws->attn_scores);
free(ws->offsets);
- free(ws->rope_cos); free(ws->rope_sin);
+ free(ws->rope_cos);
+ free(ws->rope_sin);
free(ws->bf16_widen);
free(ws);
}
-const embed_config_t *embed_model_config(const embed_model_t *model)
-{
+const embed_config_t *embed_model_config(const embed_model_t *model) {
return model ? &model->config : NULL;
}
-static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf,
- const char *name)
-{
- if (!sf || !name) return NULL;
+static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf, const char *name) {
+ if (!sf || !name)
+ return NULL;
for (int i = 0; i < sf->num_tensors; i++) {
if (strcmp(sf->tensors[i].name, name) == 0)
return &sf->tensors[i];
@@ -804,22 +816,22 @@ static const safetensor_t *find_single_safetensor(const safetensors_file_t *sf,
return NULL;
}
-embed_late_model_t *embed_late_model_load(const char *model_dir)
-{
+embed_late_model_t *embed_late_model_load(const char *model_dir) {
if (!model_dir_has_late_projection(model_dir)) {
fprintf(stderr, "embed_late_model_load: missing 1_Dense projection\n");
return NULL;
}
embed_late_model_t *late = (embed_late_model_t *)calloc(1, sizeof(*late));
- if (!late) return NULL;
+ if (!late)
+ return NULL;
late->base = model_load_range_ex(model_dir, 0, -1, 1);
- if (!late->base) goto fail;
+ if (!late->base)
+ goto fail;
char path[1024];
- int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors",
- model_dir);
+ int n = snprintf(path, sizeof(path), "%s/1_Dense/model.safetensors", model_dir);
if (n < 0 || (size_t)n >= sizeof(path)) {
fprintf(stderr, "embed_late_model_load: model path too long\n");
goto fail;
@@ -831,8 +843,7 @@ embed_late_model_t *embed_late_model_load(const char *model_dir)
goto fail;
}
- const safetensor_t *t = find_single_safetensor(late->dense_sf,
- "linear.weight");
+ const safetensor_t *t = find_single_safetensor(late->dense_sf, "linear.weight");
if (!t) {
fprintf(stderr, "embed_late_model_load: missing linear.weight\n");
goto fail;
@@ -842,9 +853,8 @@ embed_late_model_t *embed_late_model_load(const char *model_dir)
goto fail;
}
- int64_t shape[2] = { t->shape[0], late->base->config.hidden_size };
- if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight",
- shape, 2, 1))
+ int64_t shape[2] = {t->shape[0], late->base->config.hidden_size};
+ if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1))
goto fail;
late->projection.dtype = t->dtype;
@@ -857,19 +867,20 @@ fail:
return NULL;
}
-void embed_late_model_free(embed_late_model_t *model)
-{
- if (!model) return;
+void embed_late_model_free(embed_late_model_t *model) {
+ if (!model)
+ return;
embed_model_free(model->base);
safetensors_close(model->dense_sf);
free(model);
}
-embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model)
-{
- if (!model || !model->base) return NULL;
+embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model) {
+ if (!model || !model->base)
+ return NULL;
embed_late_workspace_t *ws = (embed_late_workspace_t *)calloc(1, sizeof(*ws));
- if (!ws) return NULL;
+ if (!ws)
+ return NULL;
ws->model = model;
ws->base_ws = embed_workspace_new(model->base);
if (!ws->base_ws) {
@@ -879,43 +890,41 @@ embed_late_workspace_t *embed_late_workspace_new(const embed_late_model_t *model
return ws;
}
-void embed_late_workspace_free(embed_late_workspace_t *ws)
-{
- if (!ws) return;
+void embed_late_workspace_free(embed_late_workspace_t *ws) {
+ if (!ws)
+ return;
embed_workspace_free(ws->base_ws);
free(ws->states);
free(ws);
}
-const embed_config_t *embed_late_model_config(const embed_late_model_t *model)
-{
+const embed_config_t *embed_late_model_config(const embed_late_model_t *model) {
return model && model->base ? &model->base->config : NULL;
}
-int embed_late_model_token_dim(const embed_late_model_t *model)
-{
+int embed_late_model_token_dim(const embed_late_model_t *model) {
return model ? model->token_dim : 0;
}
int embed_late_model_encode_tokens(const embed_late_model_t *model,
- embed_late_workspace_t *ws,
- const int *token_ids, int n_tokens,
- int normalize, float *out_vectors)
-{
- if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 ||
- !out_vectors || model->token_dim <= 0)
+ embed_late_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ int normalize,
+ float *out_vectors) {
+ if (!model || !ws || ws->model != model || !token_ids || n_tokens <= 0 || !out_vectors ||
+ model->token_dim <= 0)
return -1;
int hidden = model->base->config.hidden_size;
int dim = model->token_dim;
if (ensure_late_state_buffer(ws, n_tokens) != 0)
return -1;
- if (embed_model_forward_into(model->base, ws->base_ws, token_ids,
- n_tokens, ws->states) != 0)
+ if (embed_model_forward_into(model->base, ws->base_ws, token_ids, n_tokens, ws->states) != 0)
return -1;
- linear_nobias_weight(ws->base_ws, out_vectors, ws->states,
- &model->projection, n_tokens, hidden, dim);
+ linear_nobias_weight(ws->base_ws, out_vectors, ws->states, &model->projection, n_tokens, hidden,
+ dim);
if (normalize) {
for (int i = 0; i < n_tokens; i++) {
if (embed_l2_normalize(out_vectors + (size_t)i * dim, dim) != 0)
@@ -925,12 +934,12 @@ int embed_late_model_encode_tokens(const embed_late_model_t *model,
return 0;
}
-float embed_late_maxsim(const float *query_vectors, int query_tokens,
- const float *doc_vectors, int doc_tokens,
- int dim)
-{
- if (!query_vectors || !doc_vectors || query_tokens <= 0 ||
- doc_tokens <= 0 || dim <= 0)
+float embed_late_maxsim(const float *query_vectors,
+ int query_tokens,
+ const float *doc_vectors,
+ int doc_tokens,
+ int dim) {
+ if (!query_vectors || !doc_vectors || query_tokens <= 0 || doc_tokens <= 0 || dim <= 0)
return 0.0f;
float score = 0.0f;
@@ -940,20 +949,23 @@ float embed_late_maxsim(const float *query_vectors, int query_tokens,
for (int di = 0; di < doc_tokens; di++) {
const float *d = doc_vectors + (size_t)di * dim;
float dot = qwen_dot_f32(q, d, dim);
- if (dot > best) best = dot;
+ if (dot > best)
+ best = dot;
}
score += best;
}
return score;
}
-int embed_late_maxsim_batch(const float *query_vectors, int query_tokens,
- const float *doc_vectors,
- const int *doc_offsets, int docs,
- int dim, float *scores)
-{
- if (!query_vectors || !doc_vectors || !doc_offsets || !scores ||
- query_tokens <= 0 || docs <= 0 || dim <= 0 || doc_offsets[0] != 0)
+int embed_late_maxsim_batch(const float *query_vectors,
+ int query_tokens,
+ const float *doc_vectors,
+ const int *doc_offsets,
+ int docs,
+ int dim,
+ float *scores) {
+ if (!query_vectors || !doc_vectors || !doc_offsets || !scores || query_tokens <= 0 ||
+ docs <= 0 || dim <= 0 || doc_offsets[0] != 0)
return -1;
for (int i = 0; i < docs; i++) {
@@ -969,12 +981,12 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens,
enum { LATE_GROUP_TOKEN_BUDGET = 4096 };
int total = doc_offsets[docs];
int budget = LATE_GROUP_TOKEN_BUDGET;
- size_t cap = (size_t)query_tokens *
- (size_t)(total < budget ? total : budget);
+ size_t cap = (size_t)query_tokens * (size_t)(total < budget ? total : budget);
int largest = 0;
for (int i = 0; i < docs; i++) {
int len = doc_offsets[i + 1] - doc_offsets[i];
- if (len > largest) largest = len;
+ if (len > largest)
+ largest = len;
}
if ((size_t)query_tokens * (size_t)largest > cap)
cap = (size_t)query_tokens * (size_t)largest;
@@ -985,16 +997,13 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens,
int first = doc;
int start = doc_offsets[first];
doc++;
- while (doc < docs &&
- doc_offsets[doc + 1] - start <= budget &&
- doc_offsets[doc + 1] - start <=
- (int)(cap / (size_t)query_tokens))
+ while (doc < docs && doc_offsets[doc + 1] - start <= budget &&
+ doc_offsets[doc + 1] - start <= (int)(cap / (size_t)query_tokens))
doc++;
int group_tokens = doc_offsets[doc] - start;
- qwen_matmul_t(sim, query_vectors,
- doc_vectors + (size_t)start * dim,
- query_tokens, dim, group_tokens);
+ qwen_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim,
+ group_tokens);
for (int i = first; i < doc; i++) {
int s0 = doc_offsets[i] - start;
@@ -1004,7 +1013,8 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens,
const float *row = sim + (size_t)qi * group_tokens;
float best = row[s0];
for (int t = s0 + 1; t < s1; t++)
- if (row[t] > best) best = row[t];
+ if (row[t] > best)
+ best = row[t];
score += best;
}
scores[i] = score;
@@ -1020,63 +1030,58 @@ int embed_late_maxsim_batch(const float *query_vectors, int query_tokens,
int start = doc_offsets[i];
int end = doc_offsets[i + 1];
scores[i] = embed_late_maxsim(query_vectors, query_tokens,
- doc_vectors + (size_t)start * dim,
- end - start, dim);
+ doc_vectors + (size_t)start * dim, end - start, dim);
}
return 0;
}
-size_t embed_workspace_nbytes(const embed_workspace_t *ws)
-{
- if (!ws) return 0;
+size_t embed_workspace_nbytes(const embed_workspace_t *ws) {
+ if (!ws)
+ return 0;
size_t total = sizeof(*ws);
const embed_model_t *model = ws->model;
- if (!model) return total;
+ if (!model)
+ return total;
const embed_config_t *c = &model->config;
size_t row_floats = 0;
const int dims[] = {
- c->hidden_size, /* x */
- c->hidden_size, /* x_norm */
- c->q_dim, /* q */
- c->kv_dim, /* k */
- c->kv_dim, /* v */
- c->q_dim, /* attn_out */
- c->hidden_size, /* proj_out */
- c->intermediate_size, /* ffn_gate */
- c->intermediate_size, /* ffn_up */
+ c->hidden_size, /* x */
+ c->hidden_size, /* x_norm */
+ c->q_dim, /* q */
+ c->kv_dim, /* k */
+ c->kv_dim, /* v */
+ c->q_dim, /* attn_out */
+ c->hidden_size, /* proj_out */
+ c->intermediate_size, /* ffn_gate */
+ c->intermediate_size, /* ffn_up */
};
for (size_t i = 0; i < sizeof(dims) / sizeof(dims[0]); i++) {
- if (dims[i] < 0 ||
- add_size(row_floats, (size_t)dims[i], &row_floats) != 0)
+ if (dims[i] < 0 || add_size(row_floats, (size_t)dims[i], &row_floats) != 0)
return SIZE_MAX;
}
size_t buf_floats = 0;
- if (ws->buf_seq_cap > 0 &&
- mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0)
+ if (ws->buf_seq_cap > 0 && mul_size((size_t)ws->buf_seq_cap, row_floats, &buf_floats) != 0)
return SIZE_MAX;
size_t rope_floats = 0;
if (ws->rope_cache_cap > 0) {
size_t rope_rows = 0;
- if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim,
- &rope_rows) != 0 ||
+ if (mul_size((size_t)ws->rope_cache_cap, (size_t)c->head_dim, &rope_rows) != 0 ||
mul_size(rope_rows, 2, &rope_floats) != 0)
return SIZE_MAX;
}
size_t all_floats = 0, bytes = 0;
if (add_size(buf_floats, rope_floats, &all_floats) != 0 ||
- mul_size(all_floats, sizeof(float), &bytes) != 0 ||
- add_size(total, bytes, &total) != 0)
+ mul_size(all_floats, sizeof(float), &bytes) != 0 || add_size(total, bytes, &total) != 0)
return SIZE_MAX;
size_t offset_bytes = 0;
- if (ws->offsets_cap > 0 &&
- mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0)
+ if (ws->offsets_cap > 0 && mul_size((size_t)ws->offsets_cap, sizeof(int), &offset_bytes) != 0)
return SIZE_MAX;
if (add_size(total, offset_bytes, &total) != 0)
return SIZE_MAX;
@@ -1097,9 +1102,12 @@ size_t embed_workspace_nbytes(const embed_workspace_t *ws)
* Packed forward pass
* ======================================================================== */
-static int build_offsets(const embed_input_t *inputs, int batch, int require_ids,
- int *offsets, int *total_out, int *max_out)
-{
+static int build_offsets(const embed_input_t *inputs,
+ int batch,
+ int require_ids,
+ int *offsets,
+ int *total_out,
+ int *max_out) {
if (!inputs || batch <= 0 || !offsets || !total_out || !max_out)
return -1;
@@ -1124,52 +1132,54 @@ static int build_offsets(const embed_input_t *inputs, int batch, int require_ids
static int forward_packed_slice_inplace(const embed_model_t *model,
embed_workspace_t *ws,
const embed_input_t *inputs,
- int batch, const int *offsets,
- int total_seq, int max_seq,
+ int batch,
+ const int *offsets,
+ int total_seq,
+ int max_seq,
const float *input_states,
- int layer_start, int layer_end,
- int apply_final_norm)
-{
- if (!model || !ws || ws->model != model || !inputs ||
- batch <= 0 || !offsets || total_seq <= 0 || max_seq <= 0)
+ int layer_start,
+ int layer_end,
+ int apply_final_norm) {
+ if (!model || !ws || ws->model != model || !inputs || batch <= 0 || !offsets ||
+ total_seq <= 0 || max_seq <= 0)
return -1;
- const embed_config_t *cfg = &model->config;
- const embed_weights_t *w = &model->weights;
- if (layer_start < 0 || layer_start > layer_end ||
- layer_end > cfg->n_layers ||
+ const embed_config_t *cfg = &model->config;
+ const embed_weights_t *w = &model->weights;
+ if (layer_start < 0 || layer_start > layer_end || layer_end > cfg->n_layers ||
layer_start < model->layer_start || layer_end > model->layer_end ||
- (layer_start == 0 && input_states) ||
- (layer_start != 0 && !input_states) ||
+ (layer_start == 0 && input_states) || (layer_start != 0 && !input_states) ||
(layer_start == 0 && !w->embed_tokens.data) ||
(apply_final_norm && (layer_end != cfg->n_layers || !w->norm)))
return -1;
- int hidden = cfg->hidden_size;
- int n_heads = cfg->n_heads;
+ int hidden = cfg->hidden_size;
+ int n_heads = cfg->n_heads;
int n_kv_heads = cfg->n_kv_heads;
- int head_dim = cfg->head_dim;
- int q_dim = cfg->q_dim;
- int kv_dim = cfg->kv_dim;
- int inter = cfg->intermediate_size;
- float eps = cfg->rms_norm_eps;
+ int head_dim = cfg->head_dim;
+ int q_dim = cfg->q_dim;
+ int kv_dim = cfg->kv_dim;
+ int inter = cfg->intermediate_size;
+ float eps = cfg->rms_norm_eps;
- if (ensure_buffers(ws, cfg, total_seq) != 0) return -1;
- if (ensure_rope_cache(ws, cfg, max_seq) != 0) return -1;
+ if (ensure_buffers(ws, cfg, total_seq) != 0)
+ return -1;
+ if (ensure_rope_cache(ws, cfg, max_seq) != 0)
+ return -1;
ensure_attention_scores(ws, offsets, batch);
const float *rope_cos = ws->rope_cos;
const float *rope_sin = ws->rope_sin;
- float *x = ws->x;
- float *x_norm = ws->x_norm;
- float *q_buf = ws->q;
- float *k_buf = ws->k;
- float *v_buf = ws->v;
+ float *x = ws->x;
+ float *x_norm = ws->x_norm;
+ float *q_buf = ws->q;
+ float *k_buf = ws->k;
+ float *v_buf = ws->v;
float *attn_out = ws->attn_out;
float *proj_out = ws->proj_out;
float *ffn_gate = ws->ffn_gate;
- float *ffn_up = ws->ffn_up;
+ float *ffn_up = ws->ffn_up;
/* A first stage performs embedding lookup. Later stages resume from a
* packed hidden-state tensor received from the preceding stage. */
@@ -1179,12 +1189,12 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
for (int i = 0; i < inputs[b].n_tokens; i++) {
int id = inputs[b].ids[i];
if (id < 0 || id >= cfg->vocab_size) {
- fprintf(stderr, "embed: invalid token id %d at input %d position %d\n",
- id, b, i);
+ fprintf(stderr, "embed: invalid token id %d at input %d position %d\n", id, b,
+ i);
return -1;
}
- copy_weight_row(x + (size_t)(start + i) * hidden,
- &w->embed_tokens, (size_t)id, hidden);
+ copy_weight_row(x + (size_t)(start + i) * hidden, &w->embed_tokens, (size_t)id,
+ hidden);
}
}
} else {
@@ -1200,35 +1210,31 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
qwen_rms_norm(x_norm, x, l->input_norm, total_seq, hidden, eps);
- linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm,
- &l->wq, &l->wk, &l->wv,
- total_seq, hidden, q_dim, kv_dim);
+ linear_qkv_weight(ws, q_buf, k_buf, v_buf, x_norm, &l->wq, &l->wk, &l->wv, total_seq,
+ hidden, q_dim, kv_dim);
- qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
+ qwen_rms_norm_per_head(q_buf, l->q_norm, total_seq, n_heads, head_dim, eps);
qwen_rms_norm_per_head(k_buf, l->k_norm, total_seq, n_kv_heads, head_dim, eps);
for (int b = 0; b < batch; b++) {
int start = offsets[b];
int len = offsets[b + 1] - start;
- qwen_apply_rope_neox(q_buf + (size_t)start * q_dim,
- rope_cos, rope_sin, len, n_heads, head_dim);
- qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim,
- rope_cos, rope_sin, len, n_kv_heads, head_dim);
+ qwen_apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads,
+ head_dim);
+ qwen_apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len,
+ n_kv_heads, head_dim);
}
- model->attention(
- attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads,
- n_kv_heads, head_dim, scale, ws->attn_scores,
- ws->attn_scores_bytes);
+ model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads,
+ head_dim, scale, ws->attn_scores, ws->attn_scores_bytes);
linear_nobias_weight(ws, proj_out, attn_out, &l->wo, total_seq, q_dim, hidden);
qwen_add_inplace(x, proj_out, total_seq * hidden);
qwen_rms_norm(x_norm, x, l->post_attn_norm, total_seq, hidden, eps);
- linear_pair_weight(ws, ffn_gate, ffn_up, x_norm,
- &l->gate_proj, &l->up_proj,
- total_seq, hidden, inter, inter);
+ linear_pair_weight(ws, ffn_gate, ffn_up, x_norm, &l->gate_proj, &l->up_proj, total_seq,
+ hidden, inter, inter);
qwen_silu_mul_inplace(ffn_gate, ffn_up, total_seq * inter);
linear_nobias_weight(ws, proj_out, ffn_gate, &l->down_proj, total_seq, inter, hidden);
qwen_add_inplace(x, proj_out, total_seq * hidden);
@@ -1242,21 +1248,20 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
static int forward_packed_inplace(const embed_model_t *model,
embed_workspace_t *ws,
const embed_input_t *inputs,
- int batch, const int *offsets,
- int total_seq, int max_seq,
- int apply_final_norm)
-{
- return forward_packed_slice_inplace(model, ws, inputs, batch, offsets,
- total_seq, max_seq, NULL, 0,
- model->config.n_layers,
- apply_final_norm);
+ int batch,
+ const int *offsets,
+ int total_seq,
+ int max_seq,
+ int apply_final_norm) {
+ return forward_packed_slice_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, NULL,
+ 0, model->config.n_layers, apply_final_norm);
}
static int pool_embeddings(const embed_model_t *model,
const embed_workspace_t *ws,
- const int *offsets, int batch,
- float *out_embeddings)
-{
+ const int *offsets,
+ int batch,
+ float *out_embeddings) {
const embed_config_t *cfg = &model->config;
int hidden = cfg->hidden_size;
float eps = cfg->rms_norm_eps;
@@ -1297,8 +1302,7 @@ static int pool_embeddings(const embed_model_t *model,
emb[d] *= inv_len;
}
- if (cfg->normalize_embeddings &&
- embed_l2_normalize(emb, hidden) != 0)
+ if (cfg->normalize_embeddings && embed_l2_normalize(emb, hidden) != 0)
return -1;
}
return 0;
@@ -1308,38 +1312,40 @@ static int pool_embeddings(const embed_model_t *model,
* Public forward/embed APIs
* ======================================================================== */
-int embed_model_forward_into(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens,
- float *out_states)
-{
- if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states) return -1;
+int embed_model_forward_into(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_states) {
+ if (!model || !ws || !token_ids || n_tokens <= 0 || !out_states)
+ return -1;
size_t count, bytes;
- if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size,
- &count) != 0 ||
+ if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &count) != 0 ||
mul_size(count, sizeof(float), &bytes) != 0)
return -1;
- embed_input_t input = { token_ids, n_tokens };
- int offsets[2] = { 0, n_tokens };
- if (forward_packed_inplace(model, ws, &input, 1, offsets,
- n_tokens, n_tokens, 1) != 0)
+ embed_input_t input = {token_ids, n_tokens};
+ int offsets[2] = {0, n_tokens};
+ if (forward_packed_inplace(model, ws, &input, 1, offsets, n_tokens, n_tokens, 1) != 0)
return -1;
memcpy(out_states, ws->x, bytes);
return 0;
}
-int embed_pool_spans(const embed_config_t *cfg, const float *states, int n_tokens,
- const embed_span_t *spans, int n_spans,
- float *out_embeddings)
-{
- if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 ||
- !out_embeddings)
+int embed_pool_spans(const embed_config_t *cfg,
+ const float *states,
+ int n_tokens,
+ const embed_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
+ if (!cfg || !states || n_tokens <= 0 || !spans || n_spans <= 0 || !out_embeddings)
return -1;
int hidden = cfg->hidden_size;
- if (hidden <= 0) return -1;
+ if (hidden <= 0)
+ return -1;
for (int s = 0; s < n_spans; s++) {
int start = spans[s].start;
@@ -1365,17 +1371,20 @@ int embed_pool_spans(const embed_config_t *cfg, const float *states, int n_token
}
int embed_model_encode_spans_batch(const embed_model_t *model,
- embed_workspace_t *ws,
- const embed_context_input_t *inputs, int batch,
- float *out_embeddings)
-{
+ embed_workspace_t *ws,
+ const embed_context_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
if (!model || !ws || !inputs || batch <= 0 || !out_embeddings)
return -1;
- if (ensure_offsets(ws, batch) != 0) return -1;
- if ((size_t)batch > SIZE_MAX / sizeof(embed_input_t)) return -1;
+ if (ensure_offsets(ws, batch) != 0)
+ return -1;
+ if ((size_t)batch > SIZE_MAX / sizeof(embed_input_t))
+ return -1;
embed_input_t *packed_inputs = malloc((size_t)batch * sizeof(*packed_inputs));
- if (!packed_inputs) return -1;
+ if (!packed_inputs)
+ return -1;
int *offsets = ws->offsets;
int total_seq = 0;
int max_seq = 0;
@@ -1383,9 +1392,8 @@ int embed_model_encode_spans_batch(const embed_model_t *model,
for (int b = 0; b < batch; b++) {
const embed_context_input_t *input = &inputs[b];
int n_tokens = input->input.n_tokens;
- if (!input->input.ids || n_tokens <= 0 || !input->spans ||
- input->n_spans <= 0 || total_seq > INT_MAX - n_tokens ||
- total_spans > INT_MAX - input->n_spans) {
+ if (!input->input.ids || n_tokens <= 0 || !input->spans || input->n_spans <= 0 ||
+ total_seq > INT_MAX - n_tokens || total_spans > INT_MAX - input->n_spans) {
free(packed_inputs);
return -1;
}
@@ -1393,12 +1401,12 @@ int embed_model_encode_spans_batch(const embed_model_t *model,
offsets[b] = total_seq;
total_seq += n_tokens;
total_spans += input->n_spans;
- if (n_tokens > max_seq) max_seq = n_tokens;
+ if (n_tokens > max_seq)
+ max_seq = n_tokens;
for (int s = 0; s < input->n_spans; s++) {
int start = input->spans[s].start;
int len = input->spans[s].n_tokens;
- if (start < 0 || len <= 0 || start > n_tokens ||
- len > n_tokens - start) {
+ if (start < 0 || len <= 0 || start > n_tokens || len > n_tokens - start) {
free(packed_inputs);
return -1;
}
@@ -1406,8 +1414,8 @@ int embed_model_encode_spans_batch(const embed_model_t *model,
}
offsets[batch] = total_seq;
- if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets,
- total_seq, max_seq, 1) != 0) {
+ if (forward_packed_inplace(model, ws, packed_inputs, batch, offsets, total_seq, max_seq, 1) !=
+ 0) {
free(packed_inputs);
return -1;
}
@@ -1417,39 +1425,43 @@ int embed_model_encode_spans_batch(const embed_model_t *model,
int hidden = model->config.hidden_size;
for (int b = 0; b < batch; b++) {
const embed_context_input_t *input = &inputs[b];
- if (embed_pool_spans(&model->config,
- ws->x + (size_t)offsets[b] * hidden,
- input->input.n_tokens, input->spans, input->n_spans,
- out_embeddings + (size_t)span_offset * hidden) != 0)
+ if (embed_pool_spans(&model->config, ws->x + (size_t)offsets[b] * hidden,
+ input->input.n_tokens, input->spans, input->n_spans,
+ out_embeddings + (size_t)span_offset * hidden) != 0)
return -1;
span_offset += input->n_spans;
}
return 0;
}
-int embed_model_encode_spans(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens,
- const embed_span_t *spans, int n_spans,
- float *out_embeddings)
-{
+int embed_model_encode_spans(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ const embed_span_t *spans,
+ int n_spans,
+ float *out_embeddings) {
embed_context_input_t input = {
- .input = { token_ids, n_tokens },
+ .input = {token_ids, n_tokens},
.spans = spans,
.n_spans = n_spans,
};
return embed_model_encode_spans_batch(model, ws, &input, 1, out_embeddings);
}
-float *embed_model_forward(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens)
-{
- if (!model || !ws || !token_ids || n_tokens <= 0) return NULL;
+float *embed_model_forward(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens) {
+ if (!model || !ws || !token_ids || n_tokens <= 0)
+ return NULL;
size_t n;
if (mul_size((size_t)n_tokens, (size_t)model->config.hidden_size, &n) != 0)
return NULL;
float *out = malloc_floats(n);
- if (!out) return NULL;
+ if (!out)
+ return NULL;
if (embed_model_forward_into(model, ws, token_ids, n_tokens, out) != 0) {
free(out);
@@ -1458,13 +1470,16 @@ float *embed_model_forward(const embed_model_t *model, embed_workspace_t *ws,
return out;
}
-int embed_model_encode_batch(const embed_model_t *model, embed_workspace_t *ws,
- const embed_input_t *inputs, int batch,
- float *out_embeddings)
-{
- if (!model || !ws || !inputs || batch <= 0 || !out_embeddings) return -1;
+int embed_model_encode_batch(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const embed_input_t *inputs,
+ int batch,
+ float *out_embeddings) {
+ if (!model || !ws || !inputs || batch <= 0 || !out_embeddings)
+ return -1;
- if (ensure_offsets(ws, batch) != 0) return -1;
+ if (ensure_offsets(ws, batch) != 0)
+ return -1;
int *offsets = ws->offsets;
int total_seq = 0;
@@ -1472,20 +1487,19 @@ int embed_model_encode_batch(const embed_model_t *model, embed_workspace_t *ws,
if (build_offsets(inputs, batch, 1, offsets, &total_seq, &max_seq) != 0)
return -1;
- int rc = forward_packed_inplace(model, ws, inputs, batch, offsets,
- total_seq, max_seq, 0);
+ int rc = forward_packed_inplace(model, ws, inputs, batch, offsets, total_seq, max_seq, 0);
if (rc == 0)
rc = pool_embeddings(model, ws, offsets, batch, out_embeddings);
return rc;
}
-int embed_pool_batch(const embed_config_t *cfg, const float *states,
- const int *seq_lengths, int batch,
- float *out_embeddings)
-{
- if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings ||
- cfg->hidden_size <= 0)
+int embed_pool_batch(const embed_config_t *cfg,
+ const float *states,
+ const int *seq_lengths,
+ int batch,
+ float *out_embeddings) {
+ if (!cfg || !states || !seq_lengths || batch <= 0 || !out_embeddings || cfg->hidden_size <= 0)
return -1;
int offset = 0;
@@ -1496,14 +1510,12 @@ int embed_pool_batch(const embed_config_t *cfg, const float *states,
float *emb = out_embeddings + (size_t)b * cfg->hidden_size;
if (cfg->pooling_mode == EMBED_POOL_LAST_TOKEN) {
- const float *row =
- states + (size_t)(offset + len - 1) * cfg->hidden_size;
+ const float *row = states + (size_t)(offset + len - 1) * cfg->hidden_size;
memcpy(emb, row, (size_t)cfg->hidden_size * sizeof(float));
} else {
memset(emb, 0, (size_t)cfg->hidden_size * sizeof(float));
for (int t = 0; t < len; t++) {
- const float *row =
- states + (size_t)(offset + t) * cfg->hidden_size;
+ const float *row = states + (size_t)(offset + t) * cfg->hidden_size;
for (int d = 0; d < cfg->hidden_size; d++)
emb[d] += row[d];
}
@@ -1511,31 +1523,35 @@ int embed_pool_batch(const embed_config_t *cfg, const float *states,
for (int d = 0; d < cfg->hidden_size; d++)
emb[d] *= inv_len;
}
- if (cfg->normalize_embeddings &&
- embed_l2_normalize(emb, cfg->hidden_size) != 0)
+ if (cfg->normalize_embeddings && embed_l2_normalize(emb, cfg->hidden_size) != 0)
return -1;
offset += len;
}
return 0;
}
-int embed_model_encode_into(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens,
- float *out_embedding)
-{
- embed_input_t input = { token_ids, n_tokens };
+int embed_model_encode_into(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens,
+ float *out_embedding) {
+ embed_input_t input = {token_ids, n_tokens};
return embed_model_encode_batch(model, ws, &input, 1, out_embedding);
}
-float *embed_model_encode(const embed_model_t *model, embed_workspace_t *ws,
- const int *token_ids, int n_tokens)
-{
- if (!model || !ws || !token_ids || n_tokens <= 0) return NULL;
+float *embed_model_encode(const embed_model_t *model,
+ embed_workspace_t *ws,
+ const int *token_ids,
+ int n_tokens) {
+ if (!model || !ws || !token_ids || n_tokens <= 0)
+ return NULL;
int hidden = model->config.hidden_size;
- if (hidden <= 0) return NULL;
+ if (hidden <= 0)
+ return NULL;
float *emb = malloc_floats((size_t)hidden);
- if (!emb) return NULL;
+ if (!emb)
+ return NULL;
if (embed_model_encode_into(model, ws, token_ids, n_tokens, emb) != 0) {
free(emb);
@@ -1548,15 +1564,16 @@ float *embed_model_encode(const embed_model_t *model, embed_workspace_t *ws,
* Vector helpers
* ======================================================================== */
-int embed_l2_normalize(float *vec, int dim)
-{
- if (!vec || dim <= 0) return -1;
+int embed_l2_normalize(float *vec, int dim) {
+ if (!vec || dim <= 0)
+ return -1;
float norm_sq = 0.0f;
for (int i = 0; i < dim; i++)
norm_sq += vec[i] * vec[i];
- if (!(norm_sq > 0.0f) || !isfinite(norm_sq)) return -1;
+ if (!(norm_sq > 0.0f) || !isfinite(norm_sq))
+ return -1;
float inv_norm = 1.0f / sqrtf(norm_sq);
for (int i = 0; i < dim; i++)
@@ -1564,9 +1581,9 @@ int embed_l2_normalize(float *vec, int dim)
return 0;
}
-float embed_cosine_similarity(const float *a, const float *b, int dim)
-{
- if (!a || !b || dim <= 0) return 0.0f;
+float embed_cosine_similarity(const float *a, const float *b, int dim) {
+ if (!a || !b || dim <= 0)
+ return 0.0f;
float dot = 0.0f;
float norm_a_sq = 0.0f;
@@ -1577,8 +1594,7 @@ float embed_cosine_similarity(const float *a, const float *b, int dim)
norm_b_sq += b[i] * b[i];
}
- if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) ||
- !isfinite(norm_a_sq) || !isfinite(norm_b_sq))
+ if (!(norm_a_sq > 0.0f) || !(norm_b_sq > 0.0f) || !isfinite(norm_a_sq) || !isfinite(norm_b_sq))
return 0.0f;
return dot / sqrtf(norm_a_sq * norm_b_sq);
}