aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/dtype.h36
-rw-r--r--src/internal.h4
-rw-r--r--src/late.c10
-rw-r--r--src/mlx.c22
-rw-r--r--src/model.c51
-rw-r--r--src/model_internal.h9
-rw-r--r--src/safetensors.c6
7 files changed, 127 insertions, 11 deletions
diff --git a/src/dtype.h b/src/dtype.h
index 5c1ce8f..2a25cae 100644
--- a/src/dtype.h
+++ b/src/dtype.h
@@ -16,4 +16,40 @@ static inline float embed_bf16_to_f32(uint16_t bf16) {
return f;
}
+/* IEEE-754 binary16 -> binary32. Unlike BF16, F16 has a 5-bit exponent and a
+ * 10-bit mantissa, so widening is a field re-pack, not a shift: rebias the
+ * exponent (15 -> 127), shift the mantissa up by 13, and special-case zeros,
+ * subnormals, and inf/NaN. Used by the loaders to accept F16 safetensors
+ * snapshots (e.g. the public multilingual-e5 weights); there is no fused F16
+ * kernel, so weights are widened to F32 once at load. */
+static inline float embed_f16_to_f32(uint16_t f16) {
+ uint32_t sign = (uint32_t)(f16 & 0x8000u) << 16;
+ uint32_t exp = (f16 >> 10) & 0x1fu;
+ uint32_t mant = f16 & 0x3ffu;
+ uint32_t bits;
+ if (exp == 0) {
+ if (mant == 0) {
+ bits = sign; /* signed zero */
+ } else {
+ /* Subnormal half: renormalize into a normal float. Shift the
+ * mantissa left until its leading 1 reaches the hidden-bit
+ * position, decrementing the exponent for each shift. */
+ exp = 127 - 15 + 1;
+ while ((mant & 0x400u) == 0) {
+ mant <<= 1;
+ exp--;
+ }
+ mant &= 0x3ffu;
+ bits = sign | (exp << 23) | (mant << 13);
+ }
+ } else if (exp == 0x1fu) {
+ bits = sign | 0x7f800000u | (mant << 13); /* inf or NaN */
+ } else {
+ bits = sign | ((exp + (127 - 15)) << 23) | (mant << 13);
+ }
+ float f;
+ memcpy(&f, &bits, sizeof(f));
+ return f;
+}
+
#endif /* EMBED_DTYPE_H */
diff --git a/src/internal.h b/src/internal.h
index b3c2aa0..c1429de 100644
--- a/src/internal.h
+++ b/src/internal.h
@@ -7,6 +7,10 @@
typedef struct {
safetensor_dtype_t dtype;
const void *data;
+ /* 0: data is a borrowed view into the safetensors mmap (F32/BF16, the common
+ * case). 1: data is a heap buffer owned by the model and freed on unload.
+ * Set when an F16 tensor is widened to F32 at load (no fused F16 kernel). */
+ int owned;
} embed_weight_ref_t;
typedef struct {
diff --git a/src/late.c b/src/late.c
index 0829787..6ba52cc 100644
--- a/src/late.c
+++ b/src/late.c
@@ -96,8 +96,10 @@ embed_late_model_t *embed_late_model_load(const char *model_dir) {
if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1))
goto fail;
- late->projection.dtype = t->dtype;
- late->projection.data = safetensors_data(late->dense_sf, t);
+ /* Same F32/BF16-borrow, F16-widen contract as the backbone weights. */
+ late->projection = weight_ref_from_tensor(late->dense_sf, t);
+ if (!late->projection.data)
+ goto fail;
late->token_dim = (int)t->shape[0];
return late;
@@ -110,6 +112,10 @@ void embed_late_model_free(embed_late_model_t *model) {
if (!model)
return;
embed_model_free(model->base);
+ /* Free the projection before the mmap: only an F16-widened ref owns its
+ * buffer; a borrowed F32/BF16 ref points into dense_sf and must not be freed. */
+ if (model->projection.owned)
+ free((void *)model->projection.data);
safetensors_close(model->dense_sf);
free(model);
}
diff --git a/src/mlx.c b/src/mlx.c
index e606b52..4d5abe6 100644
--- a/src/mlx.c
+++ b/src/mlx.c
@@ -1,5 +1,6 @@
#include "mlx.h"
#include "config.h"
+#include "dtype.h"
#include "embed.h"
#include "safetensors.h"
@@ -88,6 +89,7 @@ static size_t safetensor_dtype_size(safetensor_dtype_t dtype) {
case DTYPE_F32:
return sizeof(float);
case DTYPE_BF16:
+ case DTYPE_F16:
return sizeof(uint16_t);
default:
return 0;
@@ -101,7 +103,7 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf,
int ndim) {
size_t elem_size = safetensor_dtype_size(t->dtype);
if (elem_size == 0) {
- fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name);
+ fprintf(stderr, "mlx: expected F32, BF16, or F16 for %s\n", name);
return 0;
}
if (t->ndim != ndim) {
@@ -156,13 +158,29 @@ load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndi
}
if (!mlx_tensor_has_supported_shape(sf, t, name, shape, ndim))
return (mlx_array){0};
+ if (t->dtype == DTYPE_F16) {
+ /* No F16 forward path on MLX (the matmul fast path keys on BF16). Widen
+ * to F32 on the host at load so the whole forward runs in F32, mirroring
+ * the CPU loader. mlx_array_new_data copies its buffer, so the temporary
+ * is freed immediately. Public F16 snapshots load through here. */
+ int64_t n = safetensor_numel(t);
+ float *tmp = (float *)malloc((size_t)n * sizeof(float));
+ if (!tmp)
+ return (mlx_array){0};
+ const uint16_t *src = (const uint16_t *)safetensors_data(sf, t);
+ for (int64_t i = 0; i < n; i++)
+ tmp[i] = embed_f16_to_f32(src[i]);
+ mlx_array a = mlx_array_new_data(tmp, shape, ndim, MLX_FLOAT32);
+ free(tmp);
+ return a;
+ }
mlx_dtype dtype;
if (t->dtype == DTYPE_F32) {
dtype = MLX_FLOAT32;
} else if (t->dtype == DTYPE_BF16) {
dtype = MLX_BFLOAT16;
} else {
- fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name);
+ fprintf(stderr, "mlx: expected F32, BF16, or F16 for %s\n", name);
return (mlx_array){0};
}
return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype);
diff --git a/src/model.c b/src/model.c
index 2956d7f..c39c526 100644
--- a/src/model.c
+++ b/src/model.c
@@ -45,6 +45,7 @@ static size_t dtype_size(safetensor_dtype_t dtype) {
case DTYPE_F32:
return sizeof(float);
case DTYPE_BF16:
+ case DTYPE_F16:
return sizeof(uint16_t);
default:
return 0;
@@ -76,9 +77,12 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
const int64_t *shape,
int ndim,
int bf16_ok) {
- if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) {
+ /* bf16_ok admits either 16-bit float format (BF16 or F16); both are widened
+ * to F32 by the caller (BF16 lazily per matmul, F16 once at load). */
+ int sixteen_ok = bf16_ok && (t->dtype == DTYPE_BF16 || t->dtype == DTYPE_F16);
+ if (t->dtype != DTYPE_F32 && !sixteen_ok) {
fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name,
- dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32");
+ dtype_name(t->dtype), bf16_ok ? "F32, BF16, or F16" : "F32");
return 0;
}
if (t->ndim != ndim) {
@@ -128,9 +132,29 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
return 1;
}
+embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t) {
+ embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0};
+ /* F32/BF16 are consumed directly by the forward pass, so the ref borrows the
+ * mmap (zero copy; BF16 is widened lazily per matmul). F16 has neither a
+ * fused kernel nor a zero-copy F32 view, so widen it once here into an owned
+ * buffer the model frees on unload. DTYPE_F16 never reaches the forward. */
+ if (t->dtype == DTYPE_F16) {
+ float *buf = safetensors_get_f32(sf, t);
+ if (!buf)
+ return ref;
+ ref.dtype = DTYPE_F32;
+ ref.data = buf;
+ ref.owned = 1;
+ } else {
+ ref.dtype = t->dtype;
+ ref.data = safetensors_data(sf, t);
+ }
+ return ref;
+}
+
static embed_weight_ref_t
load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) {
- embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL};
+ embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0};
safetensors_file_t *sf = NULL;
const safetensor_t *t = multi_safetensors_find(ms, name, &sf);
if (!t) {
@@ -140,9 +164,7 @@ load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *sha
if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1))
return ref;
- ref.dtype = t->dtype;
- ref.data = safetensors_data(sf, t);
- return ref;
+ return weight_ref_from_tensor(sf, t);
}
static float *
@@ -414,12 +436,28 @@ embed_model_t *embed_model_load(const char *model_dir) {
return model_load_range_ex(model_dir, 0, -1, 0);
}
+/* Release a weight ref's backing store. Only F16-widened refs own their buffer;
+ * borrowed F32/BF16 views into the mmap are a no-op (see weight_ref_from_tensor). */
+static void free_weight_ref(embed_weight_ref_t *w) {
+ if (w->owned)
+ free((void *)w->data);
+ w->data = NULL;
+ w->owned = 0;
+}
+
void embed_model_free(embed_model_t *model) {
if (!model)
return;
if (model->weights.layers) {
for (int i = 0; i < model->config.n_layers; i++) {
embed_layer_t *l = &model->weights.layers[i];
+ free_weight_ref(&l->wq);
+ free_weight_ref(&l->wk);
+ free_weight_ref(&l->wv);
+ free_weight_ref(&l->wo);
+ free_weight_ref(&l->gate_proj);
+ free_weight_ref(&l->up_proj);
+ free_weight_ref(&l->down_proj);
free((void *)l->q_bias);
free((void *)l->k_bias);
free((void *)l->v_bias);
@@ -434,6 +472,7 @@ void embed_model_free(embed_model_t *model) {
free((void *)l->ffn_ln_bias);
}
}
+ free_weight_ref(&model->weights.embed_tokens);
free((void *)model->weights.norm);
free((void *)model->weights.position_embeddings);
free((void *)model->weights.token_type_embeddings);
diff --git a/src/model_internal.h b/src/model_internal.h
index f61b069..acbba07 100644
--- a/src/model_internal.h
+++ b/src/model_internal.h
@@ -12,7 +12,8 @@
* file. The frontends and the GPU backends do not include this header. */
/* Validate that a safetensor matches an expected dtype/rank/shape and that its
- * bytes lie within the mapped file. bf16_ok permits BF16 in addition to F32. */
+ * bytes lie within the mapped file. bf16_ok permits either 16-bit float format
+ * (BF16 or F16) in addition to F32. */
int tensor_has_supported_shape(const safetensors_file_t *sf,
const safetensor_t *t,
const char *name,
@@ -20,6 +21,12 @@ int tensor_has_supported_shape(const safetensors_file_t *sf,
int ndim,
int bf16_ok);
+/* Build a weight ref from a located tensor: F32/BF16 borrow the mmap, F16 is
+ * widened to an owned F32 buffer (no fused F16 kernel). Shared by the model
+ * loader and the late-interaction projection loader; the owner frees ref.data
+ * when ref.owned is set. Returns ref.data == NULL on allocation failure. */
+embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t);
+
/* True if model_dir carries a 1_Dense projection (a late-interaction model). */
int model_dir_has_late_projection(const char *model_dir);
diff --git a/src/safetensors.c b/src/safetensors.c
index a7c567f..65d0d7c 100644
--- a/src/safetensors.c
+++ b/src/safetensors.c
@@ -364,6 +364,12 @@ float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t)
out[i] = embed_bf16_to_f32(src[i]);
break;
}
+ case DTYPE_F16: {
+ const uint16_t *src = (const uint16_t *)data;
+ for (int64_t i = 0; i < n; i++)
+ out[i] = embed_f16_to_f32(src[i]);
+ break;
+ }
default:
free(out);
return NULL;