diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/dtype.h | 36 | ||||
| -rw-r--r-- | src/internal.h | 4 | ||||
| -rw-r--r-- | src/late.c | 10 | ||||
| -rw-r--r-- | src/mlx.c | 22 | ||||
| -rw-r--r-- | src/model.c | 51 | ||||
| -rw-r--r-- | src/model_internal.h | 9 | ||||
| -rw-r--r-- | src/safetensors.c | 6 |
7 files changed, 127 insertions, 11 deletions
diff --git a/src/dtype.h b/src/dtype.h index 5c1ce8f..2a25cae 100644 --- a/src/dtype.h +++ b/src/dtype.h @@ -16,4 +16,40 @@ static inline float embed_bf16_to_f32(uint16_t bf16) { return f; } +/* IEEE-754 binary16 -> binary32. Unlike BF16, F16 has a 5-bit exponent and a + * 10-bit mantissa, so widening is a field re-pack, not a shift: rebias the + * exponent (15 -> 127), shift the mantissa up by 13, and special-case zeros, + * subnormals, and inf/NaN. Used by the loaders to accept F16 safetensors + * snapshots (e.g. the public multilingual-e5 weights); there is no fused F16 + * kernel, so weights are widened to F32 once at load. */ +static inline float embed_f16_to_f32(uint16_t f16) { + uint32_t sign = (uint32_t)(f16 & 0x8000u) << 16; + uint32_t exp = (f16 >> 10) & 0x1fu; + uint32_t mant = f16 & 0x3ffu; + uint32_t bits; + if (exp == 0) { + if (mant == 0) { + bits = sign; /* signed zero */ + } else { + /* Subnormal half: renormalize into a normal float. Shift the + * mantissa left until its leading 1 reaches the hidden-bit + * position, decrementing the exponent for each shift. */ + exp = 127 - 15 + 1; + while ((mant & 0x400u) == 0) { + mant <<= 1; + exp--; + } + mant &= 0x3ffu; + bits = sign | (exp << 23) | (mant << 13); + } + } else if (exp == 0x1fu) { + bits = sign | 0x7f800000u | (mant << 13); /* inf or NaN */ + } else { + bits = sign | ((exp + (127 - 15)) << 23) | (mant << 13); + } + float f; + memcpy(&f, &bits, sizeof(f)); + return f; +} + #endif /* EMBED_DTYPE_H */ diff --git a/src/internal.h b/src/internal.h index b3c2aa0..c1429de 100644 --- a/src/internal.h +++ b/src/internal.h @@ -7,6 +7,10 @@ typedef struct { safetensor_dtype_t dtype; const void *data; + /* 0: data is a borrowed view into the safetensors mmap (F32/BF16, the common + * case). 1: data is a heap buffer owned by the model and freed on unload. + * Set when an F16 tensor is widened to F32 at load (no fused F16 kernel). */ + int owned; } embed_weight_ref_t; typedef struct { @@ -96,8 +96,10 @@ embed_late_model_t *embed_late_model_load(const char *model_dir) { if (!tensor_has_supported_shape(late->dense_sf, t, "linear.weight", shape, 2, 1)) goto fail; - late->projection.dtype = t->dtype; - late->projection.data = safetensors_data(late->dense_sf, t); + /* Same F32/BF16-borrow, F16-widen contract as the backbone weights. */ + late->projection = weight_ref_from_tensor(late->dense_sf, t); + if (!late->projection.data) + goto fail; late->token_dim = (int)t->shape[0]; return late; @@ -110,6 +112,10 @@ void embed_late_model_free(embed_late_model_t *model) { if (!model) return; embed_model_free(model->base); + /* Free the projection before the mmap: only an F16-widened ref owns its + * buffer; a borrowed F32/BF16 ref points into dense_sf and must not be freed. */ + if (model->projection.owned) + free((void *)model->projection.data); safetensors_close(model->dense_sf); free(model); } @@ -1,5 +1,6 @@ #include "mlx.h" #include "config.h" +#include "dtype.h" #include "embed.h" #include "safetensors.h" @@ -88,6 +89,7 @@ static size_t safetensor_dtype_size(safetensor_dtype_t dtype) { case DTYPE_F32: return sizeof(float); case DTYPE_BF16: + case DTYPE_F16: return sizeof(uint16_t); default: return 0; @@ -101,7 +103,7 @@ static int mlx_tensor_has_supported_shape(const safetensors_file_t *sf, int ndim) { size_t elem_size = safetensor_dtype_size(t->dtype); if (elem_size == 0) { - fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); + fprintf(stderr, "mlx: expected F32, BF16, or F16 for %s\n", name); return 0; } if (t->ndim != ndim) { @@ -156,13 +158,29 @@ load_tensor(multi_safetensors_t *ms, const char *name, const int *shape, int ndi } if (!mlx_tensor_has_supported_shape(sf, t, name, shape, ndim)) return (mlx_array){0}; + if (t->dtype == DTYPE_F16) { + /* No F16 forward path on MLX (the matmul fast path keys on BF16). Widen + * to F32 on the host at load so the whole forward runs in F32, mirroring + * the CPU loader. mlx_array_new_data copies its buffer, so the temporary + * is freed immediately. Public F16 snapshots load through here. */ + int64_t n = safetensor_numel(t); + float *tmp = (float *)malloc((size_t)n * sizeof(float)); + if (!tmp) + return (mlx_array){0}; + const uint16_t *src = (const uint16_t *)safetensors_data(sf, t); + for (int64_t i = 0; i < n; i++) + tmp[i] = embed_f16_to_f32(src[i]); + mlx_array a = mlx_array_new_data(tmp, shape, ndim, MLX_FLOAT32); + free(tmp); + return a; + } mlx_dtype dtype; if (t->dtype == DTYPE_F32) { dtype = MLX_FLOAT32; } else if (t->dtype == DTYPE_BF16) { dtype = MLX_BFLOAT16; } else { - fprintf(stderr, "mlx: expected F32 or BF16 for %s\n", name); + fprintf(stderr, "mlx: expected F32, BF16, or F16 for %s\n", name); return (mlx_array){0}; } return mlx_array_new_data(safetensors_data(sf, t), shape, ndim, dtype); diff --git a/src/model.c b/src/model.c index 2956d7f..c39c526 100644 --- a/src/model.c +++ b/src/model.c @@ -45,6 +45,7 @@ static size_t dtype_size(safetensor_dtype_t dtype) { case DTYPE_F32: return sizeof(float); case DTYPE_BF16: + case DTYPE_F16: return sizeof(uint16_t); default: return 0; @@ -76,9 +77,12 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, const int64_t *shape, int ndim, int bf16_ok) { - if (t->dtype != DTYPE_F32 && !(bf16_ok && t->dtype == DTYPE_BF16)) { + /* bf16_ok admits either 16-bit float format (BF16 or F16); both are widened + * to F32 by the caller (BF16 lazily per matmul, F16 once at load). */ + int sixteen_ok = bf16_ok && (t->dtype == DTYPE_BF16 || t->dtype == DTYPE_F16); + if (t->dtype != DTYPE_F32 && !sixteen_ok) { fprintf(stderr, "embed: unsupported dtype for %s: got %s, expected %s\n", name, - dtype_name(t->dtype), bf16_ok ? "F32 or BF16" : "F32"); + dtype_name(t->dtype), bf16_ok ? "F32, BF16, or F16" : "F32"); return 0; } if (t->ndim != ndim) { @@ -128,9 +132,29 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, return 1; } +embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t) { + embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0}; + /* F32/BF16 are consumed directly by the forward pass, so the ref borrows the + * mmap (zero copy; BF16 is widened lazily per matmul). F16 has neither a + * fused kernel nor a zero-copy F32 view, so widen it once here into an owned + * buffer the model frees on unload. DTYPE_F16 never reaches the forward. */ + if (t->dtype == DTYPE_F16) { + float *buf = safetensors_get_f32(sf, t); + if (!buf) + return ref; + ref.dtype = DTYPE_F32; + ref.data = buf; + ref.owned = 1; + } else { + ref.dtype = t->dtype; + ref.data = safetensors_data(sf, t); + } + return ref; +} + static embed_weight_ref_t load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *shape, int ndim) { - embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL}; + embed_weight_ref_t ref = {DTYPE_UNKNOWN, NULL, 0}; safetensors_file_t *sf = NULL; const safetensor_t *t = multi_safetensors_find(ms, name, &sf); if (!t) { @@ -140,9 +164,7 @@ load_weight_direct(multi_safetensors_t *ms, const char *name, const int64_t *sha if (!tensor_has_supported_shape(sf, t, name, shape, ndim, 1)) return ref; - ref.dtype = t->dtype; - ref.data = safetensors_data(sf, t); - return ref; + return weight_ref_from_tensor(sf, t); } static float * @@ -414,12 +436,28 @@ embed_model_t *embed_model_load(const char *model_dir) { return model_load_range_ex(model_dir, 0, -1, 0); } +/* Release a weight ref's backing store. Only F16-widened refs own their buffer; + * borrowed F32/BF16 views into the mmap are a no-op (see weight_ref_from_tensor). */ +static void free_weight_ref(embed_weight_ref_t *w) { + if (w->owned) + free((void *)w->data); + w->data = NULL; + w->owned = 0; +} + void embed_model_free(embed_model_t *model) { if (!model) return; if (model->weights.layers) { for (int i = 0; i < model->config.n_layers; i++) { embed_layer_t *l = &model->weights.layers[i]; + free_weight_ref(&l->wq); + free_weight_ref(&l->wk); + free_weight_ref(&l->wv); + free_weight_ref(&l->wo); + free_weight_ref(&l->gate_proj); + free_weight_ref(&l->up_proj); + free_weight_ref(&l->down_proj); free((void *)l->q_bias); free((void *)l->k_bias); free((void *)l->v_bias); @@ -434,6 +472,7 @@ void embed_model_free(embed_model_t *model) { free((void *)l->ffn_ln_bias); } } + free_weight_ref(&model->weights.embed_tokens); free((void *)model->weights.norm); free((void *)model->weights.position_embeddings); free((void *)model->weights.token_type_embeddings); diff --git a/src/model_internal.h b/src/model_internal.h index f61b069..acbba07 100644 --- a/src/model_internal.h +++ b/src/model_internal.h @@ -12,7 +12,8 @@ * file. The frontends and the GPU backends do not include this header. */ /* Validate that a safetensor matches an expected dtype/rank/shape and that its - * bytes lie within the mapped file. bf16_ok permits BF16 in addition to F32. */ + * bytes lie within the mapped file. bf16_ok permits either 16-bit float format + * (BF16 or F16) in addition to F32. */ int tensor_has_supported_shape(const safetensors_file_t *sf, const safetensor_t *t, const char *name, @@ -20,6 +21,12 @@ int tensor_has_supported_shape(const safetensors_file_t *sf, int ndim, int bf16_ok); +/* Build a weight ref from a located tensor: F32/BF16 borrow the mmap, F16 is + * widened to an owned F32 buffer (no fused F16 kernel). Shared by the model + * loader and the late-interaction projection loader; the owner frees ref.data + * when ref.owned is set. Returns ref.data == NULL on allocation failure. */ +embed_weight_ref_t weight_ref_from_tensor(const safetensors_file_t *sf, const safetensor_t *t); + /* True if model_dir carries a 1_Dense projection (a late-interaction model). */ int model_dir_has_late_projection(const char *model_dir); diff --git a/src/safetensors.c b/src/safetensors.c index a7c567f..65d0d7c 100644 --- a/src/safetensors.c +++ b/src/safetensors.c @@ -364,6 +364,12 @@ float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) out[i] = embed_bf16_to_f32(src[i]); break; } + case DTYPE_F16: { + const uint16_t *src = (const uint16_t *)data; + for (int64_t i = 0; i < n; i++) + out[i] = embed_f16_to_f32(src[i]); + break; + } default: free(out); return NULL; |
