diff options
| -rw-r--r-- | src/dtype.h | 19 | ||||
| -rw-r--r-- | src/embed.c | 19 | ||||
| -rw-r--r-- | src/safetensors.c | 10 |
3 files changed, 30 insertions, 18 deletions
diff --git a/src/dtype.h b/src/dtype.h new file mode 100644 index 0000000..5c1ce8f --- /dev/null +++ b/src/dtype.h @@ -0,0 +1,19 @@ +#ifndef EMBED_DTYPE_H +#define EMBED_DTYPE_H + +#include <stdint.h> +#include <string.h> + +/* BF16 is the upper 16 bits of an IEEE-754 binary32, so widening one value is a + * left shift reinterpreted as a float. Shared by every host-side caller (the + * CPU loader, the safetensors reader); the CUDA backend keeps its own + * __nv_bfloat16 device kernels, and kernels.c keeps the SIMD buffer variant + * embed_bf16_to_f32_buf for hot, contiguous conversions. */ +static inline float embed_bf16_to_f32(uint16_t bf16) { + uint32_t bits = (uint32_t)bf16 << 16; + float f; + memcpy(&f, &bits, sizeof(f)); + return f; +} + +#endif /* EMBED_DTYPE_H */ diff --git a/src/embed.c b/src/embed.c index f482c95..9b6ccd9 100644 --- a/src/embed.c +++ b/src/embed.c @@ -1,6 +1,7 @@ #include "embed.h" #include "config.h" #include "internal.h" +#include "dtype.h" #include "kernels.h" #include "safetensors.h" @@ -270,10 +271,8 @@ load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, i } static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) { - for (int i = 0; i < n; i++) { - uint32_t u = ((uint32_t)src[i]) << 16; - memcpy(dst + i, &u, sizeof(float)); - } + for (int i = 0; i < n; i++) + dst[i] = embed_bf16_to_f32(src[i]); } static void copy_weight_row(float *dst, const embed_weight_ref_t *w, size_t row, int row_width) { @@ -336,8 +335,8 @@ static void linear_qkv_weight(embed_workspace_t *ws, if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && wv->dtype == DTYPE_BF16 && seq_len <= EMBED_BF16_QKV_FUSE_MAX_SEQ) { embed_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data, - (const uint16_t *)wk->data, (const uint16_t *)wv->data, seq_len, - in_dim, q_dim, kv_dim); + (const uint16_t *)wk->data, (const uint16_t *)wv->data, + seq_len, in_dim, q_dim, kv_dim); return; } @@ -369,7 +368,7 @@ static void linear_pair_weight(embed_workspace_t *ws, if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 && seq_len <= EMBED_BF16_PAIR_FUSE_MAX_SEQ) { embed_linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data, - (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim); + (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim); return; } @@ -1009,7 +1008,7 @@ int embed_late_maxsim_batch(const float *query_vectors, int group_tokens = doc_offsets[doc] - start; embed_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim, - group_tokens); + group_tokens); for (int i = first; i < doc; i++) { int s0 = doc_offsets[i] - start; @@ -1231,9 +1230,9 @@ static int forward_packed_slice_inplace(const embed_model_t *model, int start = offsets[b]; int len = offsets[b + 1] - start; embed_apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads, - head_dim); + head_dim); embed_apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len, - n_kv_heads, head_dim); + n_kv_heads, head_dim); } model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads, diff --git a/src/safetensors.c b/src/safetensors.c index f12549c..a7c567f 100644 --- a/src/safetensors.c +++ b/src/safetensors.c @@ -1,4 +1,5 @@ #include "safetensors.h" +#include "dtype.h" #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -342,13 +343,6 @@ int64_t safetensor_numel(const safetensor_t *t) { return n; } -static float bf16_to_f32(uint16_t bf16) { - uint32_t f32 = ((uint32_t)bf16) << 16; - float result; - memcpy(&result, &f32, sizeof(float)); - return result; -} - float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) { int64_t n = safetensor_numel(t); if (n <= 0) @@ -367,7 +361,7 @@ float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) case DTYPE_BF16: { const uint16_t *src = (const uint16_t *)data; for (int64_t i = 0; i < n; i++) - out[i] = bf16_to_f32(src[i]); + out[i] = embed_bf16_to_f32(src[i]); break; } default: |
