aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--src/dtype.h19
-rw-r--r--src/embed.c19
-rw-r--r--src/safetensors.c10
3 files changed, 30 insertions, 18 deletions
diff --git a/src/dtype.h b/src/dtype.h
new file mode 100644
index 0000000..5c1ce8f
--- /dev/null
+++ b/src/dtype.h
@@ -0,0 +1,19 @@
+#ifndef EMBED_DTYPE_H
+#define EMBED_DTYPE_H
+
+#include <stdint.h>
+#include <string.h>
+
+/* BF16 is the upper 16 bits of an IEEE-754 binary32, so widening one value is a
+ * left shift reinterpreted as a float. Shared by every host-side caller (the
+ * CPU loader, the safetensors reader); the CUDA backend keeps its own
+ * __nv_bfloat16 device kernels, and kernels.c keeps the SIMD buffer variant
+ * embed_bf16_to_f32_buf for hot, contiguous conversions. */
+static inline float embed_bf16_to_f32(uint16_t bf16) {
+ uint32_t bits = (uint32_t)bf16 << 16;
+ float f;
+ memcpy(&f, &bits, sizeof(f));
+ return f;
+}
+
+#endif /* EMBED_DTYPE_H */
diff --git a/src/embed.c b/src/embed.c
index f482c95..9b6ccd9 100644
--- a/src/embed.c
+++ b/src/embed.c
@@ -1,6 +1,7 @@
#include "embed.h"
#include "config.h"
#include "internal.h"
+#include "dtype.h"
#include "kernels.h"
#include "safetensors.h"
@@ -270,10 +271,8 @@ load_norm_f32(multi_safetensors_t *ms, const char *name, const int64_t *shape, i
}
static void bf16_row_to_f32(float *dst, const uint16_t *src, int n) {
- for (int i = 0; i < n; i++) {
- uint32_t u = ((uint32_t)src[i]) << 16;
- memcpy(dst + i, &u, sizeof(float));
- }
+ for (int i = 0; i < n; i++)
+ dst[i] = embed_bf16_to_f32(src[i]);
}
static void copy_weight_row(float *dst, const embed_weight_ref_t *w, size_t row, int row_width) {
@@ -336,8 +335,8 @@ static void linear_qkv_weight(embed_workspace_t *ws,
if (wq->dtype == DTYPE_BF16 && wk->dtype == DTYPE_BF16 && wv->dtype == DTYPE_BF16 &&
seq_len <= EMBED_BF16_QKV_FUSE_MAX_SEQ) {
embed_linear_nobias_bf16_qkv(q, k, v, x, (const uint16_t *)wq->data,
- (const uint16_t *)wk->data, (const uint16_t *)wv->data, seq_len,
- in_dim, q_dim, kv_dim);
+ (const uint16_t *)wk->data, (const uint16_t *)wv->data,
+ seq_len, in_dim, q_dim, kv_dim);
return;
}
@@ -369,7 +368,7 @@ static void linear_pair_weight(embed_workspace_t *ws,
if (wa->dtype == DTYPE_BF16 && wb->dtype == DTYPE_BF16 &&
seq_len <= EMBED_BF16_PAIR_FUSE_MAX_SEQ) {
embed_linear_nobias_bf16_pair(a, b, x, (const uint16_t *)wa->data,
- (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim);
+ (const uint16_t *)wb->data, seq_len, in_dim, a_dim, b_dim);
return;
}
@@ -1009,7 +1008,7 @@ int embed_late_maxsim_batch(const float *query_vectors,
int group_tokens = doc_offsets[doc] - start;
embed_matmul_t(sim, query_vectors, doc_vectors + (size_t)start * dim, query_tokens, dim,
- group_tokens);
+ group_tokens);
for (int i = first; i < doc; i++) {
int s0 = doc_offsets[i] - start;
@@ -1231,9 +1230,9 @@ static int forward_packed_slice_inplace(const embed_model_t *model,
int start = offsets[b];
int len = offsets[b + 1] - start;
embed_apply_rope_neox(q_buf + (size_t)start * q_dim, rope_cos, rope_sin, len, n_heads,
- head_dim);
+ head_dim);
embed_apply_rope_neox(k_buf + (size_t)start * kv_dim, rope_cos, rope_sin, len,
- n_kv_heads, head_dim);
+ n_kv_heads, head_dim);
}
model->attention(attn_out, q_buf, k_buf, v_buf, offsets, batch, n_heads, n_kv_heads,
diff --git a/src/safetensors.c b/src/safetensors.c
index f12549c..a7c567f 100644
--- a/src/safetensors.c
+++ b/src/safetensors.c
@@ -1,4 +1,5 @@
#include "safetensors.h"
+#include "dtype.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
@@ -342,13 +343,6 @@ int64_t safetensor_numel(const safetensor_t *t) {
return n;
}
-static float bf16_to_f32(uint16_t bf16) {
- uint32_t f32 = ((uint32_t)bf16) << 16;
- float result;
- memcpy(&result, &f32, sizeof(float));
- return result;
-}
-
float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t) {
int64_t n = safetensor_numel(t);
if (n <= 0)
@@ -367,7 +361,7 @@ float *safetensors_get_f32(const safetensors_file_t *sf, const safetensor_t *t)
case DTYPE_BF16: {
const uint16_t *src = (const uint16_t *)data;
for (int64_t i = 0; i < n; i++)
- out[i] = bf16_to_f32(src[i]);
+ out[i] = embed_bf16_to_f32(src[i]);
break;
}
default: