aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-10 09:03:22 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:03 +0200
commitfbdc3ae0cc650f7f7e9a2e81e63c3731651b8271 (patch)
tree270e244bdcd09ed2814e2c251f99641da3e1a19c
parent7eaec3a6d01bd598215edd4d6b466f0bf8cec24a (diff)
downloadffwd-fbdc3ae0cc650f7f7e9a2e81e63c3731651b8271.tar.gz
ffwd-fbdc3ae0cc650f7f7e9a2e81e63c3731651b8271.zip
cuda: follow model file dtype for weight storage by default
-rw-r--r--embed_cuda.cu16
1 files changed, 15 insertions, 1 deletions
diff --git a/embed_cuda.cu b/embed_cuda.cu
index c1406b4..e0ff898 100644
--- a/embed_cuda.cu
+++ b/embed_cuda.cu
@@ -144,12 +144,16 @@ static uint16_t f32_to_bf16(float f)
}
// Store weights as native BF16 on the device when set (halves weight memory and
-// feeds BF16 tensor cores); default keeps exact F32. Read at load time.
+// feeds BF16 tensor cores); default keeps exact F32 for F32 model files. When
+// the flag was never set explicitly, the loader follows the model file dtype,
+// matching the CPU and MLX backends.
static int g_weights_bf16 = 0;
+static int g_weights_bf16_set = 0;
int pplx_cuda_set_weights_bf16(int on)
{
g_weights_bf16 = on ? 1 : 0;
+ g_weights_bf16_set = 1;
return 0;
}
@@ -839,6 +843,16 @@ pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir)
ctx->cpu = cpu;
ctx->config = cpu->config;
+ /* Without an explicit --cuda-weight-dtype, store weights in the model
+ * file's dtype: BF16 snapshots load as BF16 (bit-exact pass-through),
+ * F32 snapshots keep the exact F32 default. */
+ if (!g_weights_bf16_set) {
+ g_weights_bf16 = cpu->weights.embed_tokens.dtype == DTYPE_BF16;
+ if (g_weights_bf16)
+ fprintf(stderr, "cuda: BF16 model file; storing weights as BF16 "
+ "(use --cuda-weight-dtype f32 to override)\n");
+ }
+
if (cudaStreamCreate(&ctx->stream) != cudaSuccess ||
cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS ||
cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) {