diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-10 09:03:22 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-14 08:27:03 +0200 |
| commit | fbdc3ae0cc650f7f7e9a2e81e63c3731651b8271 (patch) | |
| tree | 270e244bdcd09ed2814e2c251f99641da3e1a19c | |
| parent | 7eaec3a6d01bd598215edd4d6b466f0bf8cec24a (diff) | |
| download | ffwd-fbdc3ae0cc650f7f7e9a2e81e63c3731651b8271.tar.gz ffwd-fbdc3ae0cc650f7f7e9a2e81e63c3731651b8271.zip | |
cuda: follow model file dtype for weight storage by default
| -rw-r--r-- | embed_cuda.cu | 16 |
1 files changed, 15 insertions, 1 deletions
diff --git a/embed_cuda.cu b/embed_cuda.cu index c1406b4..e0ff898 100644 --- a/embed_cuda.cu +++ b/embed_cuda.cu @@ -144,12 +144,16 @@ static uint16_t f32_to_bf16(float f) } // Store weights as native BF16 on the device when set (halves weight memory and -// feeds BF16 tensor cores); default keeps exact F32. Read at load time. +// feeds BF16 tensor cores); default keeps exact F32 for F32 model files. When +// the flag was never set explicitly, the loader follows the model file dtype, +// matching the CPU and MLX backends. static int g_weights_bf16 = 0; +static int g_weights_bf16_set = 0; int pplx_cuda_set_weights_bf16(int on) { g_weights_bf16 = on ? 1 : 0; + g_weights_bf16_set = 1; return 0; } @@ -839,6 +843,16 @@ pplx_cuda_ctx_t *pplx_cuda_load(const char *model_dir) ctx->cpu = cpu; ctx->config = cpu->config; + /* Without an explicit --cuda-weight-dtype, store weights in the model + * file's dtype: BF16 snapshots load as BF16 (bit-exact pass-through), + * F32 snapshots keep the exact F32 default. */ + if (!g_weights_bf16_set) { + g_weights_bf16 = cpu->weights.embed_tokens.dtype == DTYPE_BF16; + if (g_weights_bf16) + fprintf(stderr, "cuda: BF16 model file; storing weights as BF16 " + "(use --cuda-weight-dtype f32 to override)\n"); + } + if (cudaStreamCreate(&ctx->stream) != cudaSuccess || cublasCreate(&ctx->blas) != CUBLAS_STATUS_SUCCESS || cublasSetStream(ctx->blas, ctx->stream) != CUBLAS_STATUS_SUCCESS) { |
