diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-23 01:07:04 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-23 01:07:04 +0200 |
| commit | 338ee4bf525175044df39add6eb559e5cc76edf6 (patch) | |
| tree | c822a844a03333827a47436374284213768ab8ba /libffwd/backend_cuda.c | |
| parent | 99c090b8538b8b93f98e560b752c8c78bdfeb507 (diff) | |
| download | ffwd-338ee4bf525175044df39add6eb559e5cc76edf6.tar.gz ffwd-338ee4bf525175044df39add6eb559e5cc76edf6.zip | |
cuda: add bert ffn16 intermediate path
Diffstat (limited to 'libffwd/backend_cuda.c')
| -rw-r--r-- | libffwd/backend_cuda.c | 12 |
1 files changed, 7 insertions, 5 deletions
diff --git a/libffwd/backend_cuda.c b/libffwd/backend_cuda.c index 2635fb2..dd05c16 100644 --- a/libffwd/backend_cuda.c +++ b/libffwd/backend_cuda.c @@ -23,16 +23,16 @@ void ffwd_cli_help(FILE *f) { fputs(" --gpu-gemm-mode MODE\n" " GPU GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n" " --gpu-weight-dtype DTYPE\n" - " GPU weight storage: f32 or bf16 (default: model dtype).\n" - " bf16 halves weight memory and uses BF16 tensor cores\n", + " GPU weight storage: f32, bf16, or f16 (default: model dtype).\n" + " bf16/f16 halve weight memory and use 16-bit tensor cores\n", f); } void ffwd_server_help(FILE *f) { fputs(" --gpu-gemm-mode MODE GPU GEMM compute: f32, tf32, bf16, or 16f\n" " (default: f32, exact)\n" - " --gpu-weight-dtype DTYPE GPU weight storage: f32 or bf16 (default:\n" - " model dtype); bf16 halves weight memory\n", + " --gpu-weight-dtype DTYPE GPU weight storage: f32, bf16, or f16\n" + " (default: model dtype); 16-bit halves memory\n", f); } @@ -44,10 +44,12 @@ int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) { if (opts->gpu_weight_dtype) { if (!strcmp(opts->gpu_weight_dtype, "bf16")) ffwd_cuda_set_weights_bf16(1); + else if (!strcmp(opts->gpu_weight_dtype, "f16") || !strcmp(opts->gpu_weight_dtype, "fp16")) + ffwd_cuda_set_weights_f16(1); else if (!strcmp(opts->gpu_weight_dtype, "f32")) ffwd_cuda_set_weights_bf16(0); else { - snprintf(err, errlen, "--gpu-weight-dtype must be f32 or bf16"); + snprintf(err, errlen, "--gpu-weight-dtype must be f32, bf16, or f16"); return -1; } } |
