aboutsummaryrefslogtreecommitdiff
path: root/libffwd/backend_cuda.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-23 01:07:04 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-23 01:07:04 +0200
commit338ee4bf525175044df39add6eb559e5cc76edf6 (patch)
treec822a844a03333827a47436374284213768ab8ba /libffwd/backend_cuda.c
parent99c090b8538b8b93f98e560b752c8c78bdfeb507 (diff)
downloadffwd-338ee4bf525175044df39add6eb559e5cc76edf6.tar.gz
ffwd-338ee4bf525175044df39add6eb559e5cc76edf6.zip
cuda: add bert ffn16 intermediate path
Diffstat (limited to 'libffwd/backend_cuda.c')
-rw-r--r--libffwd/backend_cuda.c12
1 files changed, 7 insertions, 5 deletions
diff --git a/libffwd/backend_cuda.c b/libffwd/backend_cuda.c
index 2635fb2..dd05c16 100644
--- a/libffwd/backend_cuda.c
+++ b/libffwd/backend_cuda.c
@@ -23,16 +23,16 @@ void ffwd_cli_help(FILE *f) {
fputs(" --gpu-gemm-mode MODE\n"
" GPU GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
" --gpu-weight-dtype DTYPE\n"
- " GPU weight storage: f32 or bf16 (default: model dtype).\n"
- " bf16 halves weight memory and uses BF16 tensor cores\n",
+ " GPU weight storage: f32, bf16, or f16 (default: model dtype).\n"
+ " bf16/f16 halve weight memory and use 16-bit tensor cores\n",
f);
}
void ffwd_server_help(FILE *f) {
fputs(" --gpu-gemm-mode MODE GPU GEMM compute: f32, tf32, bf16, or 16f\n"
" (default: f32, exact)\n"
- " --gpu-weight-dtype DTYPE GPU weight storage: f32 or bf16 (default:\n"
- " model dtype); bf16 halves weight memory\n",
+ " --gpu-weight-dtype DTYPE GPU weight storage: f32, bf16, or f16\n"
+ " (default: model dtype); 16-bit halves memory\n",
f);
}
@@ -44,10 +44,12 @@ int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) {
if (opts->gpu_weight_dtype) {
if (!strcmp(opts->gpu_weight_dtype, "bf16"))
ffwd_cuda_set_weights_bf16(1);
+ else if (!strcmp(opts->gpu_weight_dtype, "f16") || !strcmp(opts->gpu_weight_dtype, "fp16"))
+ ffwd_cuda_set_weights_f16(1);
else if (!strcmp(opts->gpu_weight_dtype, "f32"))
ffwd_cuda_set_weights_bf16(0);
else {
- snprintf(err, errlen, "--gpu-weight-dtype must be f32 or bf16");
+ snprintf(err, errlen, "--gpu-weight-dtype must be f32, bf16, or f16");
return -1;
}
}