1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
|
/* NVIDIA CUDA backend.
* Implements the API in ffwd.h and is linked only into CUDA builds.
*
* The context is owned by the inference thread:
* open records what to load, activate creates the context, and worker_free
* destroys it.
*
* CUDA encodes on the device and scores MaxSim on the host.
*/
#include "internal.h"
#include "cuda.h"
#include "platform.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
struct embed {
int is_late;
char *dir;
ffwd_cuda_ctx_t *ctx;
ffwd_cuda_late_ctx_t *late_ctx;
};
const char *ffwd_capability(void) { return FFWD_BACKEND_LABEL; }
void ffwd_cli_help(FILE *f) {
fputs(" --gpu-gemm-mode MODE\n"
" GPU GEMM compute mode: f32, tf32, bf16, or 16f (default: f32)\n"
" --gpu-weight-dtype DTYPE\n"
" GPU weight storage: f32, bf16, or f16 (default: model dtype).\n"
" bf16/f16 halve weight memory and use 16-bit tensor cores\n",
f);
}
void ffwd_server_help(FILE *f) {
fputs(" --gpu-gemm-mode MODE GPU GEMM compute: f32, tf32, bf16, or 16f\n"
" (default: f32, exact)\n"
" --gpu-weight-dtype DTYPE GPU weight storage: f32, bf16, or f16\n"
" (default: model dtype); 16-bit halves memory\n",
f);
}
int ffwd_init(const ffwd_options_t *opts, char *err, size_t errlen) {
if (opts->gpu_gemm_mode && ffwd_cuda_set_fast_gemm(opts->gpu_gemm_mode) != 0) {
snprintf(err, errlen, "--gpu-gemm-mode must be f32, tf32, bf16, or 16f");
return -1;
}
if (opts->gpu_weight_dtype) {
if (!strcmp(opts->gpu_weight_dtype, "bf16"))
ffwd_cuda_set_weights_bf16(1);
else if (!strcmp(opts->gpu_weight_dtype, "f16") || !strcmp(opts->gpu_weight_dtype, "fp16"))
ffwd_cuda_set_weights_f16(1);
else if (!strcmp(opts->gpu_weight_dtype, "f32"))
ffwd_cuda_set_weights_bf16(0);
else {
snprintf(err, errlen, "--gpu-weight-dtype must be f32, bf16, or f16");
return -1;
}
}
if (ffwd_verbose >= 1)
fprintf(stderr, "Using %s\n", FFWD_BACKEND_LABEL);
return 0;
}
ffwd_t *
ffwd_open(const char *model_dir, int is_late, const ffwd_options_t *opts, char *err, size_t errlen) {
(void)opts;
ffwd_t *b = (ffwd_t *)calloc(1, sizeof(*b));
if (!b || !(b->dir = strdup(model_dir))) {
snprintf(err, errlen, "out of memory");
free(b);
return NULL;
}
b->is_late = is_late;
return b;
}
int ffwd_activate(ffwd_t *b, char *err, size_t errlen) {
if (b->is_late)
b->late_ctx = ffwd_cuda_late_load(b->dir);
else
b->ctx = ffwd_cuda_load(b->dir);
if (!b->ctx && !b->late_ctx) {
snprintf(err, errlen, "failed to load model");
return -1;
}
return 0;
}
void ffwd_worker_free(ffwd_t *b) {
if (!b)
return;
if (b->ctx) {
ffwd_cuda_free(b->ctx);
b->ctx = NULL;
}
if (b->late_ctx) {
ffwd_cuda_late_free(b->late_ctx);
b->late_ctx = NULL;
}
}
void ffwd_free(ffwd_t *b) {
if (!b)
return;
ffwd_worker_free(b);
free(b->dir);
free(b);
}
const ffwd_config_t *ffwd_config(const ffwd_t *b) {
return b->is_late ? ffwd_cuda_late_config(b->late_ctx) : ffwd_cuda_config(b->ctx);
}
int ffwd_token_dim(const ffwd_t *b) { return b->is_late ? ffwd_cuda_late_token_dim(b->late_ctx) : 0; }
int ffwd_uses_dense_batches(const ffwd_t *b) {
(void)b;
return 0;
}
int ffwd_encode_batch(ffwd_t *b, const ffwd_input_t *inputs, int batch, float *out) {
return ffwd_cuda_encode_batch(b->ctx, inputs, batch, out);
}
int ffwd_encode_spans_batch(ffwd_t *b, const ffwd_context_input_t *inputs, int batch, float *out) {
return ffwd_cuda_encode_spans_batch(b->ctx, inputs, batch, out);
}
int ffwd_rerank(ffwd_t *b, const ffwd_rerank_input_t *in, float *scores) {
int dim = ffwd_cuda_late_token_dim(b->late_ctx);
int total_doc_vecs = 0;
for (int i = 0; i < in->n_docs; i++)
total_doc_vecs += in->doc_n_keep[i];
float *query = (float *)malloc((size_t)in->query_n_tokens * dim * sizeof(*query));
float *docs = (float *)malloc((size_t)total_doc_vecs * dim * sizeof(*docs));
int *offsets = (int *)malloc((size_t)(in->n_docs + 1) * sizeof(*offsets));
int rc = (query && docs && offsets) ? 0 : -1;
/* GPU encodes the query and every candidate in one packed forward; MaxSim
* runs on the host, where the grouped-GEMM scorer beats a device graph. */
if (rc == 0)
rc = ffwd_cuda_late_encode_tokens(b->late_ctx, in->query_ids, in->query_n_tokens, 1, query);
if (rc == 0)
rc = ffwd_cuda_late_encode_docs(b->late_ctx, in->doc_ids, in->doc_n_tokens, in->doc_keep,
in->doc_n_keep, in->n_docs, 1, docs, offsets);
if (rc == 0)
rc = ffwd_late_maxsim_batch(query, in->query_n_keep, docs, offsets, in->n_docs, dim, scores);
free(offsets);
free(docs);
free(query);
return rc;
}
size_t ffwd_scratch_nbytes(const ffwd_t *b) {
(void)b;
return 0;
}
int ffwd_default_batch_wait_us(void) { return 1000; }
int ffwd_preflight(
const char *const *paths, int n_paths, const ffwd_options_t *opts, char *err, size_t errlen) {
(void)paths;
(void)n_paths;
(void)opts;
(void)err;
(void)errlen;
return 0;
}
|