aboutsummaryrefslogtreecommitdiff
path: root/bench/bench_tokenizer.c
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-25 17:58:46 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-25 17:58:46 +0200
commitf7ef995ff54da671f9d7665b3d92ff25934c1e8b (patch)
tree1401d04c0f464b32bffd63f787a63a82c6959e97 /bench/bench_tokenizer.c
parent417f861f3661c2253258b5683a627d961e81fc68 (diff)
downloadffwd-f7ef995ff54da671f9d7665b3d92ff25934c1e8b.tar.gz
ffwd-f7ef995ff54da671f9d7665b3d92ff25934c1e8b.zip
chore: run clang-format
Diffstat (limited to 'bench/bench_tokenizer.c')
-rw-r--r--bench/bench_tokenizer.c28
1 files changed, 13 insertions, 15 deletions
diff --git a/bench/bench_tokenizer.c b/bench/bench_tokenizer.c
index 9bb8822..a5a16ee 100644
--- a/bench/bench_tokenizer.c
+++ b/bench/bench_tokenizer.c
@@ -39,9 +39,7 @@ typedef struct {
void *ws;
} bench_tok_t;
-static int path_exists(const char *path) {
- return access(path, R_OK) == 0;
-}
+static int path_exists(const char *path) { return access(path, R_OK) == 0; }
static int load_tokenizer(bench_tok_t *bt, const char *path) {
char p[1024];
@@ -88,28 +86,28 @@ static int *tok_encode(bench_tok_t *bt, const char *text, int *n) {
static int *tok_encode_ws(bench_tok_t *bt, const char *text, int *n) {
switch (bt->kind) {
case TOK_WORDPIECE:
- return tok_wp_encode_with_workspace((tok_wp_t *)bt->tok,
- (tok_wp_workspace_t *)bt->ws, text, n);
+ return tok_wp_encode_with_workspace((tok_wp_t *)bt->tok, (tok_wp_workspace_t *)bt->ws, text,
+ n);
case TOK_SENTENCEPIECE:
- return tok_spm_encode_with_workspace(
- (tok_spm_t *)bt->tok, (tok_spm_workspace_t *)bt->ws, text, n);
+ return tok_spm_encode_with_workspace((tok_spm_t *)bt->tok, (tok_spm_workspace_t *)bt->ws,
+ text, n);
default:
- return tok_bpe_encode_with_workspace((tok_bpe_t *)bt->tok,
- (tok_bpe_workspace_t *)bt->ws, text, n);
+ return tok_bpe_encode_with_workspace((tok_bpe_t *)bt->tok, (tok_bpe_workspace_t *)bt->ws,
+ text, n);
}
}
static int tok_encode_into(bench_tok_t *bt, const char *text, int *out, int cap, int *n) {
switch (bt->kind) {
case TOK_WORDPIECE:
- return tok_wp_encode_into((tok_wp_t *)bt->tok,
- (tok_wp_workspace_t *)bt->ws, text, out, cap, n);
+ return tok_wp_encode_into((tok_wp_t *)bt->tok, (tok_wp_workspace_t *)bt->ws, text, out, cap,
+ n);
case TOK_SENTENCEPIECE:
- return tok_spm_encode_into((tok_spm_t *)bt->tok,
- (tok_spm_workspace_t *)bt->ws, text, out, cap, n);
+ return tok_spm_encode_into((tok_spm_t *)bt->tok, (tok_spm_workspace_t *)bt->ws, text, out,
+ cap, n);
default:
- return tok_bpe_encode_into((tok_bpe_t *)bt->tok,
- (tok_bpe_workspace_t *)bt->ws, text, out, cap, n);
+ return tok_bpe_encode_into((tok_bpe_t *)bt->tok, (tok_bpe_workspace_t *)bt->ws, text, out,
+ cap, n);
}
}