aboutsummaryrefslogtreecommitdiff
path: root/libffwd/wordpiece.h
blob: f8a63d695ed68af869beb87a037cc0d2881726b5 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
/*
 * BERT-family WordPiece tokenizer.
 *
 * Implements BERT tokenization: BasicTokenizer (cleanup, optional lowercase
 * and accent stripping, CJK spacing, whitespace splitting, and punctuation
 * splitting), then greedy longest-match WordPiece using vocab.txt.
 *
 * Encodes text only. The frontend adds [CLS]/[SEP], just as the byte-level BPE
 * backend leaves terminal and separator tokens to the frontend.
 *
 * Public API mirrors bpe.h: load, encode with reusable workspace, decode,
 * resolve named special tokens, and free. This lets both tokenizers share one
 * tokenizer interface later.
 */

#ifndef TOK_WP_H
#define TOK_WP_H

typedef struct {
    char **id_to_token; /* [vocab_size] vocab.txt entries, line order = id */
    int vocab_size;
    int unk_id;             /* [UNK] id, resolved at load */
    int do_lower_case;      /* lowercase + strip accents before WordPiece */
    int max_chars_per_word; /* words longer than this map to [UNK] (BERT: 100) */

    /* Opaque token-string -> id hash map. */
    void *vocab_map;
    int vocab_map_cap;
} tok_wp_t;

typedef struct tok_wp_workspace tok_wp_workspace_t;

/* Load <model_dir>/vocab.txt and do_lower_case from
 * <model_dir>/tokenizer_config.json; defaults to lowercase.
 * Returns NULL on error: missing vocab.txt, missing [UNK], or allocation failure. */
tok_wp_t *tok_wp_load(const char *model_dir);

/* Decode one id to its vocab token string, or NULL.
 * Returns an internal pointer. */
const char *tok_wp_decode(const tok_wp_t *tok, int id);

/* Resolve a named token (e.g. "[CLS]") to its id, or -1 when absent. */
int tok_wp_token_id(const tok_wp_t *tok, const char *token);

/* Encode UTF-8 text into a malloc'd id array and set *out_n_tokens.
 * Returns NULL on error and sets *out_n_tokens = 0. */
int *tok_wp_encode(const tok_wp_t *tok, const char *text, int *out_n_tokens);

/* Reusable scratch space for repeated encoding with fewer allocations. */
tok_wp_workspace_t *tok_wp_workspace_new(void);
void tok_wp_workspace_free(tok_wp_workspace_t *ws);

/* Encode into caller-provided out_ids[0..out_cap-1].
 * Returns 0 on success, -1 on error, or -2 if out_cap is too small.
 * If non-NULL, *out_n_tokens is always set to tokens needed/emitted. */
int tok_wp_encode_into(const tok_wp_t *tok,
                       tok_wp_workspace_t *ws,
                       const char *text,
                       int *out_ids,
                       int out_cap,
                       int *out_n_tokens);

/* Same tokenization as tok_wp_encode().
 * Reuses ws internally and only allocates the returned int[]. */
int *tok_wp_encode_with_workspace(const tok_wp_t *tok,
                                  tok_wp_workspace_t *ws,
                                  const char *text,
                                  int *out_n_tokens);

void tok_wp_free(tok_wp_t *tok);

#endif /* TOK_WP_H */