aboutsummaryrefslogtreecommitdiff
path: root/liblangdet/sentencepiece.h
blob: 0d8cf445570f392dc4406fd750bf4e15d69ccc50 (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
/*
 * XLM-R SentencePiece Unigram tokenizer.
 * Encodes text using the checkpoint's precompiled normalization table.
 * The caller adds the leading <s> and trailing </s> tokens.
 */

#ifndef TOK_SPM_H
#define TOK_SPM_H

#include <stddef.h>
#include <stdint.h>

enum {
    TOK_XLMR_BOS_ID = 0,
    TOK_XLMR_EOS_ID = 2,
    TOK_XLMR_UNK_ID = 3,
};

typedef struct {
    char **id_to_piece;   /* [piece_count] raw SentencePiece ids */
    uint16_t *piece_lens; /* byte length for id_to_piece entries */
    float *scores;        /* SentencePiece score per raw id */
    int piece_count;      /* raw SentencePiece vocabulary size */
    int max_piece_len;    /* longest normal piece in bytes */
    double unknown_score; /* score assigned to an unmatched UTF-8 codepoint */

    void *piece_map; /* piece bytes -> raw SentencePiece id */
    int piece_map_cap;

    uint32_t *xcda;
    size_t xcda_count;
    char *prefix_replacements;
    size_t prefix_replacements_len;
} tok_spm_t;

typedef struct tok_spm_workspace tok_spm_workspace_t;

/* Load model_dir/sentencepiece.bpe.model with XLM-R vocabulary and normalization.
 * Return NULL on missing or unsupported model, malformed protobuf, or OOM. */
tok_spm_t *tok_spm_load(const char *model_dir);

tok_spm_workspace_t *tok_spm_workspace_new(void);
void tok_spm_workspace_free(tok_spm_workspace_t *ws);

/* Return malloc-owned model-space token IDs, reusing workspace scratch.
 * Empty text returns NULL with a zero count. */
int *tok_spm_encode_with_workspace(const tok_spm_t *tok,
                                   tok_spm_workspace_t *ws,
                                   const char *text,
                                   int *out_n_tokens);

void tok_spm_free(tok_spm_t *tok);

#endif /* TOK_SPM_H */