aboutsummaryrefslogtreecommitdiff
path: root/src/tokenizer_bpe.h
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:10:56 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 05:57:19 +0200
commit6b969d25ce136eaf695222bd1a10bff1ecac0220 (patch)
treec23afb50c28b9deaddae5c6526a74ee11cf2d865 /src/tokenizer_bpe.h
parentd4bdb8dae753462f621507d5fc3582e0e53da35c (diff)
downloadffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.tar.gz
ffwd-6b969d25ce136eaf695222bd1a10bff1ecac0220.zip
all: split Makefile and reorg library/tools
Diffstat (limited to 'src/tokenizer_bpe.h')
-rw-r--r--src/tokenizer_bpe.h86
1 files changed, 0 insertions, 86 deletions
diff --git a/src/tokenizer_bpe.h b/src/tokenizer_bpe.h
deleted file mode 100644
index 8d13d47..0000000
--- a/src/tokenizer_bpe.h
+++ /dev/null
@@ -1,86 +0,0 @@
-/*
- * embed_tokenizer.h - Qwen2/Qwen3 BPE tokenizer (GPT-2 byte-level)
- *
- * Supports:
- * - decode token ID -> UTF-8 text
- * - encode UTF-8 text -> token IDs (vocab.json + merges.txt)
- */
-
-#ifndef EMBED_TOKENIZER_H
-#define EMBED_TOKENIZER_H
-
-/* Public-API export annotation. The libraries are built with
- * -fvisibility=hidden, so only declarations carrying EMBED_API are exported
- * from the shared library; everything else stays internal. */
-#ifndef EMBED_API
-#if defined(__GNUC__)
-#define EMBED_API __attribute__((visibility("default")))
-#else
-#define EMBED_API
-#endif
-#endif
-
-typedef struct {
- char **id_to_text; /* [vocab_size] decoded text strings */
- char **id_to_bpe; /* [vocab_size] raw BPE token strings from vocab.json */
- int vocab_size;
-
- /* Internal hash maps (opaque to callers) */
- void *vocab_map;
- int vocab_map_cap;
- void *merge_map;
- int merge_map_cap;
- /* Integer-pair BPE merge table: (left_id, right_id) -> (rank, merged_id).
- * NULL when the vocab could not resolve every merge; encoding then falls
- * back to the string-based merge loop. */
- void *int_merges;
- int int_merges_cap;
- /* Byte-level unicode codepoint -> single-character token id (512 slots),
- * or NULL when any byte token is missing from the vocab. */
- int *cp_to_id;
-} embed_tokenizer_t;
-
-typedef struct embed_tokenizer_workspace embed_tokenizer_workspace_t;
-
-/* Load tokenizer from vocab.json in model directory */
-EMBED_API embed_tokenizer_t *embed_tokenizer_load(const char *vocab_json_path);
-
-/* Decode a single token ID to text. Returns pointer to internal string. */
-EMBED_API const char *embed_tokenizer_decode(const embed_tokenizer_t *tok, int token_id);
-
-/* Look up the id of a raw vocab.json token string (e.g. "<|endoftext|>").
- * Returns -1 when the token is not in the vocab. */
-EMBED_API int embed_tokenizer_token_id(const embed_tokenizer_t *tok, const char *token);
-
-/* Encode UTF-8 text into token IDs using BPE.
- * Returns malloc'd array of token IDs and sets *out_n_tokens.
- * Returns NULL on error (and sets *out_n_tokens to 0). */
-EMBED_API int *
-embed_tokenizer_encode(const embed_tokenizer_t *tok, const char *text, int *out_n_tokens);
-
-/* Reusable scratch for allocation-light repeated encoding. */
-EMBED_API embed_tokenizer_workspace_t *embed_tokenizer_workspace_new(void);
-EMBED_API void embed_tokenizer_workspace_free(embed_tokenizer_workspace_t *ws);
-
-/* Encode into caller-provided out_ids[0..out_cap-1].
- * Returns 0 on success, -1 on error, -2 if out_cap is too small.
- * In all cases *out_n_tokens is set to the number of tokens needed/emitted
- * when out_n_tokens is non-NULL. */
-EMBED_API int embed_tokenizer_encode_into(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *text,
- int *out_ids,
- int out_cap,
- int *out_n_tokens);
-
-/* Same tokenization as embed_tokenizer_encode(), but reuses ws internally and
- * performs only the final returned int[] allocation. */
-EMBED_API int *embed_tokenizer_encode_with_workspace(const embed_tokenizer_t *tok,
- embed_tokenizer_workspace_t *ws,
- const char *text,
- int *out_n_tokens);
-
-/* Free tokenizer */
-EMBED_API void embed_tokenizer_free(embed_tokenizer_t *tok);
-
-#endif /* EMBED_TOKENIZER_H */