1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
|
/* Parity driver for the BPE tokenizer. Loads vocab.json (merges.txt is read
* from the same directory), encodes each argument through the three encode
* entry points (and cross-checks they agree), then prints "<count> <id> <id>
* ..." per line. check_tokenizer_parity.py builds this with
* `make parity-tokenizer-driver` and compares the ids against the stored
* reference vectors in tests/test-vectors/bpe/. Links libtok. */
#include "bpe.h"
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
int main(int argc, char **argv) {
if (argc < 2) {
fprintf(stderr, "usage: %s VOCAB_JSON [TEXT...]\n", argv[0]);
return 2;
}
tok_bpe_t *tok = tok_bpe_load(argv[1]);
if (!tok) {
fprintf(stderr, "failed to load tokenizer: %s\n", argv[1]);
return 1;
}
tok_bpe_workspace_t *ws = tok_bpe_workspace_new();
if (!ws) {
fprintf(stderr, "failed to allocate tokenizer workspace\n");
tok_bpe_free(tok);
return 1;
}
for (int a = 2; a < argc; a++) {
int n = 0;
int *ids = tok_bpe_encode(tok, argv[a], &n);
if (!ids && n != 0) {
fprintf(stderr, "failed to encode argument %d\n", a - 2);
tok_bpe_free(tok);
return 1;
}
int n_ws = 0;
int *ids_ws = tok_bpe_encode_with_workspace(tok, ws, argv[a], &n_ws);
if (n_ws != n || (n_ws > 0 && !ids_ws)) {
fprintf(stderr, "workspace encode count mismatch for argument %d\n", a - 2);
free(ids);
free(ids_ws);
tok_bpe_workspace_free(ws);
tok_bpe_free(tok);
return 1;
}
for (int i = 0; i < n; i++) {
if (ids[i] != ids_ws[i]) {
fprintf(stderr, "workspace encode mismatch for argument %d token %d\n", a - 2, i);
free(ids);
free(ids_ws);
tok_bpe_workspace_free(ws);
tok_bpe_free(tok);
return 1;
}
}
int cap = (int)(strlen(argv[a]) * 2 + 8);
if (cap < 1)
cap = 1;
int *into = (int *)malloc((size_t)cap * sizeof(int));
int n_into = 0;
int rc = tok_bpe_encode_into(tok, ws, argv[a], into, cap, &n_into);
if (rc == -2) {
int *tmp = (int *)realloc(into, (size_t)n_into * sizeof(int));
if (!tmp) {
free(into);
free(ids);
free(ids_ws);
tok_bpe_workspace_free(ws);
tok_bpe_free(tok);
return 1;
}
into = tmp;
cap = n_into;
rc = tok_bpe_encode_into(tok, ws, argv[a], into, cap, &n_into);
}
if (rc != 0 || n_into != n) {
fprintf(stderr, "encode_into mismatch for argument %d\n", a - 2);
free(into);
free(ids);
free(ids_ws);
tok_bpe_workspace_free(ws);
tok_bpe_free(tok);
return 1;
}
for (int i = 0; i < n; i++) {
if (ids[i] != into[i]) {
fprintf(stderr, "encode_into mismatch for argument %d token %d\n", a - 2, i);
free(into);
free(ids);
free(ids_ws);
tok_bpe_workspace_free(ws);
tok_bpe_free(tok);
return 1;
}
}
printf("%d", n);
for (int i = 0; i < n; i++)
printf(" %d", ids[i]);
printf("\n");
free(into);
free(ids_ws);
free(ids);
}
tok_bpe_workspace_free(ws);
tok_bpe_free(tok);
return 0;
}
|