diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:38:02 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:38:02 +0200 |
| commit | 91c6aa07597462705a7f30ee5b97e2c7943b5616 (patch) | |
| tree | b16f686279a1f1b0691a6c0d65a544084fcc1c1e /tools | |
| parent | 6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (diff) | |
| download | langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.tar.gz langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.zip | |
Remove unused checkpoint, tokenizer, and attention compatibility paths
Diffstat (limited to 'tools')
| -rw-r--r-- | tools/test_safetensors.c | 74 | ||||
| -rw-r--r-- | tools/verify.py | 30 |
2 files changed, 97 insertions, 7 deletions
diff --git a/tools/test_safetensors.c b/tools/test_safetensors.c new file mode 100644 index 0000000..5a00c54 --- /dev/null +++ b/tools/test_safetensors.c @@ -0,0 +1,74 @@ +#define _POSIX_C_SOURCE 200809L +#include "safetensors.h" + +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include <unistd.h> + +#define CHECK(condition) do { \ + if (!(condition)) { \ + fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \ + exit(1); \ + } \ +} while (0) + +static safetensors_file_t *read_header(const char *header) { + char path[] = "build/safetensors-test-XXXXXX"; + int fd = mkstemp(path); + CHECK(fd >= 0); + FILE *file = fdopen(fd, "wb"); + CHECK(file); + size_t length = strlen(header); + uint64_t padded = (length + 7) & ~(uint64_t)7; + const float values[] = {1.25f, -2.5f}; + CHECK(fwrite(&padded, sizeof(padded), 1, file) == 1); + CHECK(fwrite(header, 1, length, file) == length); + for (size_t i = length; i < padded; i++) + CHECK(fputc(' ', file) != EOF); + CHECK(fwrite(values, sizeof(values), 1, file) == 1); + CHECK(fclose(file) == 0); + safetensors_file_t *sf = safetensors_open(path); + CHECK(unlink(path) == 0); + return sf; +} + +int main(void) { + safetensors_file_t *sf = read_header( + "{\"__metadata__\":{\"format\":\"pt\"}," + "\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,8]}}"); + CHECK(sf && sf->num_tensors == 1); + const safetensor_t *tensor = safetensors_find(sf, "weight"); + CHECK(tensor && tensor->dtype == DTYPE_F32 && tensor->ndim == 1 && tensor->shape[0] == 2); + CHECK(tensor->data_offset == 0 && tensor->data_size == 8); + const float *values = safetensors_data(sf, tensor); + CHECK(values[0] == 1.25f && values[1] == -2.5f); + CHECK(!safetensors_find(sf, "missing")); + safetensors_close(sf); + + sf = read_header("{\"position_ids\":{\"dtype\":\"I64\",\"shape\":[1,1],\"data_offsets\":[0,8]}}"); + CHECK(sf && safetensors_find(sf, "position_ids")->dtype == DTYPE_UNKNOWN); + safetensors_close(sf); + safetensors_close(NULL); + + const char *invalid[] = { + "{", + "[]", + "{\"weight\":null}", + "{\"weight\":{\"shape\":[2],\"data_offsets\":[0,8]}}", + "{\"weight\":{\"dtype\":\"F32\",\"shape\":2,\"data_offsets\":[0,8]}}", + "{\"weight\":{\"dtype\":\"F32\",\"shape\":[-1],\"data_offsets\":[0,8]}}", + "{\"weight\":{\"dtype\":\"F32\",\"shape\":[1.5],\"data_offsets\":[0,8]}}", + "{\"weight\":{\"dtype\":\"F32\",\"shape\":[9223372036854775808],\"data_offsets\":[0,8]}}", + "{\"weight\":{\"dtype\":\"F32\",\"shape\":[1,1,1,1,1,1,1,1,1],\"data_offsets\":[0,8]}}", + "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0]}}", + "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[-1,8]}}", + "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[8,0]}}", + "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,9]}}", + "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,18446744073709551616]}}", + }; + for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); i++) + CHECK(!read_header(invalid[i])); + puts("PASS: single-file tensor loading, metadata, lookup, and malformed header rejection."); + return 0; +} diff --git a/tools/verify.py b/tools/verify.py index 197f68f..d8bccf4 100644 --- a/tools/verify.py +++ b/tools/verify.py @@ -102,15 +102,33 @@ def check_failures(model_dir): missing_tokenizer.mkdir() shutil.copyfile(model_dir / "config.json", missing_tokenizer / "config.json") (missing_tokenizer / "model.safetensors").symlink_to(model_dir / "model.safetensors") - for directory in (base / "absent", empty, invalid_config, missing_weights, - corrupt_weights, missing_tokenizer): + directories = [base / "absent", empty, invalid_config, missing_weights, + corrupt_weights, missing_tokenizer] + for name, weights, tokenizer in ( + ("sharded-weights", "model-00001-of-00001.safetensors", "sentencepiece.bpe.model"), + ("alternate-tokenizer", "model.safetensors", "tokenizer.model"), + ("alternate-spiece", "model.safetensors", "spiece.model"), + ("bert", "model.safetensors", "sentencepiece.bpe.model"), + ): + directory = base / name + directory.mkdir() + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + if name == "bert": + config["model_type"] = "bert" + config["architectures"] = ["BertForSequenceClassification"] + (directory / "config.json").write_text(json.dumps(config), encoding="utf-8") + (directory / weights).symlink_to(model_dir / "model.safetensors") + (directory / tokenizer).symlink_to(model_dir / "sentencepiece.bpe.model") + directories.append(directory) + for directory in directories: result = subprocess.run( [str(CLI), "-d", str(directory), "hello"], capture_output=True, timeout=30, ) require(result.returncode == 1 and result.stderr and not result.stdout, f"Failed model load did not return a clean error: {directory.name}") - print("PASS: invalid CLI arguments, NUL input, and six failed model-loading cases.", flush=True) + print(f"PASS: invalid CLI arguments, NUL input, and {len(directories)} failed model-loading cases.", + flush=True) def check_reference(model_dir): @@ -118,15 +136,14 @@ def check_reference(model_dir): tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, use_fast=True) require(tokenizer.is_fast, "Expected the Hugging Face fast tokenizer") cases = list(CASES) - # Adjacent sequence lengths exercise both sides of the 512-token limit. + # Boundary cases straddle the BLAS attention threshold and the token limit. unit = "test " require(len(tokenizer(unit, add_special_tokens=False)["input_ids"]) == 1, "Boundary fixture must encode to one token") - cases.extend((f"length-{n}", unit * (n - 2)) for n in (511, 512, 513)) + cases.extend((f"length-{n}", unit * (n - 2)) for n in (127, 128, 129, 511, 512, 513)) texts = [text for _, text in cases] native_ids = json_lines([TOKEN_IDS, model_dir, *texts]) require(len(native_ids) == len(cases), "Missing native tokenization results") - reference_ids = [] for (name, text), ids in zip(cases, native_ids): expected = tokenizer(text, truncation=False, verbose=False)["input_ids"] if name.startswith("length-"): @@ -134,7 +151,6 @@ def check_reference(model_dir): if name == "long": require(len(expected) > MAX_TOKENS, "Long fixture must exceed the token limit") require(ids == expected, f"{name}: token IDs differ\nnative: {ids}\nreference: {expected}") - reference_ids.append(expected) print(f"PASS: exact token IDs for {len(cases)} inputs.", flush=True) model = AutoModelForSequenceClassification.from_pretrained( |
