aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:38:02 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:38:02 +0200
commit91c6aa07597462705a7f30ee5b97e2c7943b5616 (patch)
treeb16f686279a1f1b0691a6c0d65a544084fcc1c1e /tools
parent6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (diff)
downloadlangdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.tar.gz
langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.zip
Remove unused checkpoint, tokenizer, and attention compatibility paths
Diffstat (limited to 'tools')
-rw-r--r--tools/test_safetensors.c74
-rw-r--r--tools/verify.py30
2 files changed, 97 insertions, 7 deletions
diff --git a/tools/test_safetensors.c b/tools/test_safetensors.c
new file mode 100644
index 0000000..5a00c54
--- /dev/null
+++ b/tools/test_safetensors.c
@@ -0,0 +1,74 @@
+#define _POSIX_C_SOURCE 200809L
+#include "safetensors.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <unistd.h>
+
+#define CHECK(condition) do { \
+ if (!(condition)) { \
+ fprintf(stderr, "%s:%d: %s\n", __FILE__, __LINE__, #condition); \
+ exit(1); \
+ } \
+} while (0)
+
+static safetensors_file_t *read_header(const char *header) {
+ char path[] = "build/safetensors-test-XXXXXX";
+ int fd = mkstemp(path);
+ CHECK(fd >= 0);
+ FILE *file = fdopen(fd, "wb");
+ CHECK(file);
+ size_t length = strlen(header);
+ uint64_t padded = (length + 7) & ~(uint64_t)7;
+ const float values[] = {1.25f, -2.5f};
+ CHECK(fwrite(&padded, sizeof(padded), 1, file) == 1);
+ CHECK(fwrite(header, 1, length, file) == length);
+ for (size_t i = length; i < padded; i++)
+ CHECK(fputc(' ', file) != EOF);
+ CHECK(fwrite(values, sizeof(values), 1, file) == 1);
+ CHECK(fclose(file) == 0);
+ safetensors_file_t *sf = safetensors_open(path);
+ CHECK(unlink(path) == 0);
+ return sf;
+}
+
+int main(void) {
+ safetensors_file_t *sf = read_header(
+ "{\"__metadata__\":{\"format\":\"pt\"},"
+ "\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,8]}}");
+ CHECK(sf && sf->num_tensors == 1);
+ const safetensor_t *tensor = safetensors_find(sf, "weight");
+ CHECK(tensor && tensor->dtype == DTYPE_F32 && tensor->ndim == 1 && tensor->shape[0] == 2);
+ CHECK(tensor->data_offset == 0 && tensor->data_size == 8);
+ const float *values = safetensors_data(sf, tensor);
+ CHECK(values[0] == 1.25f && values[1] == -2.5f);
+ CHECK(!safetensors_find(sf, "missing"));
+ safetensors_close(sf);
+
+ sf = read_header("{\"position_ids\":{\"dtype\":\"I64\",\"shape\":[1,1],\"data_offsets\":[0,8]}}");
+ CHECK(sf && safetensors_find(sf, "position_ids")->dtype == DTYPE_UNKNOWN);
+ safetensors_close(sf);
+ safetensors_close(NULL);
+
+ const char *invalid[] = {
+ "{",
+ "[]",
+ "{\"weight\":null}",
+ "{\"weight\":{\"shape\":[2],\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":2,\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[-1],\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[1.5],\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[9223372036854775808],\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[1,1,1,1,1,1,1,1,1],\"data_offsets\":[0,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[-1,8]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[8,0]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,9]}}",
+ "{\"weight\":{\"dtype\":\"F32\",\"shape\":[2],\"data_offsets\":[0,18446744073709551616]}}",
+ };
+ for (size_t i = 0; i < sizeof(invalid) / sizeof(invalid[0]); i++)
+ CHECK(!read_header(invalid[i]));
+ puts("PASS: single-file tensor loading, metadata, lookup, and malformed header rejection.");
+ return 0;
+}
diff --git a/tools/verify.py b/tools/verify.py
index 197f68f..d8bccf4 100644
--- a/tools/verify.py
+++ b/tools/verify.py
@@ -102,15 +102,33 @@ def check_failures(model_dir):
missing_tokenizer.mkdir()
shutil.copyfile(model_dir / "config.json", missing_tokenizer / "config.json")
(missing_tokenizer / "model.safetensors").symlink_to(model_dir / "model.safetensors")
- for directory in (base / "absent", empty, invalid_config, missing_weights,
- corrupt_weights, missing_tokenizer):
+ directories = [base / "absent", empty, invalid_config, missing_weights,
+ corrupt_weights, missing_tokenizer]
+ for name, weights, tokenizer in (
+ ("sharded-weights", "model-00001-of-00001.safetensors", "sentencepiece.bpe.model"),
+ ("alternate-tokenizer", "model.safetensors", "tokenizer.model"),
+ ("alternate-spiece", "model.safetensors", "spiece.model"),
+ ("bert", "model.safetensors", "sentencepiece.bpe.model"),
+ ):
+ directory = base / name
+ directory.mkdir()
+ config = json.loads((model_dir / "config.json").read_text(encoding="utf-8"))
+ if name == "bert":
+ config["model_type"] = "bert"
+ config["architectures"] = ["BertForSequenceClassification"]
+ (directory / "config.json").write_text(json.dumps(config), encoding="utf-8")
+ (directory / weights).symlink_to(model_dir / "model.safetensors")
+ (directory / tokenizer).symlink_to(model_dir / "sentencepiece.bpe.model")
+ directories.append(directory)
+ for directory in directories:
result = subprocess.run(
[str(CLI), "-d", str(directory), "hello"],
capture_output=True, timeout=30,
)
require(result.returncode == 1 and result.stderr and not result.stdout,
f"Failed model load did not return a clean error: {directory.name}")
- print("PASS: invalid CLI arguments, NUL input, and six failed model-loading cases.", flush=True)
+ print(f"PASS: invalid CLI arguments, NUL input, and {len(directories)} failed model-loading cases.",
+ flush=True)
def check_reference(model_dir):
@@ -118,15 +136,14 @@ def check_reference(model_dir):
tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, use_fast=True)
require(tokenizer.is_fast, "Expected the Hugging Face fast tokenizer")
cases = list(CASES)
- # Adjacent sequence lengths exercise both sides of the 512-token limit.
+ # Boundary cases straddle the BLAS attention threshold and the token limit.
unit = "test "
require(len(tokenizer(unit, add_special_tokens=False)["input_ids"]) == 1,
"Boundary fixture must encode to one token")
- cases.extend((f"length-{n}", unit * (n - 2)) for n in (511, 512, 513))
+ cases.extend((f"length-{n}", unit * (n - 2)) for n in (127, 128, 129, 511, 512, 513))
texts = [text for _, text in cases]
native_ids = json_lines([TOKEN_IDS, model_dir, *texts])
require(len(native_ids) == len(cases), "Missing native tokenization results")
- reference_ids = []
for (name, text), ids in zip(cases, native_ids):
expected = tokenizer(text, truncation=False, verbose=False)["input_ids"]
if name.startswith("length-"):
@@ -134,7 +151,6 @@ def check_reference(model_dir):
if name == "long":
require(len(expected) > MAX_TOKENS, "Long fixture must exceed the token limit")
require(ids == expected, f"{name}: token IDs differ\nnative: {ids}\nreference: {expected}")
- reference_ids.append(expected)
print(f"PASS: exact token IDs for {len(cases)} inputs.", flush=True)
model = AutoModelForSequenceClassification.from_pretrained(