aboutsummaryrefslogtreecommitdiff
path: root/tools/verify.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:38:02 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 14:38:02 +0200
commit91c6aa07597462705a7f30ee5b97e2c7943b5616 (patch)
treeb16f686279a1f1b0691a6c0d65a544084fcc1c1e /tools/verify.py
parent6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (diff)
downloadlangdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.tar.gz
langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.zip
Remove unused checkpoint, tokenizer, and attention compatibility paths
Diffstat (limited to 'tools/verify.py')
-rw-r--r--tools/verify.py30
1 files changed, 23 insertions, 7 deletions
diff --git a/tools/verify.py b/tools/verify.py
index 197f68f..d8bccf4 100644
--- a/tools/verify.py
+++ b/tools/verify.py
@@ -102,15 +102,33 @@ def check_failures(model_dir):
missing_tokenizer.mkdir()
shutil.copyfile(model_dir / "config.json", missing_tokenizer / "config.json")
(missing_tokenizer / "model.safetensors").symlink_to(model_dir / "model.safetensors")
- for directory in (base / "absent", empty, invalid_config, missing_weights,
- corrupt_weights, missing_tokenizer):
+ directories = [base / "absent", empty, invalid_config, missing_weights,
+ corrupt_weights, missing_tokenizer]
+ for name, weights, tokenizer in (
+ ("sharded-weights", "model-00001-of-00001.safetensors", "sentencepiece.bpe.model"),
+ ("alternate-tokenizer", "model.safetensors", "tokenizer.model"),
+ ("alternate-spiece", "model.safetensors", "spiece.model"),
+ ("bert", "model.safetensors", "sentencepiece.bpe.model"),
+ ):
+ directory = base / name
+ directory.mkdir()
+ config = json.loads((model_dir / "config.json").read_text(encoding="utf-8"))
+ if name == "bert":
+ config["model_type"] = "bert"
+ config["architectures"] = ["BertForSequenceClassification"]
+ (directory / "config.json").write_text(json.dumps(config), encoding="utf-8")
+ (directory / weights).symlink_to(model_dir / "model.safetensors")
+ (directory / tokenizer).symlink_to(model_dir / "sentencepiece.bpe.model")
+ directories.append(directory)
+ for directory in directories:
result = subprocess.run(
[str(CLI), "-d", str(directory), "hello"],
capture_output=True, timeout=30,
)
require(result.returncode == 1 and result.stderr and not result.stdout,
f"Failed model load did not return a clean error: {directory.name}")
- print("PASS: invalid CLI arguments, NUL input, and six failed model-loading cases.", flush=True)
+ print(f"PASS: invalid CLI arguments, NUL input, and {len(directories)} failed model-loading cases.",
+ flush=True)
def check_reference(model_dir):
@@ -118,15 +136,14 @@ def check_reference(model_dir):
tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, use_fast=True)
require(tokenizer.is_fast, "Expected the Hugging Face fast tokenizer")
cases = list(CASES)
- # Adjacent sequence lengths exercise both sides of the 512-token limit.
+ # Boundary cases straddle the BLAS attention threshold and the token limit.
unit = "test "
require(len(tokenizer(unit, add_special_tokens=False)["input_ids"]) == 1,
"Boundary fixture must encode to one token")
- cases.extend((f"length-{n}", unit * (n - 2)) for n in (511, 512, 513))
+ cases.extend((f"length-{n}", unit * (n - 2)) for n in (127, 128, 129, 511, 512, 513))
texts = [text for _, text in cases]
native_ids = json_lines([TOKEN_IDS, model_dir, *texts])
require(len(native_ids) == len(cases), "Missing native tokenization results")
- reference_ids = []
for (name, text), ids in zip(cases, native_ids):
expected = tokenizer(text, truncation=False, verbose=False)["input_ids"]
if name.startswith("length-"):
@@ -134,7 +151,6 @@ def check_reference(model_dir):
if name == "long":
require(len(expected) > MAX_TOKENS, "Long fixture must exceed the token limit")
require(ids == expected, f"{name}: token IDs differ\nnative: {ids}\nreference: {expected}")
- reference_ids.append(expected)
print(f"PASS: exact token IDs for {len(cases)} inputs.", flush=True)
model = AutoModelForSequenceClassification.from_pretrained(