diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:38:02 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 14:38:02 +0200 |
| commit | 91c6aa07597462705a7f30ee5b97e2c7943b5616 (patch) | |
| tree | b16f686279a1f1b0691a6c0d65a544084fcc1c1e /tools/verify.py | |
| parent | 6f98fc8cf3f7cc4b9d523d16492a07d3e2aea8c9 (diff) | |
| download | langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.tar.gz langdet-91c6aa07597462705a7f30ee5b97e2c7943b5616.zip | |
Remove unused checkpoint, tokenizer, and attention compatibility paths
Diffstat (limited to 'tools/verify.py')
| -rw-r--r-- | tools/verify.py | 30 |
1 files changed, 23 insertions, 7 deletions
diff --git a/tools/verify.py b/tools/verify.py index 197f68f..d8bccf4 100644 --- a/tools/verify.py +++ b/tools/verify.py @@ -102,15 +102,33 @@ def check_failures(model_dir): missing_tokenizer.mkdir() shutil.copyfile(model_dir / "config.json", missing_tokenizer / "config.json") (missing_tokenizer / "model.safetensors").symlink_to(model_dir / "model.safetensors") - for directory in (base / "absent", empty, invalid_config, missing_weights, - corrupt_weights, missing_tokenizer): + directories = [base / "absent", empty, invalid_config, missing_weights, + corrupt_weights, missing_tokenizer] + for name, weights, tokenizer in ( + ("sharded-weights", "model-00001-of-00001.safetensors", "sentencepiece.bpe.model"), + ("alternate-tokenizer", "model.safetensors", "tokenizer.model"), + ("alternate-spiece", "model.safetensors", "spiece.model"), + ("bert", "model.safetensors", "sentencepiece.bpe.model"), + ): + directory = base / name + directory.mkdir() + config = json.loads((model_dir / "config.json").read_text(encoding="utf-8")) + if name == "bert": + config["model_type"] = "bert" + config["architectures"] = ["BertForSequenceClassification"] + (directory / "config.json").write_text(json.dumps(config), encoding="utf-8") + (directory / weights).symlink_to(model_dir / "model.safetensors") + (directory / tokenizer).symlink_to(model_dir / "sentencepiece.bpe.model") + directories.append(directory) + for directory in directories: result = subprocess.run( [str(CLI), "-d", str(directory), "hello"], capture_output=True, timeout=30, ) require(result.returncode == 1 and result.stderr and not result.stdout, f"Failed model load did not return a clean error: {directory.name}") - print("PASS: invalid CLI arguments, NUL input, and six failed model-loading cases.", flush=True) + print(f"PASS: invalid CLI arguments, NUL input, and {len(directories)} failed model-loading cases.", + flush=True) def check_reference(model_dir): @@ -118,15 +136,14 @@ def check_reference(model_dir): tokenizer = AutoTokenizer.from_pretrained(model_dir, local_files_only=True, use_fast=True) require(tokenizer.is_fast, "Expected the Hugging Face fast tokenizer") cases = list(CASES) - # Adjacent sequence lengths exercise both sides of the 512-token limit. + # Boundary cases straddle the BLAS attention threshold and the token limit. unit = "test " require(len(tokenizer(unit, add_special_tokens=False)["input_ids"]) == 1, "Boundary fixture must encode to one token") - cases.extend((f"length-{n}", unit * (n - 2)) for n in (511, 512, 513)) + cases.extend((f"length-{n}", unit * (n - 2)) for n in (127, 128, 129, 511, 512, 513)) texts = [text for _, text in cases] native_ids = json_lines([TOKEN_IDS, model_dir, *texts]) require(len(native_ids) == len(cases), "Missing native tokenization results") - reference_ids = [] for (name, text), ids in zip(cases, native_ids): expected = tokenizer(text, truncation=False, verbose=False)["input_ids"] if name.startswith("length-"): @@ -134,7 +151,6 @@ def check_reference(model_dir): if name == "long": require(len(expected) > MAX_TOKENS, "Long fixture must exceed the token limit") require(ids == expected, f"{name}: token IDs differ\nnative: {ids}\nreference: {expected}") - reference_ids.append(expected) print(f"PASS: exact token IDs for {len(cases)} inputs.", flush=True) model = AutoModelForSequenceClassification.from_pretrained( |
