From dbdf714bb57578431924e5d069089bdfd9f4c604 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 20 Jun 2026 20:31:06 +0200 Subject: devtools: add scripts for unicode generation, model inspection, embedding ranking, and more --- devtools/inspect_model_artifacts.py | 260 ++++++++++++++++++++++++++++++++++++ 1 file changed, 260 insertions(+) create mode 100755 devtools/inspect_model_artifacts.py (limited to 'devtools/inspect_model_artifacts.py') diff --git a/devtools/inspect_model_artifacts.py b/devtools/inspect_model_artifacts.py new file mode 100755 index 0000000..0265832 --- /dev/null +++ b/devtools/inspect_model_artifacts.py @@ -0,0 +1,260 @@ +#!/usr/bin/env python3 +"""Inspect and detect local model snapshots, across model families. + +For each model it detects: + - the architecture (from config.json: model_type / architectures / dims), + - the tokenizer family - BPE (vocab.json + merges.txt), WordPiece (vocab.txt), + or SentencePiece (sentencepiece.bpe.model / spiece.model / tokenizer.model), + - the weight layout - single safetensors, sharded safetensors, or legacy + pytorch_model.bin, + - the serving kind - base encoder, pooled sentence-embedding, or projected / + late-interaction (1_Dense) - plus the Sentence-Transformers pipeline. + +ONNX exports and other extras are reported when present. + +A MODEL_DIR argument may be a model directory (one with config.json) or any +directory to search. Point it at a single snapshot, a folder of models, or the +whole Hugging Face hub cache (~/.cache/huggingface/hub) and it finds each model. + +A snapshot is "ok" when it has config.json, some weights, and some tokenizer; +everything else is informational. + +Usage: inspect_model_artifacts.py MODEL_DIR [MODEL_DIR ...] +""" + +from __future__ import annotations + +import argparse +import json +import os +from pathlib import Path + + +TOKENIZER_ARTIFACTS = ( + "tokenizer.json", + "tokenizer_config.json", + "special_tokens_map.json", + "added_tokens.json", + "vocab.json", + "merges.txt", + "vocab.txt", + "sentencepiece.bpe.model", + "spiece.model", + "tokenizer.model", +) + +# The first family whose required files are all present wins. +TOKENIZER_FAMILIES = ( + ("BPE", ("vocab.json", "merges.txt")), + ("WordPiece", ("vocab.txt",)), + ("SentencePiece", ("sentencepiece.bpe.model",)), + ("SentencePiece", ("spiece.model",)), + ("SentencePiece", ("tokenizer.model",)), +) + +EXTRA_ARTIFACTS = ( + "config_sentence_transformers.json", + "sentence_bert_config.json", + "modules.json", + "1_Pooling", + "1_Dense", + "onnx", + "generation_config.json", +) + + +def human_bytes(size: int) -> str: + value = float(size) + for unit in ("B", "KiB", "MiB", "GiB", "TiB"): + if value < 1024.0 or unit == "TiB": + return f"{value:.1f} {unit}" + value /= 1024.0 + return f"{value:.1f} TiB" + + +def describe(path: Path) -> str: + if not path.exists(): + return "MISSING" + if path.is_dir(): + return f"{sum(1 for _ in path.iterdir())} entries" + return human_bytes(path.resolve().stat().st_size) + + +def fmt(label: str, path: Path, indent: int = 4, width: int = 32) -> str: + return f"{' ' * indent}{label:<{width}} {describe(path)}" + + +def is_model_dir(d: Path) -> bool: + return (d / "config.json").is_file() + + +def discover(root: Path) -> list[Path]: + """Return the model directories at or under `root`. + + A model directory has a config.json. If `root` is itself one, return it; + otherwise search descendants (e.g. a folder of models or the HF hub cache), + skipping onnx sub-exports and dropping any dir nested under another model. + """ + root = root.expanduser().resolve() + if is_model_dir(root): + return [root] + found = { + cfg.parent + for cfg in root.rglob("config.json") + if cfg.parent.name != "onnx" + } + out = [] + for d in sorted(found): + if not any(d != o and str(d).startswith(str(o) + os.sep) for o in found): + out.append(d) + return out + + +def tokenizer_family(d: Path) -> str: + for name, required in TOKENIZER_FAMILIES: + if all((d / f).exists() for f in required): + return name + if (d / "tokenizer.json").exists(): + return "fast-only (tokenizer.json)" + return "none" + + +def config_summary(d: Path) -> str | None: + cfg = d / "config.json" + if not cfg.exists(): + return None + try: + data = json.loads(cfg.read_text()) + except (OSError, ValueError): + return "config.json: unreadable" + parts = [] + if data.get("model_type"): + parts.append(str(data["model_type"])) + arch = data.get("architectures") or [] + if arch: + parts.append(str(arch[0])) + for key in ("hidden_size", "num_hidden_layers", "num_attention_heads", + "num_key_value_heads", "vocab_size", "max_position_embeddings"): + if key in data: + parts.append(f"{key}={data[key]}") + return " ".join(parts) if parts else "config.json: no recognized fields" + + +def modules_pipeline(d: Path) -> str | None: + mj = d / "modules.json" + if not mj.exists(): + return None + try: + mods = json.loads(mj.read_text()) + except (OSError, ValueError): + return None + names = [(m.get("type") or "").rsplit(".", 1)[-1] or m.get("name") or "?" + for m in mods] + return " -> ".join(n for n in names if n) or None + + +def model_kind(d: Path) -> str: + if (d / "1_Dense").is_dir(): + return "projected / late-interaction (1_Dense)" + if (d / "1_Pooling").is_dir() or (d / "modules.json").exists(): + return "pooled sentence-embedding" + return "base encoder / LM" + + +def weight_lines(d: Path) -> tuple[list[str], bool]: + """Return (formatted lines, whether any weights were found).""" + lines: list[str] = [] + st_index = d / "model.safetensors.index.json" + st_shards = sorted(d.glob("model-*-of-*.safetensors")) + pt_index = d / "pytorch_model.bin.index.json" + pt_shards = sorted(d.glob("pytorch_model-*-of-*.bin")) + onnx = sorted((d / "onnx").glob("*.onnx")) if (d / "onnx").is_dir() else [] + found = False + + if (d / "model.safetensors").exists(): + lines.append(fmt("model.safetensors", d / "model.safetensors")) + found = True + if st_index.exists() and st_shards: + lines.append(fmt("model.safetensors.index.json", st_index)) + lines += [fmt(s.name, s, indent=6) for s in st_shards] + found = True + if (d / "pytorch_model.bin").exists(): + lines.append(fmt("pytorch_model.bin", d / "pytorch_model.bin")) + found = True + if pt_index.exists() and pt_shards: + lines.append(fmt("pytorch_model.bin.index.json", pt_index)) + lines += [fmt(s.name, s, indent=6) for s in pt_shards] + found = True + if onnx: + lines += [fmt(f"onnx/{o.name}", o) for o in onnx] + found = True + if not found: + lines.append(" (no safetensors / pytorch / onnx weights)") + return lines, found + + +def inspect(model_dir: Path) -> bool: + d = model_dir.expanduser().resolve() + print(f"snapshot: {d}") + if not d.is_dir(): + print(" NOT A DIRECTORY") + return False + + print(f" model: {config_summary(d) or 'config.json MISSING'}") + print(f" kind: {model_kind(d)}") + pipeline = modules_pipeline(d) + if pipeline: + print(f" pipeline: {pipeline}") + + family = tokenizer_family(d) + print(f" tokenizer: {family}") + for name in TOKENIZER_ARTIFACTS: + if (d / name).exists(): + print(fmt(name, d / name)) + + print(" weights:") + wlines, has_weights = weight_lines(d) + print("\n".join(wlines)) + + extras = [name for name in EXTRA_ARTIFACTS if (d / name).exists()] + if extras: + print(" extras:") + for name in extras: + print(fmt(name, d / name)) + + ok = (d / "config.json").exists() and has_weights and family != "none" + print(f" verdict: {'ok' if ok else 'incomplete'}") + return ok + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR") + args = ap.parse_args() + + ok = True + first = True + for name in args.model_dirs: + root = Path(os.path.expandvars(name)).expanduser().resolve() + found = discover(root) + if not found: + if not first: + print() + print(f"snapshot: {root}\n no config.json found at or under this path") + ok = first = False + continue + if len(found) > 1: + if not first: + print() + print(f"# {root}: {len(found)} models") + first = False + for d in found: + if not first: + print() + ok = inspect(d) and ok + first = False + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) -- cgit v1.3.1