aboutsummaryrefslogtreecommitdiff
path: root/devtools/inspect_model_artifacts.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:31:06 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commitdbdf714bb57578431924e5d069089bdfd9f4c604 (patch)
tree4f5b59ef132220310e1e4b1dde64f5e38782369a /devtools/inspect_model_artifacts.py
parente9ecaeaf21050f39888d33d2c8f4216d595d4975 (diff)
downloadffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.tar.gz
ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.zip
devtools: add scripts for unicode generation, model inspection, embedding ranking, and more
Diffstat (limited to '')
-rwxr-xr-xdevtools/inspect_model_artifacts.py260
1 files changed, 260 insertions, 0 deletions
diff --git a/devtools/inspect_model_artifacts.py b/devtools/inspect_model_artifacts.py
new file mode 100755
index 0000000..0265832
--- /dev/null
+++ b/devtools/inspect_model_artifacts.py
@@ -0,0 +1,260 @@
+#!/usr/bin/env python3
+"""Inspect and detect local model snapshots, across model families.
+
+For each model it detects:
+ - the architecture (from config.json: model_type / architectures / dims),
+ - the tokenizer family - BPE (vocab.json + merges.txt), WordPiece (vocab.txt),
+ or SentencePiece (sentencepiece.bpe.model / spiece.model / tokenizer.model),
+ - the weight layout - single safetensors, sharded safetensors, or legacy
+ pytorch_model.bin,
+ - the serving kind - base encoder, pooled sentence-embedding, or projected /
+ late-interaction (1_Dense) - plus the Sentence-Transformers pipeline.
+
+ONNX exports and other extras are reported when present.
+
+A MODEL_DIR argument may be a model directory (one with config.json) or any
+directory to search. Point it at a single snapshot, a folder of models, or the
+whole Hugging Face hub cache (~/.cache/huggingface/hub) and it finds each model.
+
+A snapshot is "ok" when it has config.json, some weights, and some tokenizer;
+everything else is informational.
+
+Usage: inspect_model_artifacts.py MODEL_DIR [MODEL_DIR ...]
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+
+TOKENIZER_ARTIFACTS = (
+ "tokenizer.json",
+ "tokenizer_config.json",
+ "special_tokens_map.json",
+ "added_tokens.json",
+ "vocab.json",
+ "merges.txt",
+ "vocab.txt",
+ "sentencepiece.bpe.model",
+ "spiece.model",
+ "tokenizer.model",
+)
+
+# The first family whose required files are all present wins.
+TOKENIZER_FAMILIES = (
+ ("BPE", ("vocab.json", "merges.txt")),
+ ("WordPiece", ("vocab.txt",)),
+ ("SentencePiece", ("sentencepiece.bpe.model",)),
+ ("SentencePiece", ("spiece.model",)),
+ ("SentencePiece", ("tokenizer.model",)),
+)
+
+EXTRA_ARTIFACTS = (
+ "config_sentence_transformers.json",
+ "sentence_bert_config.json",
+ "modules.json",
+ "1_Pooling",
+ "1_Dense",
+ "onnx",
+ "generation_config.json",
+)
+
+
+def human_bytes(size: int) -> str:
+ value = float(size)
+ for unit in ("B", "KiB", "MiB", "GiB", "TiB"):
+ if value < 1024.0 or unit == "TiB":
+ return f"{value:.1f} {unit}"
+ value /= 1024.0
+ return f"{value:.1f} TiB"
+
+
+def describe(path: Path) -> str:
+ if not path.exists():
+ return "MISSING"
+ if path.is_dir():
+ return f"{sum(1 for _ in path.iterdir())} entries"
+ return human_bytes(path.resolve().stat().st_size)
+
+
+def fmt(label: str, path: Path, indent: int = 4, width: int = 32) -> str:
+ return f"{' ' * indent}{label:<{width}} {describe(path)}"
+
+
+def is_model_dir(d: Path) -> bool:
+ return (d / "config.json").is_file()
+
+
+def discover(root: Path) -> list[Path]:
+ """Return the model directories at or under `root`.
+
+ A model directory has a config.json. If `root` is itself one, return it;
+ otherwise search descendants (e.g. a folder of models or the HF hub cache),
+ skipping onnx sub-exports and dropping any dir nested under another model.
+ """
+ root = root.expanduser().resolve()
+ if is_model_dir(root):
+ return [root]
+ found = {
+ cfg.parent
+ for cfg in root.rglob("config.json")
+ if cfg.parent.name != "onnx"
+ }
+ out = []
+ for d in sorted(found):
+ if not any(d != o and str(d).startswith(str(o) + os.sep) for o in found):
+ out.append(d)
+ return out
+
+
+def tokenizer_family(d: Path) -> str:
+ for name, required in TOKENIZER_FAMILIES:
+ if all((d / f).exists() for f in required):
+ return name
+ if (d / "tokenizer.json").exists():
+ return "fast-only (tokenizer.json)"
+ return "none"
+
+
+def config_summary(d: Path) -> str | None:
+ cfg = d / "config.json"
+ if not cfg.exists():
+ return None
+ try:
+ data = json.loads(cfg.read_text())
+ except (OSError, ValueError):
+ return "config.json: unreadable"
+ parts = []
+ if data.get("model_type"):
+ parts.append(str(data["model_type"]))
+ arch = data.get("architectures") or []
+ if arch:
+ parts.append(str(arch[0]))
+ for key in ("hidden_size", "num_hidden_layers", "num_attention_heads",
+ "num_key_value_heads", "vocab_size", "max_position_embeddings"):
+ if key in data:
+ parts.append(f"{key}={data[key]}")
+ return " ".join(parts) if parts else "config.json: no recognized fields"
+
+
+def modules_pipeline(d: Path) -> str | None:
+ mj = d / "modules.json"
+ if not mj.exists():
+ return None
+ try:
+ mods = json.loads(mj.read_text())
+ except (OSError, ValueError):
+ return None
+ names = [(m.get("type") or "").rsplit(".", 1)[-1] or m.get("name") or "?"
+ for m in mods]
+ return " -> ".join(n for n in names if n) or None
+
+
+def model_kind(d: Path) -> str:
+ if (d / "1_Dense").is_dir():
+ return "projected / late-interaction (1_Dense)"
+ if (d / "1_Pooling").is_dir() or (d / "modules.json").exists():
+ return "pooled sentence-embedding"
+ return "base encoder / LM"
+
+
+def weight_lines(d: Path) -> tuple[list[str], bool]:
+ """Return (formatted lines, whether any weights were found)."""
+ lines: list[str] = []
+ st_index = d / "model.safetensors.index.json"
+ st_shards = sorted(d.glob("model-*-of-*.safetensors"))
+ pt_index = d / "pytorch_model.bin.index.json"
+ pt_shards = sorted(d.glob("pytorch_model-*-of-*.bin"))
+ onnx = sorted((d / "onnx").glob("*.onnx")) if (d / "onnx").is_dir() else []
+ found = False
+
+ if (d / "model.safetensors").exists():
+ lines.append(fmt("model.safetensors", d / "model.safetensors"))
+ found = True
+ if st_index.exists() and st_shards:
+ lines.append(fmt("model.safetensors.index.json", st_index))
+ lines += [fmt(s.name, s, indent=6) for s in st_shards]
+ found = True
+ if (d / "pytorch_model.bin").exists():
+ lines.append(fmt("pytorch_model.bin", d / "pytorch_model.bin"))
+ found = True
+ if pt_index.exists() and pt_shards:
+ lines.append(fmt("pytorch_model.bin.index.json", pt_index))
+ lines += [fmt(s.name, s, indent=6) for s in pt_shards]
+ found = True
+ if onnx:
+ lines += [fmt(f"onnx/{o.name}", o) for o in onnx]
+ found = True
+ if not found:
+ lines.append(" (no safetensors / pytorch / onnx weights)")
+ return lines, found
+
+
+def inspect(model_dir: Path) -> bool:
+ d = model_dir.expanduser().resolve()
+ print(f"snapshot: {d}")
+ if not d.is_dir():
+ print(" NOT A DIRECTORY")
+ return False
+
+ print(f" model: {config_summary(d) or 'config.json MISSING'}")
+ print(f" kind: {model_kind(d)}")
+ pipeline = modules_pipeline(d)
+ if pipeline:
+ print(f" pipeline: {pipeline}")
+
+ family = tokenizer_family(d)
+ print(f" tokenizer: {family}")
+ for name in TOKENIZER_ARTIFACTS:
+ if (d / name).exists():
+ print(fmt(name, d / name))
+
+ print(" weights:")
+ wlines, has_weights = weight_lines(d)
+ print("\n".join(wlines))
+
+ extras = [name for name in EXTRA_ARTIFACTS if (d / name).exists()]
+ if extras:
+ print(" extras:")
+ for name in extras:
+ print(fmt(name, d / name))
+
+ ok = (d / "config.json").exists() and has_weights and family != "none"
+ print(f" verdict: {'ok' if ok else 'incomplete'}")
+ return ok
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR")
+ args = ap.parse_args()
+
+ ok = True
+ first = True
+ for name in args.model_dirs:
+ root = Path(os.path.expandvars(name)).expanduser().resolve()
+ found = discover(root)
+ if not found:
+ if not first:
+ print()
+ print(f"snapshot: {root}\n no config.json found at or under this path")
+ ok = first = False
+ continue
+ if len(found) > 1:
+ if not first:
+ print()
+ print(f"# {root}: {len(found)} models")
+ first = False
+ for d in found:
+ if not first:
+ print()
+ ok = inspect(d) and ok
+ first = False
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())