aboutsummaryrefslogtreecommitdiff
path: root/tools/inspect_model_artifacts.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-12 05:12:04 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-14 08:27:10 +0200
commit9eb4b29f2f8d29b93bdd8a3224a2b0d9143d894f (patch)
treed9fcb899ebfe3ef2f8fa35d693e5992b1fa336ab /tools/inspect_model_artifacts.py
parent978f4f7969190265e127e8a3d704bb173fbd171c (diff)
downloadffwd-9eb4b29f2f8d29b93bdd8a3224a2b0d9143d894f.tar.gz
ffwd-9eb4b29f2f8d29b93bdd8a3224a2b0d9143d894f.zip
tools: add tools for model conversion/inspection and ranking embeddings
Diffstat (limited to 'tools/inspect_model_artifacts.py')
-rwxr-xr-xtools/inspect_model_artifacts.py88
1 files changed, 88 insertions, 0 deletions
diff --git a/tools/inspect_model_artifacts.py b/tools/inspect_model_artifacts.py
new file mode 100755
index 0000000..2feabe2
--- /dev/null
+++ b/tools/inspect_model_artifacts.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Inspect local pplx-embed snapshots."""
+
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+
+
+PRIMARY_FILES = (
+ "config.json",
+ "tokenizer_config.json",
+ "tokenizer.json",
+ "vocab.json",
+ "merges.txt",
+)
+
+
+def human_bytes(size: int) -> str:
+ units = ("B", "KiB", "MiB", "GiB", "TiB")
+ value = float(size)
+ for unit in units:
+ if value < 1024.0 or unit == units[-1]:
+ return f"{value:.1f} {unit}"
+ value /= 1024.0
+ raise AssertionError("unreachable")
+
+
+def describe(path: Path) -> str:
+ if not path.exists():
+ return "MISSING"
+ resolved = path.resolve()
+ suffix = ""
+ if path.is_symlink():
+ suffix = f" -> {resolved}"
+ return f"{human_bytes(resolved.stat().st_size):>10}{suffix}"
+
+
+def inspect(model_dir: Path) -> bool:
+ model_dir = model_dir.expanduser().resolve()
+ print(f"snapshot: {model_dir}")
+ ok = True
+ for name in PRIMARY_FILES:
+ path = model_dir / name
+ print(f" {name:<30} {describe(path)}")
+ ok = path.exists() and ok
+
+ model = model_dir / "model.safetensors"
+ index = model_dir / "model.safetensors.index.json"
+ shards = sorted(model_dir.glob("model-*-of-*.safetensors"))
+ if model.exists():
+ print(f" {'model.safetensors':<30} {describe(model)}")
+ elif index.exists() and shards:
+ print(f" {'model.safetensors.index.json':<30} {describe(index)}")
+ for path in shards:
+ print(f" {path.name:<28} {describe(path)}")
+ else:
+ print(f" {'model.safetensors':<30} MISSING")
+ ok = False
+
+ onnx_dir = model_dir / "onnx"
+ onnx_files = sorted(onnx_dir.glob("*")) if onnx_dir.is_dir() else []
+ print(" onnx/")
+ if not onnx_files:
+ print(" MISSING")
+ return False
+ for path in onnx_files:
+ print(f" {path.name:<28} {describe(path)}")
+ ok = path.exists() and ok
+ return ok
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR")
+ args = ap.parse_args()
+
+ ok = True
+ for i, name in enumerate(args.model_dirs):
+ if i:
+ print()
+ ok = inspect(Path(os.path.expandvars(name))) and ok
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())