diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:31:06 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | dbdf714bb57578431924e5d069089bdfd9f4c604 (patch) | |
| tree | 4f5b59ef132220310e1e4b1dde64f5e38782369a /tools/inspect_model_artifacts.py | |
| parent | e9ecaeaf21050f39888d33d2c8f4216d595d4975 (diff) | |
| download | ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.tar.gz ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.zip | |
devtools: add scripts for unicode generation, model inspection, embedding ranking, and more
Diffstat (limited to 'tools/inspect_model_artifacts.py')
| -rwxr-xr-x | tools/inspect_model_artifacts.py | 88 |
1 files changed, 0 insertions, 88 deletions
diff --git a/tools/inspect_model_artifacts.py b/tools/inspect_model_artifacts.py deleted file mode 100755 index 2feabe2..0000000 --- a/tools/inspect_model_artifacts.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -"""Inspect local pplx-embed snapshots.""" - -from __future__ import annotations - -import argparse -import os -from pathlib import Path - - -PRIMARY_FILES = ( - "config.json", - "tokenizer_config.json", - "tokenizer.json", - "vocab.json", - "merges.txt", -) - - -def human_bytes(size: int) -> str: - units = ("B", "KiB", "MiB", "GiB", "TiB") - value = float(size) - for unit in units: - if value < 1024.0 or unit == units[-1]: - return f"{value:.1f} {unit}" - value /= 1024.0 - raise AssertionError("unreachable") - - -def describe(path: Path) -> str: - if not path.exists(): - return "MISSING" - resolved = path.resolve() - suffix = "" - if path.is_symlink(): - suffix = f" -> {resolved}" - return f"{human_bytes(resolved.stat().st_size):>10}{suffix}" - - -def inspect(model_dir: Path) -> bool: - model_dir = model_dir.expanduser().resolve() - print(f"snapshot: {model_dir}") - ok = True - for name in PRIMARY_FILES: - path = model_dir / name - print(f" {name:<30} {describe(path)}") - ok = path.exists() and ok - - model = model_dir / "model.safetensors" - index = model_dir / "model.safetensors.index.json" - shards = sorted(model_dir.glob("model-*-of-*.safetensors")) - if model.exists(): - print(f" {'model.safetensors':<30} {describe(model)}") - elif index.exists() and shards: - print(f" {'model.safetensors.index.json':<30} {describe(index)}") - for path in shards: - print(f" {path.name:<28} {describe(path)}") - else: - print(f" {'model.safetensors':<30} MISSING") - ok = False - - onnx_dir = model_dir / "onnx" - onnx_files = sorted(onnx_dir.glob("*")) if onnx_dir.is_dir() else [] - print(" onnx/") - if not onnx_files: - print(" MISSING") - return False - for path in onnx_files: - print(f" {path.name:<28} {describe(path)}") - ok = path.exists() and ok - return ok - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR") - args = ap.parse_args() - - ok = True - for i, name in enumerate(args.model_dirs): - if i: - print() - ok = inspect(Path(os.path.expandvars(name))) and ok - return 0 if ok else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) |
