aboutsummaryrefslogtreecommitdiff
path: root/tools/inspect_model_artifacts.py
blob: 2feabe2669bdbfdf3f24ce6817a1d90187fa9cae (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
#!/usr/bin/env python3
"""Inspect local pplx-embed snapshots."""

from __future__ import annotations

import argparse
import os
from pathlib import Path


PRIMARY_FILES = (
    "config.json",
    "tokenizer_config.json",
    "tokenizer.json",
    "vocab.json",
    "merges.txt",
)


def human_bytes(size: int) -> str:
    units = ("B", "KiB", "MiB", "GiB", "TiB")
    value = float(size)
    for unit in units:
        if value < 1024.0 or unit == units[-1]:
            return f"{value:.1f} {unit}"
        value /= 1024.0
    raise AssertionError("unreachable")


def describe(path: Path) -> str:
    if not path.exists():
        return "MISSING"
    resolved = path.resolve()
    suffix = ""
    if path.is_symlink():
        suffix = f" -> {resolved}"
    return f"{human_bytes(resolved.stat().st_size):>10}{suffix}"


def inspect(model_dir: Path) -> bool:
    model_dir = model_dir.expanduser().resolve()
    print(f"snapshot: {model_dir}")
    ok = True
    for name in PRIMARY_FILES:
        path = model_dir / name
        print(f"  {name:<30} {describe(path)}")
        ok = path.exists() and ok

    model = model_dir / "model.safetensors"
    index = model_dir / "model.safetensors.index.json"
    shards = sorted(model_dir.glob("model-*-of-*.safetensors"))
    if model.exists():
        print(f"  {'model.safetensors':<30} {describe(model)}")
    elif index.exists() and shards:
        print(f"  {'model.safetensors.index.json':<30} {describe(index)}")
        for path in shards:
            print(f"    {path.name:<28} {describe(path)}")
    else:
        print(f"  {'model.safetensors':<30} MISSING")
        ok = False

    onnx_dir = model_dir / "onnx"
    onnx_files = sorted(onnx_dir.glob("*")) if onnx_dir.is_dir() else []
    print("  onnx/")
    if not onnx_files:
        print("    MISSING")
        return False
    for path in onnx_files:
        print(f"    {path.name:<28} {describe(path)}")
        ok = path.exists() and ok
    return ok


def main() -> int:
    ap = argparse.ArgumentParser(description=__doc__)
    ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR")
    args = ap.parse_args()

    ok = True
    for i, name in enumerate(args.model_dirs):
        if i:
            print()
        ok = inspect(Path(os.path.expandvars(name))) and ok
    return 0 if ok else 1


if __name__ == "__main__":
    raise SystemExit(main())