aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rwxr-xr-xtools/convert_model_bf16.py195
-rwxr-xr-xtools/inspect_model_artifacts.py88
-rwxr-xr-xtools/rank_embeddings.py94
3 files changed, 377 insertions, 0 deletions
diff --git a/tools/convert_model_bf16.py b/tools/convert_model_bf16.py
new file mode 100755
index 0000000..d59ccf2
--- /dev/null
+++ b/tools/convert_model_bf16.py
@@ -0,0 +1,195 @@
+#!/usr/bin/env python3
+"""Convert a local safetensors model directory to BF16.
+
+The output directory keeps non-safetensors files as symlinks by default and
+writes converted safetensors files. F32 tensors are converted to BF16 with
+round-to-nearest-even. Existing BF16 tensors are copied unchanged.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import shutil
+import struct
+from pathlib import Path
+
+import numpy as np
+
+
+COPY_CHUNK = 64 * 1024 * 1024
+F32_CHUNK_ELEMS = 16 * 1024 * 1024
+
+
+def read_safetensors_header(path: Path) -> tuple[dict, int]:
+ with path.open("rb") as f:
+ raw = f.read(8)
+ if len(raw) != 8:
+ raise ValueError(f"{path}: too small for safetensors header")
+ header_len = struct.unpack("<Q", raw)[0]
+ header_raw = f.read(header_len)
+ if len(header_raw) != header_len:
+ raise ValueError(f"{path}: truncated safetensors header")
+ return json.loads(header_raw), 8 + header_len
+
+
+def tensor_nbytes(spec: dict) -> int:
+ start, end = spec["data_offsets"]
+ return int(end) - int(start)
+
+
+def converted_tensor_nbytes(spec: dict) -> int:
+ if spec["dtype"] == "F32":
+ return tensor_nbytes(spec) // 2
+ return tensor_nbytes(spec)
+
+
+def build_output_header(header: dict) -> tuple[dict, dict[str, tuple[int, int]]]:
+ out_header = {}
+ offsets = {}
+ cursor = 0
+
+ for name, spec in header.items():
+ if name == "__metadata__":
+ out_header[name] = spec
+ continue
+
+ out_spec = dict(spec)
+ nbytes = converted_tensor_nbytes(spec)
+ offsets[name] = (cursor, cursor + nbytes)
+ out_spec["data_offsets"] = [cursor, cursor + nbytes]
+ if spec["dtype"] == "F32":
+ out_spec["dtype"] = "BF16"
+ out_header[name] = out_spec
+ cursor += nbytes
+
+ return out_header, offsets
+
+
+def copy_range(src, dst, nbytes: int) -> None:
+ left = nbytes
+ while left:
+ chunk = src.read(min(left, COPY_CHUNK))
+ if not chunk:
+ raise IOError("unexpected EOF while copying tensor data")
+ dst.write(chunk)
+ left -= len(chunk)
+
+
+def write_f32_as_bf16(src, dst, nbytes: int) -> None:
+ if nbytes % 4 != 0:
+ raise ValueError("F32 tensor byte size is not divisible by 4")
+
+ elems_left = nbytes // 4
+ while elems_left:
+ count = min(elems_left, F32_CHUNK_ELEMS)
+ raw = src.read(count * 4)
+ if len(raw) != count * 4:
+ raise IOError("unexpected EOF while converting F32 tensor data")
+
+ bits = np.frombuffer(raw, dtype="<u4").astype(np.uint32, copy=True)
+ lsb = (bits >> np.uint32(16)) & np.uint32(1)
+ bf16 = ((bits + np.uint32(0x7FFF) + lsb) >> np.uint32(16)).astype("<u2")
+ dst.write(bf16.tobytes())
+ elems_left -= count
+
+
+def convert_safetensors(src_path: Path, dst_path: Path,
+ verbose: bool = False) -> tuple[int, int]:
+ header, data_start = read_safetensors_header(src_path)
+ out_header, _offsets = build_output_header(header)
+ out_header_raw = json.dumps(out_header, separators=(",", ":")).encode("utf-8")
+
+ tensors = [(name, spec) for name, spec in header.items() if name != "__metadata__"]
+ input_bytes = sum(tensor_nbytes(spec) for _, spec in tensors)
+ output_bytes = sum(converted_tensor_nbytes(spec) for _, spec in tensors)
+
+ dst_path.parent.mkdir(parents=True, exist_ok=True)
+ tmp_path = dst_path.with_suffix(dst_path.suffix + ".tmp")
+ with src_path.open("rb") as src, tmp_path.open("wb") as dst:
+ dst.write(struct.pack("<Q", len(out_header_raw)))
+ dst.write(out_header_raw)
+
+ for name, spec in tensors:
+ start, _end = spec["data_offsets"]
+ src.seek(data_start + int(start))
+ nbytes = tensor_nbytes(spec)
+ dtype = spec["dtype"]
+ if dtype == "F32":
+ write_f32_as_bf16(src, dst, nbytes)
+ elif dtype == "BF16":
+ copy_range(src, dst, nbytes)
+ else:
+ copy_range(src, dst, nbytes)
+ if verbose:
+ print(f" {name}: {dtype} {nbytes / (1024 ** 2):.1f} MB", flush=True)
+
+ tmp_path.replace(dst_path)
+ return input_bytes, output_bytes
+
+
+def link_or_copy(src: Path, dst: Path, copy_files: bool) -> None:
+ dst.parent.mkdir(parents=True, exist_ok=True)
+ if copy_files:
+ if src.is_dir():
+ shutil.copytree(src, dst, symlinks=True)
+ else:
+ shutil.copy2(src, dst)
+ else:
+ target = src.resolve()
+ os.symlink(target, dst, target_is_directory=src.is_dir())
+
+
+def convert_model(src_dir: Path, dst_dir: Path, copy_files: bool,
+ verbose: bool = False) -> None:
+ if not src_dir.is_dir():
+ raise SystemExit(f"source model dir does not exist: {src_dir}")
+ if dst_dir.exists() and any(dst_dir.iterdir()):
+ raise SystemExit(f"output dir already exists and is not empty: {dst_dir}")
+ dst_dir.mkdir(parents=True, exist_ok=True)
+
+ safetensors = sorted(src_dir.glob("*.safetensors"))
+ if not safetensors:
+ raise SystemExit(f"no safetensors files found in {src_dir}")
+
+ total_in = 0
+ total_out = 0
+ for src_path in safetensors:
+ dst_path = dst_dir / src_path.name
+ print(f"converting {src_path.name}", flush=True)
+ n_in, n_out = convert_safetensors(src_path.resolve(), dst_path, verbose)
+ total_in += n_in
+ total_out += n_out
+
+ for child in src_dir.iterdir():
+ if child.name.endswith(".safetensors"):
+ continue
+ dst_child = dst_dir / child.name
+ if dst_child.exists() or dst_child.is_symlink():
+ continue
+ link_or_copy(child, dst_child, copy_files)
+
+ print(
+ f"done: tensor data {total_in / (1024 ** 3):.2f} GiB -> "
+ f"{total_out / (1024 ** 3):.2f} GiB",
+ flush=True,
+ )
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--src", required=True, help="source model directory")
+ ap.add_argument("--dst", required=True, help="output model directory")
+ ap.add_argument("--copy-files", action="store_true",
+ help="copy non-safetensors files instead of symlinking")
+ ap.add_argument("-v", "--verbose", action="store_true",
+ help="print every tensor as it is converted")
+ args = ap.parse_args()
+
+ convert_model(Path(args.src), Path(args.dst), args.copy_files, args.verbose)
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/inspect_model_artifacts.py b/tools/inspect_model_artifacts.py
new file mode 100755
index 0000000..2feabe2
--- /dev/null
+++ b/tools/inspect_model_artifacts.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+"""Inspect local pplx-embed snapshots."""
+
+from __future__ import annotations
+
+import argparse
+import os
+from pathlib import Path
+
+
+PRIMARY_FILES = (
+ "config.json",
+ "tokenizer_config.json",
+ "tokenizer.json",
+ "vocab.json",
+ "merges.txt",
+)
+
+
+def human_bytes(size: int) -> str:
+ units = ("B", "KiB", "MiB", "GiB", "TiB")
+ value = float(size)
+ for unit in units:
+ if value < 1024.0 or unit == units[-1]:
+ return f"{value:.1f} {unit}"
+ value /= 1024.0
+ raise AssertionError("unreachable")
+
+
+def describe(path: Path) -> str:
+ if not path.exists():
+ return "MISSING"
+ resolved = path.resolve()
+ suffix = ""
+ if path.is_symlink():
+ suffix = f" -> {resolved}"
+ return f"{human_bytes(resolved.stat().st_size):>10}{suffix}"
+
+
+def inspect(model_dir: Path) -> bool:
+ model_dir = model_dir.expanduser().resolve()
+ print(f"snapshot: {model_dir}")
+ ok = True
+ for name in PRIMARY_FILES:
+ path = model_dir / name
+ print(f" {name:<30} {describe(path)}")
+ ok = path.exists() and ok
+
+ model = model_dir / "model.safetensors"
+ index = model_dir / "model.safetensors.index.json"
+ shards = sorted(model_dir.glob("model-*-of-*.safetensors"))
+ if model.exists():
+ print(f" {'model.safetensors':<30} {describe(model)}")
+ elif index.exists() and shards:
+ print(f" {'model.safetensors.index.json':<30} {describe(index)}")
+ for path in shards:
+ print(f" {path.name:<28} {describe(path)}")
+ else:
+ print(f" {'model.safetensors':<30} MISSING")
+ ok = False
+
+ onnx_dir = model_dir / "onnx"
+ onnx_files = sorted(onnx_dir.glob("*")) if onnx_dir.is_dir() else []
+ print(" onnx/")
+ if not onnx_files:
+ print(" MISSING")
+ return False
+ for path in onnx_files:
+ print(f" {path.name:<28} {describe(path)}")
+ ok = path.exists() and ok
+ return ok
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR")
+ args = ap.parse_args()
+
+ ok = True
+ for i, name in enumerate(args.model_dirs):
+ if i:
+ print()
+ ok = inspect(Path(os.path.expandvars(name))) and ok
+ return 0 if ok else 1
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())
diff --git a/tools/rank_embeddings.py b/tools/rank_embeddings.py
new file mode 100755
index 0000000..c547a41
--- /dev/null
+++ b/tools/rank_embeddings.py
@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Rank stdin JSON embeddings by cosine similarity to a query row.
+
+Usage:
+ ./pplx-embed ... --stream <<'EOF' | scripts/rank_embeddings.py
+ query: what is the capital of France?
+ document: Paris is the capital of France.
+ document: Berlin is the capital of Germany.
+ EOF
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import sys
+
+
+def cosine(a: list[float], b: list[float]) -> float:
+ dot = 0.0
+ aa = 0.0
+ bb = 0.0
+ for x, y in zip(a, b):
+ dot += x * y
+ aa += x * x
+ bb += y * y
+ if aa == 0.0 or bb == 0.0:
+ return float("nan")
+ return dot / math.sqrt(aa * bb)
+
+
+def read_rows() -> list[dict]:
+ rows = []
+ for lineno, line in enumerate(sys.stdin, 1):
+ line = line.strip()
+ if not line:
+ continue
+ try:
+ row = json.loads(line)
+ except json.JSONDecodeError as exc:
+ raise SystemExit(f"line {lineno}: invalid JSON: {exc}") from exc
+ if "error" in row:
+ raise SystemExit(f"line {lineno}: stdin error: {row['error']}")
+ emb = row.get("embedding")
+ if not isinstance(emb, list) or not emb:
+ raise SystemExit(f"line {lineno}: missing embedding array")
+ rows.append(row)
+ return rows
+
+
+def main() -> int:
+ ap = argparse.ArgumentParser(description=__doc__)
+ ap.add_argument("--query-index", type=int, default=0,
+ help="embedding row to use as query, default: 0")
+ ap.add_argument("--include-query", action="store_true",
+ help="include the query row in the ranked output")
+ ap.add_argument("--labels", nargs="*",
+ help="optional labels for rows, in input order")
+ args = ap.parse_args()
+
+ rows = read_rows()
+ if not rows:
+ raise SystemExit("no embeddings on stdin")
+ if args.query_index < 0 or args.query_index >= len(rows):
+ raise SystemExit(f"--query-index out of range for {len(rows)} rows")
+ if args.labels and len(args.labels) != len(rows):
+ raise SystemExit("--labels count must match embedding row count")
+
+ query = rows[args.query_index]["embedding"]
+ ranked = []
+ for i, row in enumerate(rows):
+ if i == args.query_index and not args.include_query:
+ continue
+ emb = row["embedding"]
+ if len(emb) != len(query):
+ raise SystemExit(f"row {i}: dimension mismatch")
+ ranked.append((cosine(query, emb), i, row))
+
+ ranked.sort(key=lambda item: item[0], reverse=True)
+
+ print(f"query row: {args.query_index}")
+ print("rank row cosine tokens label")
+ print("---- --- ---------- ------ -----")
+ for rank, (score, i, row) in enumerate(ranked, 1):
+ label = args.labels[i] if args.labels else f"row {i}"
+ tokens = row.get("tokens", "-")
+ print(f"{rank:>4} {i:>3} {score:>10.6f} {tokens:>6} {label}")
+
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())