From 9eb4b29f2f8d29b93bdd8a3224a2b0d9143d894f Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Fri, 12 Jun 2026 05:12:04 +0200 Subject: tools: add tools for model conversion/inspection and ranking embeddings --- tools/convert_model_bf16.py | 195 +++++++++++++++++++++++++++++++++++++++ tools/inspect_model_artifacts.py | 88 ++++++++++++++++++ tools/rank_embeddings.py | 94 +++++++++++++++++++ 3 files changed, 377 insertions(+) create mode 100755 tools/convert_model_bf16.py create mode 100755 tools/inspect_model_artifacts.py create mode 100755 tools/rank_embeddings.py diff --git a/tools/convert_model_bf16.py b/tools/convert_model_bf16.py new file mode 100755 index 0000000..d59ccf2 --- /dev/null +++ b/tools/convert_model_bf16.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Convert a local safetensors model directory to BF16. + +The output directory keeps non-safetensors files as symlinks by default and +writes converted safetensors files. F32 tensors are converted to BF16 with +round-to-nearest-even. Existing BF16 tensors are copied unchanged. +""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import struct +from pathlib import Path + +import numpy as np + + +COPY_CHUNK = 64 * 1024 * 1024 +F32_CHUNK_ELEMS = 16 * 1024 * 1024 + + +def read_safetensors_header(path: Path) -> tuple[dict, int]: + with path.open("rb") as f: + raw = f.read(8) + if len(raw) != 8: + raise ValueError(f"{path}: too small for safetensors header") + header_len = struct.unpack(" int: + start, end = spec["data_offsets"] + return int(end) - int(start) + + +def converted_tensor_nbytes(spec: dict) -> int: + if spec["dtype"] == "F32": + return tensor_nbytes(spec) // 2 + return tensor_nbytes(spec) + + +def build_output_header(header: dict) -> tuple[dict, dict[str, tuple[int, int]]]: + out_header = {} + offsets = {} + cursor = 0 + + for name, spec in header.items(): + if name == "__metadata__": + out_header[name] = spec + continue + + out_spec = dict(spec) + nbytes = converted_tensor_nbytes(spec) + offsets[name] = (cursor, cursor + nbytes) + out_spec["data_offsets"] = [cursor, cursor + nbytes] + if spec["dtype"] == "F32": + out_spec["dtype"] = "BF16" + out_header[name] = out_spec + cursor += nbytes + + return out_header, offsets + + +def copy_range(src, dst, nbytes: int) -> None: + left = nbytes + while left: + chunk = src.read(min(left, COPY_CHUNK)) + if not chunk: + raise IOError("unexpected EOF while copying tensor data") + dst.write(chunk) + left -= len(chunk) + + +def write_f32_as_bf16(src, dst, nbytes: int) -> None: + if nbytes % 4 != 0: + raise ValueError("F32 tensor byte size is not divisible by 4") + + elems_left = nbytes // 4 + while elems_left: + count = min(elems_left, F32_CHUNK_ELEMS) + raw = src.read(count * 4) + if len(raw) != count * 4: + raise IOError("unexpected EOF while converting F32 tensor data") + + bits = np.frombuffer(raw, dtype="> np.uint32(16)) & np.uint32(1) + bf16 = ((bits + np.uint32(0x7FFF) + lsb) >> np.uint32(16)).astype(" tuple[int, int]: + header, data_start = read_safetensors_header(src_path) + out_header, _offsets = build_output_header(header) + out_header_raw = json.dumps(out_header, separators=(",", ":")).encode("utf-8") + + tensors = [(name, spec) for name, spec in header.items() if name != "__metadata__"] + input_bytes = sum(tensor_nbytes(spec) for _, spec in tensors) + output_bytes = sum(converted_tensor_nbytes(spec) for _, spec in tensors) + + dst_path.parent.mkdir(parents=True, exist_ok=True) + tmp_path = dst_path.with_suffix(dst_path.suffix + ".tmp") + with src_path.open("rb") as src, tmp_path.open("wb") as dst: + dst.write(struct.pack(" None: + dst.parent.mkdir(parents=True, exist_ok=True) + if copy_files: + if src.is_dir(): + shutil.copytree(src, dst, symlinks=True) + else: + shutil.copy2(src, dst) + else: + target = src.resolve() + os.symlink(target, dst, target_is_directory=src.is_dir()) + + +def convert_model(src_dir: Path, dst_dir: Path, copy_files: bool, + verbose: bool = False) -> None: + if not src_dir.is_dir(): + raise SystemExit(f"source model dir does not exist: {src_dir}") + if dst_dir.exists() and any(dst_dir.iterdir()): + raise SystemExit(f"output dir already exists and is not empty: {dst_dir}") + dst_dir.mkdir(parents=True, exist_ok=True) + + safetensors = sorted(src_dir.glob("*.safetensors")) + if not safetensors: + raise SystemExit(f"no safetensors files found in {src_dir}") + + total_in = 0 + total_out = 0 + for src_path in safetensors: + dst_path = dst_dir / src_path.name + print(f"converting {src_path.name}", flush=True) + n_in, n_out = convert_safetensors(src_path.resolve(), dst_path, verbose) + total_in += n_in + total_out += n_out + + for child in src_dir.iterdir(): + if child.name.endswith(".safetensors"): + continue + dst_child = dst_dir / child.name + if dst_child.exists() or dst_child.is_symlink(): + continue + link_or_copy(child, dst_child, copy_files) + + print( + f"done: tensor data {total_in / (1024 ** 3):.2f} GiB -> " + f"{total_out / (1024 ** 3):.2f} GiB", + flush=True, + ) + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--src", required=True, help="source model directory") + ap.add_argument("--dst", required=True, help="output model directory") + ap.add_argument("--copy-files", action="store_true", + help="copy non-safetensors files instead of symlinking") + ap.add_argument("-v", "--verbose", action="store_true", + help="print every tensor as it is converted") + args = ap.parse_args() + + convert_model(Path(args.src), Path(args.dst), args.copy_files, args.verbose) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/inspect_model_artifacts.py b/tools/inspect_model_artifacts.py new file mode 100755 index 0000000..2feabe2 --- /dev/null +++ b/tools/inspect_model_artifacts.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""Inspect local pplx-embed snapshots.""" + +from __future__ import annotations + +import argparse +import os +from pathlib import Path + + +PRIMARY_FILES = ( + "config.json", + "tokenizer_config.json", + "tokenizer.json", + "vocab.json", + "merges.txt", +) + + +def human_bytes(size: int) -> str: + units = ("B", "KiB", "MiB", "GiB", "TiB") + value = float(size) + for unit in units: + if value < 1024.0 or unit == units[-1]: + return f"{value:.1f} {unit}" + value /= 1024.0 + raise AssertionError("unreachable") + + +def describe(path: Path) -> str: + if not path.exists(): + return "MISSING" + resolved = path.resolve() + suffix = "" + if path.is_symlink(): + suffix = f" -> {resolved}" + return f"{human_bytes(resolved.stat().st_size):>10}{suffix}" + + +def inspect(model_dir: Path) -> bool: + model_dir = model_dir.expanduser().resolve() + print(f"snapshot: {model_dir}") + ok = True + for name in PRIMARY_FILES: + path = model_dir / name + print(f" {name:<30} {describe(path)}") + ok = path.exists() and ok + + model = model_dir / "model.safetensors" + index = model_dir / "model.safetensors.index.json" + shards = sorted(model_dir.glob("model-*-of-*.safetensors")) + if model.exists(): + print(f" {'model.safetensors':<30} {describe(model)}") + elif index.exists() and shards: + print(f" {'model.safetensors.index.json':<30} {describe(index)}") + for path in shards: + print(f" {path.name:<28} {describe(path)}") + else: + print(f" {'model.safetensors':<30} MISSING") + ok = False + + onnx_dir = model_dir / "onnx" + onnx_files = sorted(onnx_dir.glob("*")) if onnx_dir.is_dir() else [] + print(" onnx/") + if not onnx_files: + print(" MISSING") + return False + for path in onnx_files: + print(f" {path.name:<28} {describe(path)}") + ok = path.exists() and ok + return ok + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR") + args = ap.parse_args() + + ok = True + for i, name in enumerate(args.model_dirs): + if i: + print() + ok = inspect(Path(os.path.expandvars(name))) and ok + return 0 if ok else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/rank_embeddings.py b/tools/rank_embeddings.py new file mode 100755 index 0000000..c547a41 --- /dev/null +++ b/tools/rank_embeddings.py @@ -0,0 +1,94 @@ +#!/usr/bin/env python3 +"""Rank stdin JSON embeddings by cosine similarity to a query row. + +Usage: + ./pplx-embed ... --stream <<'EOF' | scripts/rank_embeddings.py + query: what is the capital of France? + document: Paris is the capital of France. + document: Berlin is the capital of Germany. + EOF +""" + +from __future__ import annotations + +import argparse +import json +import math +import sys + + +def cosine(a: list[float], b: list[float]) -> float: + dot = 0.0 + aa = 0.0 + bb = 0.0 + for x, y in zip(a, b): + dot += x * y + aa += x * x + bb += y * y + if aa == 0.0 or bb == 0.0: + return float("nan") + return dot / math.sqrt(aa * bb) + + +def read_rows() -> list[dict]: + rows = [] + for lineno, line in enumerate(sys.stdin, 1): + line = line.strip() + if not line: + continue + try: + row = json.loads(line) + except json.JSONDecodeError as exc: + raise SystemExit(f"line {lineno}: invalid JSON: {exc}") from exc + if "error" in row: + raise SystemExit(f"line {lineno}: stdin error: {row['error']}") + emb = row.get("embedding") + if not isinstance(emb, list) or not emb: + raise SystemExit(f"line {lineno}: missing embedding array") + rows.append(row) + return rows + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__) + ap.add_argument("--query-index", type=int, default=0, + help="embedding row to use as query, default: 0") + ap.add_argument("--include-query", action="store_true", + help="include the query row in the ranked output") + ap.add_argument("--labels", nargs="*", + help="optional labels for rows, in input order") + args = ap.parse_args() + + rows = read_rows() + if not rows: + raise SystemExit("no embeddings on stdin") + if args.query_index < 0 or args.query_index >= len(rows): + raise SystemExit(f"--query-index out of range for {len(rows)} rows") + if args.labels and len(args.labels) != len(rows): + raise SystemExit("--labels count must match embedding row count") + + query = rows[args.query_index]["embedding"] + ranked = [] + for i, row in enumerate(rows): + if i == args.query_index and not args.include_query: + continue + emb = row["embedding"] + if len(emb) != len(query): + raise SystemExit(f"row {i}: dimension mismatch") + ranked.append((cosine(query, emb), i, row)) + + ranked.sort(key=lambda item: item[0], reverse=True) + + print(f"query row: {args.query_index}") + print("rank row cosine tokens label") + print("---- --- ---------- ------ -----") + for rank, (score, i, row) in enumerate(ranked, 1): + label = args.labels[i] if args.labels else f"row {i}" + tokens = row.get("tokens", "-") + print(f"{rank:>4} {i:>3} {score:>10.6f} {tokens:>6} {label}") + + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) -- cgit v1.3.1