From dbdf714bb57578431924e5d069089bdfd9f4c604 Mon Sep 17 00:00:00 2001 From: Onur Gündüz Date: Sat, 20 Jun 2026 20:31:06 +0200 Subject: devtools: add scripts for unicode generation, model inspection, embedding ranking, and more --- tools/convert_model_bf16.py | 195 ------------------------------ tools/gen_wordpiece_unicode.py | 139 ---------------------- tools/inspect_model_artifacts.py | 88 -------------- tools/openapi.yaml | 249 --------------------------------------- tools/rank_embeddings.py | 94 --------------- 5 files changed, 765 deletions(-) delete mode 100755 tools/convert_model_bf16.py delete mode 100644 tools/gen_wordpiece_unicode.py delete mode 100755 tools/inspect_model_artifacts.py delete mode 100644 tools/openapi.yaml delete mode 100755 tools/rank_embeddings.py (limited to 'tools') diff --git a/tools/convert_model_bf16.py b/tools/convert_model_bf16.py deleted file mode 100755 index d59ccf2..0000000 --- a/tools/convert_model_bf16.py +++ /dev/null @@ -1,195 +0,0 @@ -#!/usr/bin/env python3 -"""Convert a local safetensors model directory to BF16. - -The output directory keeps non-safetensors files as symlinks by default and -writes converted safetensors files. F32 tensors are converted to BF16 with -round-to-nearest-even. Existing BF16 tensors are copied unchanged. -""" - -from __future__ import annotations - -import argparse -import json -import os -import shutil -import struct -from pathlib import Path - -import numpy as np - - -COPY_CHUNK = 64 * 1024 * 1024 -F32_CHUNK_ELEMS = 16 * 1024 * 1024 - - -def read_safetensors_header(path: Path) -> tuple[dict, int]: - with path.open("rb") as f: - raw = f.read(8) - if len(raw) != 8: - raise ValueError(f"{path}: too small for safetensors header") - header_len = struct.unpack(" int: - start, end = spec["data_offsets"] - return int(end) - int(start) - - -def converted_tensor_nbytes(spec: dict) -> int: - if spec["dtype"] == "F32": - return tensor_nbytes(spec) // 2 - return tensor_nbytes(spec) - - -def build_output_header(header: dict) -> tuple[dict, dict[str, tuple[int, int]]]: - out_header = {} - offsets = {} - cursor = 0 - - for name, spec in header.items(): - if name == "__metadata__": - out_header[name] = spec - continue - - out_spec = dict(spec) - nbytes = converted_tensor_nbytes(spec) - offsets[name] = (cursor, cursor + nbytes) - out_spec["data_offsets"] = [cursor, cursor + nbytes] - if spec["dtype"] == "F32": - out_spec["dtype"] = "BF16" - out_header[name] = out_spec - cursor += nbytes - - return out_header, offsets - - -def copy_range(src, dst, nbytes: int) -> None: - left = nbytes - while left: - chunk = src.read(min(left, COPY_CHUNK)) - if not chunk: - raise IOError("unexpected EOF while copying tensor data") - dst.write(chunk) - left -= len(chunk) - - -def write_f32_as_bf16(src, dst, nbytes: int) -> None: - if nbytes % 4 != 0: - raise ValueError("F32 tensor byte size is not divisible by 4") - - elems_left = nbytes // 4 - while elems_left: - count = min(elems_left, F32_CHUNK_ELEMS) - raw = src.read(count * 4) - if len(raw) != count * 4: - raise IOError("unexpected EOF while converting F32 tensor data") - - bits = np.frombuffer(raw, dtype="> np.uint32(16)) & np.uint32(1) - bf16 = ((bits + np.uint32(0x7FFF) + lsb) >> np.uint32(16)).astype(" tuple[int, int]: - header, data_start = read_safetensors_header(src_path) - out_header, _offsets = build_output_header(header) - out_header_raw = json.dumps(out_header, separators=(",", ":")).encode("utf-8") - - tensors = [(name, spec) for name, spec in header.items() if name != "__metadata__"] - input_bytes = sum(tensor_nbytes(spec) for _, spec in tensors) - output_bytes = sum(converted_tensor_nbytes(spec) for _, spec in tensors) - - dst_path.parent.mkdir(parents=True, exist_ok=True) - tmp_path = dst_path.with_suffix(dst_path.suffix + ".tmp") - with src_path.open("rb") as src, tmp_path.open("wb") as dst: - dst.write(struct.pack(" None: - dst.parent.mkdir(parents=True, exist_ok=True) - if copy_files: - if src.is_dir(): - shutil.copytree(src, dst, symlinks=True) - else: - shutil.copy2(src, dst) - else: - target = src.resolve() - os.symlink(target, dst, target_is_directory=src.is_dir()) - - -def convert_model(src_dir: Path, dst_dir: Path, copy_files: bool, - verbose: bool = False) -> None: - if not src_dir.is_dir(): - raise SystemExit(f"source model dir does not exist: {src_dir}") - if dst_dir.exists() and any(dst_dir.iterdir()): - raise SystemExit(f"output dir already exists and is not empty: {dst_dir}") - dst_dir.mkdir(parents=True, exist_ok=True) - - safetensors = sorted(src_dir.glob("*.safetensors")) - if not safetensors: - raise SystemExit(f"no safetensors files found in {src_dir}") - - total_in = 0 - total_out = 0 - for src_path in safetensors: - dst_path = dst_dir / src_path.name - print(f"converting {src_path.name}", flush=True) - n_in, n_out = convert_safetensors(src_path.resolve(), dst_path, verbose) - total_in += n_in - total_out += n_out - - for child in src_dir.iterdir(): - if child.name.endswith(".safetensors"): - continue - dst_child = dst_dir / child.name - if dst_child.exists() or dst_child.is_symlink(): - continue - link_or_copy(child, dst_child, copy_files) - - print( - f"done: tensor data {total_in / (1024 ** 3):.2f} GiB -> " - f"{total_out / (1024 ** 3):.2f} GiB", - flush=True, - ) - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--src", required=True, help="source model directory") - ap.add_argument("--dst", required=True, help="output model directory") - ap.add_argument("--copy-files", action="store_true", - help="copy non-safetensors files instead of symlinking") - ap.add_argument("-v", "--verbose", action="store_true", - help="print every tensor as it is converted") - args = ap.parse_args() - - convert_model(Path(args.src), Path(args.dst), args.copy_files, args.verbose) - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py deleted file mode 100644 index 24649f1..0000000 --- a/tools/gen_wordpiece_unicode.py +++ /dev/null @@ -1,139 +0,0 @@ -#!/usr/bin/env python3 -"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece -BasicTokenizer's punctuation and control classification, matching transformers' -BertTokenizer._is_punctuation and _is_control exactly. - -Punctuation is the ASCII set (33-47, 58-64, 91-96, 123-126) plus every codepoint -whose Unicode category starts with "P". Control is categories Cc, Cf, Co minus -tab/newline/carriage-return (which route to whitespace). The tables are emitted -as sorted, merged [lo, hi] ranges for binary search in the C tokenizer. - -Run when bumping the target Unicode version (the C side is data-only otherwise): - - uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h -""" - -import unicodedata - -MAX_CP = 0x110000 - - -def merge(cps): - """Collapse a sorted codepoint list into contiguous [lo, hi] ranges.""" - ranges = [] - for cp in cps: - if ranges and cp == ranges[-1][1] + 1: - ranges[-1][1] = cp - else: - ranges.append([cp, cp]) - return ranges - - -def is_punct(cp): - if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126): - return True - return unicodedata.category(chr(cp)).startswith("P") - - -def is_control(cp): - if cp in (0x09, 0x0A, 0x0D): - return False - # Cc/Cf/Co are the assigned control, format and private-use chars. Cn - # (unassigned) is intentionally excluded: it never occurs in real text and - # would bind the table to a single Unicode version. - return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co") - - -# Hangul syllables decompose by formula (the standard algorithm), so they are -# excluded from the fold table to keep it small; the C side handles them. -HANGUL_BASE, HANGUL_COUNT = 0xAC00, 11172 - - -def fold(cp): - """do_lower_case fold: Python str.lower then NFD then drop combining marks - (Mn), matching transformers' lowercase + _run_strip_accents. Returns the - output codepoint list (empty when the codepoint is a dropped mark).""" - s = chr(cp).lower() - return [ord(c) for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"] - - -def emit_fold(entries): - out = ["static const wp_fold_t WP_FOLD[] = {"] - for cp, f in entries: - outs = ", ".join("0x%X" % c for c in (f + [0, 0, 0])[:3]) - out.append(" {0x%X, {%s}, %d}," % (cp, outs, len(f))) - out.append("};") - out.append("enum { WP_FOLD_N = (int)(sizeof(WP_FOLD) / sizeof(WP_FOLD[0])) };") - return "\n".join(out) - - -def emit(name, ranges): - out = [f"static const wp_range_t {name}[] = {{"] - line = " " - for lo, hi in ranges: - s = "{0x%X, 0x%X}, " % (lo, hi) - if len(line) + len(s) > 96: - out.append(line.rstrip()) - line = " " - line += s - if line.strip(): - out.append(line.rstrip()) - out.append("};") - out.append(f"enum {{ {name}_N = (int)(sizeof({name}) / sizeof({name}[0])) }};") - return "\n".join(out) - - -def main(): - punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)]) - control = merge([cp for cp in range(MAX_CP) if is_control(cp)]) - hangul = range(HANGUL_BASE, HANGUL_BASE + HANGUL_COUNT) - fold_entries = [ - (cp, f) - for cp in range(MAX_CP) - if cp not in hangul - for f in [fold(cp)] - if f != [cp] - ] - print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata - * (Unicode {unicodedata.unidata_version}). Do not edit by hand. - * - * Tables for the BasicTokenizer, matching transformers' BertTokenizer. - * - * WP_PUNCT_RANGES / WP_CONTROL_RANGES: _is_punctuation and _is_control as sorted - * [lo, hi] ranges. Punctuation is the ASCII set (33-47/58-64/91-96/123-126) plus - * every category P* codepoint. Control is categories Cc, Cf, Co minus - * tab/newline/carriage-return (those route to whitespace). Unassigned (Cn) is - * intentionally not control: it never occurs in real text and would bind the - * table to one Unicode version. - * - * WP_FOLD: the do_lower_case fold (Python str.lower then NFD then drop Mn marks), - * sorted by codepoint. n is the output length (0 drops a combining mark; up to - * 3). Hangul syllables (U+AC00..U+D7A3) decompose by formula on the C side and - * are excluded here. */ -#ifndef WORDPIECE_UNICODE_H -#define WORDPIECE_UNICODE_H - -#include - -typedef struct {{ - uint32_t lo; - uint32_t hi; -}} wp_range_t; - -typedef struct {{ - uint32_t cp; - uint32_t out[3]; - uint8_t n; -}} wp_fold_t; - -{emit("WP_PUNCT_RANGES", punct)} - -{emit("WP_CONTROL_RANGES", control)} - -{emit_fold(fold_entries)} - -#endif /* WORDPIECE_UNICODE_H */""") - - -if __name__ == "__main__": - main() diff --git a/tools/inspect_model_artifacts.py b/tools/inspect_model_artifacts.py deleted file mode 100755 index 2feabe2..0000000 --- a/tools/inspect_model_artifacts.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -"""Inspect local pplx-embed snapshots.""" - -from __future__ import annotations - -import argparse -import os -from pathlib import Path - - -PRIMARY_FILES = ( - "config.json", - "tokenizer_config.json", - "tokenizer.json", - "vocab.json", - "merges.txt", -) - - -def human_bytes(size: int) -> str: - units = ("B", "KiB", "MiB", "GiB", "TiB") - value = float(size) - for unit in units: - if value < 1024.0 or unit == units[-1]: - return f"{value:.1f} {unit}" - value /= 1024.0 - raise AssertionError("unreachable") - - -def describe(path: Path) -> str: - if not path.exists(): - return "MISSING" - resolved = path.resolve() - suffix = "" - if path.is_symlink(): - suffix = f" -> {resolved}" - return f"{human_bytes(resolved.stat().st_size):>10}{suffix}" - - -def inspect(model_dir: Path) -> bool: - model_dir = model_dir.expanduser().resolve() - print(f"snapshot: {model_dir}") - ok = True - for name in PRIMARY_FILES: - path = model_dir / name - print(f" {name:<30} {describe(path)}") - ok = path.exists() and ok - - model = model_dir / "model.safetensors" - index = model_dir / "model.safetensors.index.json" - shards = sorted(model_dir.glob("model-*-of-*.safetensors")) - if model.exists(): - print(f" {'model.safetensors':<30} {describe(model)}") - elif index.exists() and shards: - print(f" {'model.safetensors.index.json':<30} {describe(index)}") - for path in shards: - print(f" {path.name:<28} {describe(path)}") - else: - print(f" {'model.safetensors':<30} MISSING") - ok = False - - onnx_dir = model_dir / "onnx" - onnx_files = sorted(onnx_dir.glob("*")) if onnx_dir.is_dir() else [] - print(" onnx/") - if not onnx_files: - print(" MISSING") - return False - for path in onnx_files: - print(f" {path.name:<28} {describe(path)}") - ok = path.exists() and ok - return ok - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR") - args = ap.parse_args() - - ok = True - for i, name in enumerate(args.model_dirs): - if i: - print() - ok = inspect(Path(os.path.expandvars(name))) and ok - return 0 if ok else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/tools/openapi.yaml b/tools/openapi.yaml deleted file mode 100644 index 6dd3818..0000000 --- a/tools/openapi.yaml +++ /dev/null @@ -1,249 +0,0 @@ -openapi: 3.0.3 -info: - title: embed.c server API - version: "1.0.0" - description: > - HTTP API served by `embed-server`. Each loaded model has an operator-chosen - label; request `model` fields match that label. Embedding encoding follows - the serving API selected at load time (`api=openai` or `api=perplexity`). - Always compare embeddings with cosine similarity. - license: - name: MIT -servers: - - url: http://127.0.0.1:8000 -# Auth is optional: it applies only when the server is started with --api-key -# (or EMBED_API_KEY). The empty requirement allows unauthenticated calls. -security: - - {} - - bearerAuth: [] -paths: - /v1/embeddings: - post: - summary: Create pooled embeddings - operationId: createEmbeddings - requestBody: - required: true - content: - application/json: - schema: { $ref: "#/components/schemas/EmbeddingsRequest" } - responses: - "200": - description: One embedding per input, in request order. - content: - application/json: - schema: { $ref: "#/components/schemas/EmbeddingsResponse" } - "422": { $ref: "#/components/responses/ValidationError" } - /v1/contextualizedembeddings: - post: - summary: Create contextual (late-chunking) document embeddings - operationId: createContextualizedEmbeddings - description: > - Each document is a list of chunk strings. Chunks are embedded with - document-level context and returned per document, per chunk, in order. - requestBody: - required: true - content: - application/json: - schema: { $ref: "#/components/schemas/ContextualRequest" } - responses: - "200": - description: One list of chunk embeddings per document, in order. - content: - application/json: - schema: { $ref: "#/components/schemas/ContextualResponse" } - "422": { $ref: "#/components/responses/ValidationError" } - /v1/rerank: - post: - summary: Late-interaction (MaxSim) reranking - operationId: rerank - description: > - Scores each document against the query with token-level MaxSim and - returns documents ranked by raw score (not a probability). - requestBody: - required: true - content: - application/json: - schema: { $ref: "#/components/schemas/RerankRequest" } - responses: - "200": - description: Documents ranked by descending relevance score. - content: - application/json: - schema: { $ref: "#/components/schemas/RerankResponse" } - "422": { $ref: "#/components/responses/ValidationError" } -components: - securitySchemes: - bearerAuth: - type: http - scheme: bearer - description: > - Single shared token set with --api-key. Not a real auth system; - terminate TLS and authenticate at a reverse proxy in production. - schemas: - EmbeddingsRequest: - type: object - required: [model, input] - properties: - model: - type: string - description: Served label chosen when starting `embed-server`. - example: qwen - input: - description: A single string or an array of up to 512 strings. - oneOf: - - type: string - - type: array - items: { type: string } - minItems: 1 - maxItems: 512 - encoding_format: - type: string - description: > - Output encoding, validated against the serving API selected for the - loaded label. `api=openai` accepts `float` (default; true float32) - and `base64` (base64 of little-endian float32). `api=perplexity` - accepts `base64_int8` (default), `base64_binary` (sign bits, needs - `dimensions` divisible by 8), and `float` (the int8-decoded view). - A format outside the label's serving API returns 422. - enum: [float, base64, base64_int8, base64_binary] - dimensions: - type: integer - description: > - Matryoshka truncation, from the configured load-time `min_dim` up to - the model's native dimension. Truncated normalized vectors are - re-normalized. - EmbeddingsResponse: - type: object - properties: - object: { type: string, example: list } - data: - type: array - items: { $ref: "#/components/schemas/Embedding" } - model: { type: string } - usage: { $ref: "#/components/schemas/Usage" } - Embedding: - type: object - properties: - object: { type: string, example: embedding } - index: { type: integer } - embedding: - description: > - A float array when `encoding_format` is `float`, otherwise a base64 - string (`base64`, `base64_int8`, or `base64_binary`). - oneOf: - - type: array - items: { type: number, format: float } - - type: string - ContextualRequest: - type: object - required: [model, input] - properties: - model: - type: string - description: Served contextual-model label. - example: pplx-context - input: - description: Array of documents; each document is an array of chunk strings. - type: array - items: - type: array - items: { type: string } - minItems: 1 - encoding_format: - type: string - enum: [float, base64, base64_int8, base64_binary] - dimensions: { type: integer } - ContextualResponse: - type: object - properties: - object: { type: string, example: list } - data: - type: array - description: One entry per document, in request order. - items: - type: object - properties: - object: { type: string, example: list } - index: { type: integer } - data: - type: array - description: One embedding per chunk, in document order. - items: { $ref: "#/components/schemas/Embedding" } - model: { type: string } - usage: { $ref: "#/components/schemas/Usage" } - RerankRequest: - type: object - required: [model, query, documents] - properties: - model: - type: string - description: Served late-model label. - example: pplx-late - query: { type: string } - documents: - type: array - items: { type: string } - minItems: 1 - maxItems: 1000 - top_n: - type: integer - description: Number of results to return. Defaults to all. `top_k` is an alias. - top_k: - type: integer - return_documents: - type: boolean - default: false - description: Include the source document text in each result. - additionalProperties: false - RerankResponse: - type: object - properties: - object: { type: string, example: list } - model: { type: string } - results: - type: array - items: - type: object - properties: - index: - type: integer - description: Index of the document in the request array. - relevance_score: - type: number - format: float - description: Raw MaxSim score (not a probability). - document: - type: string - description: Present only when `return_documents` is true. - usage: - type: object - properties: - query_tokens: { type: integer } - document_tokens: { type: integer } - total_tokens: { type: integer } - Usage: - type: object - properties: - prompt_tokens: { type: integer } - total_tokens: { type: integer } - ValidationDetail: - type: object - properties: - loc: - type: array - description: Path to the offending field, e.g. ["body","input",0]. - items: {} - msg: { type: string } - type: { type: string } - ValidationError: - type: object - properties: - detail: - type: array - items: { $ref: "#/components/schemas/ValidationDetail" } - responses: - ValidationError: - description: Request body failed validation. - content: - application/json: - schema: { $ref: "#/components/schemas/ValidationError" } diff --git a/tools/rank_embeddings.py b/tools/rank_embeddings.py deleted file mode 100755 index ddc1e42..0000000 --- a/tools/rank_embeddings.py +++ /dev/null @@ -1,94 +0,0 @@ -#!/usr/bin/env python3 -"""Rank stdin JSON embeddings by cosine similarity to a query row. - -Usage: - ./embed ... --stream <<'EOF' | tools/rank_embeddings.py - query: what is the capital of France? - document: Paris is the capital of France. - document: Berlin is the capital of Germany. - EOF -""" - -from __future__ import annotations - -import argparse -import json -import math -import sys - - -def cosine(a: list[float], b: list[float]) -> float: - dot = 0.0 - aa = 0.0 - bb = 0.0 - for x, y in zip(a, b): - dot += x * y - aa += x * x - bb += y * y - if aa == 0.0 or bb == 0.0: - return float("nan") - return dot / math.sqrt(aa * bb) - - -def read_rows() -> list[dict]: - rows = [] - for lineno, line in enumerate(sys.stdin, 1): - line = line.strip() - if not line: - continue - try: - row = json.loads(line) - except json.JSONDecodeError as exc: - raise SystemExit(f"line {lineno}: invalid JSON: {exc}") from exc - if "error" in row: - raise SystemExit(f"line {lineno}: stdin error: {row['error']}") - emb = row.get("embedding") - if not isinstance(emb, list) or not emb: - raise SystemExit(f"line {lineno}: missing embedding array") - rows.append(row) - return rows - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("--query-index", type=int, default=0, - help="embedding row to use as query, default: 0") - ap.add_argument("--include-query", action="store_true", - help="include the query row in the ranked output") - ap.add_argument("--labels", nargs="*", - help="optional labels for rows, in input order") - args = ap.parse_args() - - rows = read_rows() - if not rows: - raise SystemExit("no embeddings on stdin") - if args.query_index < 0 or args.query_index >= len(rows): - raise SystemExit(f"--query-index out of range for {len(rows)} rows") - if args.labels and len(args.labels) != len(rows): - raise SystemExit("--labels count must match embedding row count") - - query = rows[args.query_index]["embedding"] - ranked = [] - for i, row in enumerate(rows): - if i == args.query_index and not args.include_query: - continue - emb = row["embedding"] - if len(emb) != len(query): - raise SystemExit(f"row {i}: dimension mismatch") - ranked.append((cosine(query, emb), i, row)) - - ranked.sort(key=lambda item: item[0], reverse=True) - - print(f"query row: {args.query_index}") - print("rank row cosine tokens label") - print("---- --- ---------- ------ -----") - for rank, (score, i, row) in enumerate(ranked, 1): - label = args.labels[i] if args.labels else f"row {i}" - tokens = row.get("tokens", "-") - print(f"{rank:>4} {i:>3} {score:>10.6f} {tokens:>6} {label}") - - return 0 - - -if __name__ == "__main__": - raise SystemExit(main()) -- cgit v1.3.1