aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:31:06 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commitdbdf714bb57578431924e5d069089bdfd9f4c604 (patch)
tree4f5b59ef132220310e1e4b1dde64f5e38782369a /tools
parente9ecaeaf21050f39888d33d2c8f4216d595d4975 (diff)
downloadffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.tar.gz
ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.zip
devtools: add scripts for unicode generation, model inspection, embedding ranking, and more
Diffstat (limited to 'tools')
-rwxr-xr-xtools/convert_model_bf16.py195
-rw-r--r--tools/gen_wordpiece_unicode.py139
-rwxr-xr-xtools/inspect_model_artifacts.py88
-rw-r--r--tools/openapi.yaml249
-rwxr-xr-xtools/rank_embeddings.py94
5 files changed, 0 insertions, 765 deletions
diff --git a/tools/convert_model_bf16.py b/tools/convert_model_bf16.py
deleted file mode 100755
index d59ccf2..0000000
--- a/tools/convert_model_bf16.py
+++ /dev/null
@@ -1,195 +0,0 @@
-#!/usr/bin/env python3
-"""Convert a local safetensors model directory to BF16.
-
-The output directory keeps non-safetensors files as symlinks by default and
-writes converted safetensors files. F32 tensors are converted to BF16 with
-round-to-nearest-even. Existing BF16 tensors are copied unchanged.
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import os
-import shutil
-import struct
-from pathlib import Path
-
-import numpy as np
-
-
-COPY_CHUNK = 64 * 1024 * 1024
-F32_CHUNK_ELEMS = 16 * 1024 * 1024
-
-
-def read_safetensors_header(path: Path) -> tuple[dict, int]:
- with path.open("rb") as f:
- raw = f.read(8)
- if len(raw) != 8:
- raise ValueError(f"{path}: too small for safetensors header")
- header_len = struct.unpack("<Q", raw)[0]
- header_raw = f.read(header_len)
- if len(header_raw) != header_len:
- raise ValueError(f"{path}: truncated safetensors header")
- return json.loads(header_raw), 8 + header_len
-
-
-def tensor_nbytes(spec: dict) -> int:
- start, end = spec["data_offsets"]
- return int(end) - int(start)
-
-
-def converted_tensor_nbytes(spec: dict) -> int:
- if spec["dtype"] == "F32":
- return tensor_nbytes(spec) // 2
- return tensor_nbytes(spec)
-
-
-def build_output_header(header: dict) -> tuple[dict, dict[str, tuple[int, int]]]:
- out_header = {}
- offsets = {}
- cursor = 0
-
- for name, spec in header.items():
- if name == "__metadata__":
- out_header[name] = spec
- continue
-
- out_spec = dict(spec)
- nbytes = converted_tensor_nbytes(spec)
- offsets[name] = (cursor, cursor + nbytes)
- out_spec["data_offsets"] = [cursor, cursor + nbytes]
- if spec["dtype"] == "F32":
- out_spec["dtype"] = "BF16"
- out_header[name] = out_spec
- cursor += nbytes
-
- return out_header, offsets
-
-
-def copy_range(src, dst, nbytes: int) -> None:
- left = nbytes
- while left:
- chunk = src.read(min(left, COPY_CHUNK))
- if not chunk:
- raise IOError("unexpected EOF while copying tensor data")
- dst.write(chunk)
- left -= len(chunk)
-
-
-def write_f32_as_bf16(src, dst, nbytes: int) -> None:
- if nbytes % 4 != 0:
- raise ValueError("F32 tensor byte size is not divisible by 4")
-
- elems_left = nbytes // 4
- while elems_left:
- count = min(elems_left, F32_CHUNK_ELEMS)
- raw = src.read(count * 4)
- if len(raw) != count * 4:
- raise IOError("unexpected EOF while converting F32 tensor data")
-
- bits = np.frombuffer(raw, dtype="<u4").astype(np.uint32, copy=True)
- lsb = (bits >> np.uint32(16)) & np.uint32(1)
- bf16 = ((bits + np.uint32(0x7FFF) + lsb) >> np.uint32(16)).astype("<u2")
- dst.write(bf16.tobytes())
- elems_left -= count
-
-
-def convert_safetensors(src_path: Path, dst_path: Path,
- verbose: bool = False) -> tuple[int, int]:
- header, data_start = read_safetensors_header(src_path)
- out_header, _offsets = build_output_header(header)
- out_header_raw = json.dumps(out_header, separators=(",", ":")).encode("utf-8")
-
- tensors = [(name, spec) for name, spec in header.items() if name != "__metadata__"]
- input_bytes = sum(tensor_nbytes(spec) for _, spec in tensors)
- output_bytes = sum(converted_tensor_nbytes(spec) for _, spec in tensors)
-
- dst_path.parent.mkdir(parents=True, exist_ok=True)
- tmp_path = dst_path.with_suffix(dst_path.suffix + ".tmp")
- with src_path.open("rb") as src, tmp_path.open("wb") as dst:
- dst.write(struct.pack("<Q", len(out_header_raw)))
- dst.write(out_header_raw)
-
- for name, spec in tensors:
- start, _end = spec["data_offsets"]
- src.seek(data_start + int(start))
- nbytes = tensor_nbytes(spec)
- dtype = spec["dtype"]
- if dtype == "F32":
- write_f32_as_bf16(src, dst, nbytes)
- elif dtype == "BF16":
- copy_range(src, dst, nbytes)
- else:
- copy_range(src, dst, nbytes)
- if verbose:
- print(f" {name}: {dtype} {nbytes / (1024 ** 2):.1f} MB", flush=True)
-
- tmp_path.replace(dst_path)
- return input_bytes, output_bytes
-
-
-def link_or_copy(src: Path, dst: Path, copy_files: bool) -> None:
- dst.parent.mkdir(parents=True, exist_ok=True)
- if copy_files:
- if src.is_dir():
- shutil.copytree(src, dst, symlinks=True)
- else:
- shutil.copy2(src, dst)
- else:
- target = src.resolve()
- os.symlink(target, dst, target_is_directory=src.is_dir())
-
-
-def convert_model(src_dir: Path, dst_dir: Path, copy_files: bool,
- verbose: bool = False) -> None:
- if not src_dir.is_dir():
- raise SystemExit(f"source model dir does not exist: {src_dir}")
- if dst_dir.exists() and any(dst_dir.iterdir()):
- raise SystemExit(f"output dir already exists and is not empty: {dst_dir}")
- dst_dir.mkdir(parents=True, exist_ok=True)
-
- safetensors = sorted(src_dir.glob("*.safetensors"))
- if not safetensors:
- raise SystemExit(f"no safetensors files found in {src_dir}")
-
- total_in = 0
- total_out = 0
- for src_path in safetensors:
- dst_path = dst_dir / src_path.name
- print(f"converting {src_path.name}", flush=True)
- n_in, n_out = convert_safetensors(src_path.resolve(), dst_path, verbose)
- total_in += n_in
- total_out += n_out
-
- for child in src_dir.iterdir():
- if child.name.endswith(".safetensors"):
- continue
- dst_child = dst_dir / child.name
- if dst_child.exists() or dst_child.is_symlink():
- continue
- link_or_copy(child, dst_child, copy_files)
-
- print(
- f"done: tensor data {total_in / (1024 ** 3):.2f} GiB -> "
- f"{total_out / (1024 ** 3):.2f} GiB",
- flush=True,
- )
-
-
-def main() -> int:
- ap = argparse.ArgumentParser(description=__doc__)
- ap.add_argument("--src", required=True, help="source model directory")
- ap.add_argument("--dst", required=True, help="output model directory")
- ap.add_argument("--copy-files", action="store_true",
- help="copy non-safetensors files instead of symlinking")
- ap.add_argument("-v", "--verbose", action="store_true",
- help="print every tensor as it is converted")
- args = ap.parse_args()
-
- convert_model(Path(args.src), Path(args.dst), args.copy_files, args.verbose)
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/tools/gen_wordpiece_unicode.py b/tools/gen_wordpiece_unicode.py
deleted file mode 100644
index 24649f1..0000000
--- a/tools/gen_wordpiece_unicode.py
+++ /dev/null
@@ -1,139 +0,0 @@
-#!/usr/bin/env python3
-"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece
-BasicTokenizer's punctuation and control classification, matching transformers'
-BertTokenizer._is_punctuation and _is_control exactly.
-
-Punctuation is the ASCII set (33-47, 58-64, 91-96, 123-126) plus every codepoint
-whose Unicode category starts with "P". Control is categories Cc, Cf, Co minus
-tab/newline/carriage-return (which route to whitespace). The tables are emitted
-as sorted, merged [lo, hi] ranges for binary search in the C tokenizer.
-
-Run when bumping the target Unicode version (the C side is data-only otherwise):
-
- uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h
-"""
-
-import unicodedata
-
-MAX_CP = 0x110000
-
-
-def merge(cps):
- """Collapse a sorted codepoint list into contiguous [lo, hi] ranges."""
- ranges = []
- for cp in cps:
- if ranges and cp == ranges[-1][1] + 1:
- ranges[-1][1] = cp
- else:
- ranges.append([cp, cp])
- return ranges
-
-
-def is_punct(cp):
- if (33 <= cp <= 47) or (58 <= cp <= 64) or (91 <= cp <= 96) or (123 <= cp <= 126):
- return True
- return unicodedata.category(chr(cp)).startswith("P")
-
-
-def is_control(cp):
- if cp in (0x09, 0x0A, 0x0D):
- return False
- # Cc/Cf/Co are the assigned control, format and private-use chars. Cn
- # (unassigned) is intentionally excluded: it never occurs in real text and
- # would bind the table to a single Unicode version.
- return unicodedata.category(chr(cp)) in ("Cc", "Cf", "Co")
-
-
-# Hangul syllables decompose by formula (the standard algorithm), so they are
-# excluded from the fold table to keep it small; the C side handles them.
-HANGUL_BASE, HANGUL_COUNT = 0xAC00, 11172
-
-
-def fold(cp):
- """do_lower_case fold: Python str.lower then NFD then drop combining marks
- (Mn), matching transformers' lowercase + _run_strip_accents. Returns the
- output codepoint list (empty when the codepoint is a dropped mark)."""
- s = chr(cp).lower()
- return [ord(c) for c in unicodedata.normalize("NFD", s) if unicodedata.category(c) != "Mn"]
-
-
-def emit_fold(entries):
- out = ["static const wp_fold_t WP_FOLD[] = {"]
- for cp, f in entries:
- outs = ", ".join("0x%X" % c for c in (f + [0, 0, 0])[:3])
- out.append(" {0x%X, {%s}, %d}," % (cp, outs, len(f)))
- out.append("};")
- out.append("enum { WP_FOLD_N = (int)(sizeof(WP_FOLD) / sizeof(WP_FOLD[0])) };")
- return "\n".join(out)
-
-
-def emit(name, ranges):
- out = [f"static const wp_range_t {name}[] = {{"]
- line = " "
- for lo, hi in ranges:
- s = "{0x%X, 0x%X}, " % (lo, hi)
- if len(line) + len(s) > 96:
- out.append(line.rstrip())
- line = " "
- line += s
- if line.strip():
- out.append(line.rstrip())
- out.append("};")
- out.append(f"enum {{ {name}_N = (int)(sizeof({name}) / sizeof({name}[0])) }};")
- return "\n".join(out)
-
-
-def main():
- punct = merge([cp for cp in range(MAX_CP) if is_punct(cp)])
- control = merge([cp for cp in range(MAX_CP) if is_control(cp)])
- hangul = range(HANGUL_BASE, HANGUL_BASE + HANGUL_COUNT)
- fold_entries = [
- (cp, f)
- for cp in range(MAX_CP)
- if cp not in hangul
- for f in [fold(cp)]
- if f != [cp]
- ]
- print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata
- * (Unicode {unicodedata.unidata_version}). Do not edit by hand.
- *
- * Tables for the BasicTokenizer, matching transformers' BertTokenizer.
- *
- * WP_PUNCT_RANGES / WP_CONTROL_RANGES: _is_punctuation and _is_control as sorted
- * [lo, hi] ranges. Punctuation is the ASCII set (33-47/58-64/91-96/123-126) plus
- * every category P* codepoint. Control is categories Cc, Cf, Co minus
- * tab/newline/carriage-return (those route to whitespace). Unassigned (Cn) is
- * intentionally not control: it never occurs in real text and would bind the
- * table to one Unicode version.
- *
- * WP_FOLD: the do_lower_case fold (Python str.lower then NFD then drop Mn marks),
- * sorted by codepoint. n is the output length (0 drops a combining mark; up to
- * 3). Hangul syllables (U+AC00..U+D7A3) decompose by formula on the C side and
- * are excluded here. */
-#ifndef WORDPIECE_UNICODE_H
-#define WORDPIECE_UNICODE_H
-
-#include <stdint.h>
-
-typedef struct {{
- uint32_t lo;
- uint32_t hi;
-}} wp_range_t;
-
-typedef struct {{
- uint32_t cp;
- uint32_t out[3];
- uint8_t n;
-}} wp_fold_t;
-
-{emit("WP_PUNCT_RANGES", punct)}
-
-{emit("WP_CONTROL_RANGES", control)}
-
-{emit_fold(fold_entries)}
-
-#endif /* WORDPIECE_UNICODE_H */""")
-
-
-if __name__ == "__main__":
- main()
diff --git a/tools/inspect_model_artifacts.py b/tools/inspect_model_artifacts.py
deleted file mode 100755
index 2feabe2..0000000
--- a/tools/inspect_model_artifacts.py
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/usr/bin/env python3
-"""Inspect local pplx-embed snapshots."""
-
-from __future__ import annotations
-
-import argparse
-import os
-from pathlib import Path
-
-
-PRIMARY_FILES = (
- "config.json",
- "tokenizer_config.json",
- "tokenizer.json",
- "vocab.json",
- "merges.txt",
-)
-
-
-def human_bytes(size: int) -> str:
- units = ("B", "KiB", "MiB", "GiB", "TiB")
- value = float(size)
- for unit in units:
- if value < 1024.0 or unit == units[-1]:
- return f"{value:.1f} {unit}"
- value /= 1024.0
- raise AssertionError("unreachable")
-
-
-def describe(path: Path) -> str:
- if not path.exists():
- return "MISSING"
- resolved = path.resolve()
- suffix = ""
- if path.is_symlink():
- suffix = f" -> {resolved}"
- return f"{human_bytes(resolved.stat().st_size):>10}{suffix}"
-
-
-def inspect(model_dir: Path) -> bool:
- model_dir = model_dir.expanduser().resolve()
- print(f"snapshot: {model_dir}")
- ok = True
- for name in PRIMARY_FILES:
- path = model_dir / name
- print(f" {name:<30} {describe(path)}")
- ok = path.exists() and ok
-
- model = model_dir / "model.safetensors"
- index = model_dir / "model.safetensors.index.json"
- shards = sorted(model_dir.glob("model-*-of-*.safetensors"))
- if model.exists():
- print(f" {'model.safetensors':<30} {describe(model)}")
- elif index.exists() and shards:
- print(f" {'model.safetensors.index.json':<30} {describe(index)}")
- for path in shards:
- print(f" {path.name:<28} {describe(path)}")
- else:
- print(f" {'model.safetensors':<30} MISSING")
- ok = False
-
- onnx_dir = model_dir / "onnx"
- onnx_files = sorted(onnx_dir.glob("*")) if onnx_dir.is_dir() else []
- print(" onnx/")
- if not onnx_files:
- print(" MISSING")
- return False
- for path in onnx_files:
- print(f" {path.name:<28} {describe(path)}")
- ok = path.exists() and ok
- return ok
-
-
-def main() -> int:
- ap = argparse.ArgumentParser(description=__doc__)
- ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR")
- args = ap.parse_args()
-
- ok = True
- for i, name in enumerate(args.model_dirs):
- if i:
- print()
- ok = inspect(Path(os.path.expandvars(name))) and ok
- return 0 if ok else 1
-
-
-if __name__ == "__main__":
- raise SystemExit(main())
diff --git a/tools/openapi.yaml b/tools/openapi.yaml
deleted file mode 100644
index 6dd3818..0000000
--- a/tools/openapi.yaml
+++ /dev/null
@@ -1,249 +0,0 @@
-openapi: 3.0.3
-info:
- title: embed.c server API
- version: "1.0.0"
- description: >
- HTTP API served by `embed-server`. Each loaded model has an operator-chosen
- label; request `model` fields match that label. Embedding encoding follows
- the serving API selected at load time (`api=openai` or `api=perplexity`).
- Always compare embeddings with cosine similarity.
- license:
- name: MIT
-servers:
- - url: http://127.0.0.1:8000
-# Auth is optional: it applies only when the server is started with --api-key
-# (or EMBED_API_KEY). The empty requirement allows unauthenticated calls.
-security:
- - {}
- - bearerAuth: []
-paths:
- /v1/embeddings:
- post:
- summary: Create pooled embeddings
- operationId: createEmbeddings
- requestBody:
- required: true
- content:
- application/json:
- schema: { $ref: "#/components/schemas/EmbeddingsRequest" }
- responses:
- "200":
- description: One embedding per input, in request order.
- content:
- application/json:
- schema: { $ref: "#/components/schemas/EmbeddingsResponse" }
- "422": { $ref: "#/components/responses/ValidationError" }
- /v1/contextualizedembeddings:
- post:
- summary: Create contextual (late-chunking) document embeddings
- operationId: createContextualizedEmbeddings
- description: >
- Each document is a list of chunk strings. Chunks are embedded with
- document-level context and returned per document, per chunk, in order.
- requestBody:
- required: true
- content:
- application/json:
- schema: { $ref: "#/components/schemas/ContextualRequest" }
- responses:
- "200":
- description: One list of chunk embeddings per document, in order.
- content:
- application/json:
- schema: { $ref: "#/components/schemas/ContextualResponse" }
- "422": { $ref: "#/components/responses/ValidationError" }
- /v1/rerank:
- post:
- summary: Late-interaction (MaxSim) reranking
- operationId: rerank
- description: >
- Scores each document against the query with token-level MaxSim and
- returns documents ranked by raw score (not a probability).
- requestBody:
- required: true
- content:
- application/json:
- schema: { $ref: "#/components/schemas/RerankRequest" }
- responses:
- "200":
- description: Documents ranked by descending relevance score.
- content:
- application/json:
- schema: { $ref: "#/components/schemas/RerankResponse" }
- "422": { $ref: "#/components/responses/ValidationError" }
-components:
- securitySchemes:
- bearerAuth:
- type: http
- scheme: bearer
- description: >
- Single shared token set with --api-key. Not a real auth system;
- terminate TLS and authenticate at a reverse proxy in production.
- schemas:
- EmbeddingsRequest:
- type: object
- required: [model, input]
- properties:
- model:
- type: string
- description: Served label chosen when starting `embed-server`.
- example: qwen
- input:
- description: A single string or an array of up to 512 strings.
- oneOf:
- - type: string
- - type: array
- items: { type: string }
- minItems: 1
- maxItems: 512
- encoding_format:
- type: string
- description: >
- Output encoding, validated against the serving API selected for the
- loaded label. `api=openai` accepts `float` (default; true float32)
- and `base64` (base64 of little-endian float32). `api=perplexity`
- accepts `base64_int8` (default), `base64_binary` (sign bits, needs
- `dimensions` divisible by 8), and `float` (the int8-decoded view).
- A format outside the label's serving API returns 422.
- enum: [float, base64, base64_int8, base64_binary]
- dimensions:
- type: integer
- description: >
- Matryoshka truncation, from the configured load-time `min_dim` up to
- the model's native dimension. Truncated normalized vectors are
- re-normalized.
- EmbeddingsResponse:
- type: object
- properties:
- object: { type: string, example: list }
- data:
- type: array
- items: { $ref: "#/components/schemas/Embedding" }
- model: { type: string }
- usage: { $ref: "#/components/schemas/Usage" }
- Embedding:
- type: object
- properties:
- object: { type: string, example: embedding }
- index: { type: integer }
- embedding:
- description: >
- A float array when `encoding_format` is `float`, otherwise a base64
- string (`base64`, `base64_int8`, or `base64_binary`).
- oneOf:
- - type: array
- items: { type: number, format: float }
- - type: string
- ContextualRequest:
- type: object
- required: [model, input]
- properties:
- model:
- type: string
- description: Served contextual-model label.
- example: pplx-context
- input:
- description: Array of documents; each document is an array of chunk strings.
- type: array
- items:
- type: array
- items: { type: string }
- minItems: 1
- encoding_format:
- type: string
- enum: [float, base64, base64_int8, base64_binary]
- dimensions: { type: integer }
- ContextualResponse:
- type: object
- properties:
- object: { type: string, example: list }
- data:
- type: array
- description: One entry per document, in request order.
- items:
- type: object
- properties:
- object: { type: string, example: list }
- index: { type: integer }
- data:
- type: array
- description: One embedding per chunk, in document order.
- items: { $ref: "#/components/schemas/Embedding" }
- model: { type: string }
- usage: { $ref: "#/components/schemas/Usage" }
- RerankRequest:
- type: object
- required: [model, query, documents]
- properties:
- model:
- type: string
- description: Served late-model label.
- example: pplx-late
- query: { type: string }
- documents:
- type: array
- items: { type: string }
- minItems: 1
- maxItems: 1000
- top_n:
- type: integer
- description: Number of results to return. Defaults to all. `top_k` is an alias.
- top_k:
- type: integer
- return_documents:
- type: boolean
- default: false
- description: Include the source document text in each result.
- additionalProperties: false
- RerankResponse:
- type: object
- properties:
- object: { type: string, example: list }
- model: { type: string }
- results:
- type: array
- items:
- type: object
- properties:
- index:
- type: integer
- description: Index of the document in the request array.
- relevance_score:
- type: number
- format: float
- description: Raw MaxSim score (not a probability).
- document:
- type: string
- description: Present only when `return_documents` is true.
- usage:
- type: object
- properties:
- query_tokens: { type: integer }
- document_tokens: { type: integer }
- total_tokens: { type: integer }
- Usage:
- type: object
- properties:
- prompt_tokens: { type: integer }
- total_tokens: { type: integer }
- ValidationDetail:
- type: object
- properties:
- loc:
- type: array
- description: Path to the offending field, e.g. ["body","input",0].
- items: {}
- msg: { type: string }
- type: { type: string }
- ValidationError:
- type: object
- properties:
- detail:
- type: array
- items: { $ref: "#/components/schemas/ValidationDetail" }
- responses:
- ValidationError:
- description: Request body failed validation.
- content:
- application/json:
- schema: { $ref: "#/components/schemas/ValidationError" }
diff --git a/tools/rank_embeddings.py b/tools/rank_embeddings.py
deleted file mode 100755
index ddc1e42..0000000
--- a/tools/rank_embeddings.py
+++ /dev/null
@@ -1,94 +0,0 @@
-#!/usr/bin/env python3
-"""Rank stdin JSON embeddings by cosine similarity to a query row.
-
-Usage:
- ./embed ... --stream <<'EOF' | tools/rank_embeddings.py
- query: what is the capital of France?
- document: Paris is the capital of France.
- document: Berlin is the capital of Germany.
- EOF
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import math
-import sys
-
-
-def cosine(a: list[float], b: list[float]) -> float:
- dot = 0.0
- aa = 0.0
- bb = 0.0
- for x, y in zip(a, b):
- dot += x * y
- aa += x * x
- bb += y * y
- if aa == 0.0 or bb == 0.0:
- return float("nan")
- return dot / math.sqrt(aa * bb)
-
-
-def read_rows() -> list[dict]:
- rows = []
- for lineno, line in enumerate(sys.stdin, 1):
- line = line.strip()
- if not line:
- continue
- try:
- row = json.loads(line)
- except json.JSONDecodeError as exc:
- raise SystemExit(f"line {lineno}: invalid JSON: {exc}") from exc
- if "error" in row:
- raise SystemExit(f"line {lineno}: stdin error: {row['error']}")
- emb = row.get("embedding")
- if not isinstance(emb, list) or not emb:
- raise SystemExit(f"line {lineno}: missing embedding array")
- rows.append(row)
- return rows
-
-
-def main() -> int:
- ap = argparse.ArgumentParser(description=__doc__)
- ap.add_argument("--query-index", type=int, default=0,
- help="embedding row to use as query, default: 0")
- ap.add_argument("--include-query", action="store_true",
- help="include the query row in the ranked output")
- ap.add_argument("--labels", nargs="*",
- help="optional labels for rows, in input order")
- args = ap.parse_args()
-
- rows = read_rows()
- if not rows:
- raise SystemExit("no embeddings on stdin")
- if args.query_index < 0 or args.query_index >= len(rows):
- raise SystemExit(f"--query-index out of range for {len(rows)} rows")
- if args.labels and len(args.labels) != len(rows):
- raise SystemExit("--labels count must match embedding row count")
-
- query = rows[args.query_index]["embedding"]
- ranked = []
- for i, row in enumerate(rows):
- if i == args.query_index and not args.include_query:
- continue
- emb = row["embedding"]
- if len(emb) != len(query):
- raise SystemExit(f"row {i}: dimension mismatch")
- ranked.append((cosine(query, emb), i, row))
-
- ranked.sort(key=lambda item: item[0], reverse=True)
-
- print(f"query row: {args.query_index}")
- print("rank row cosine tokens label")
- print("---- --- ---------- ------ -----")
- for rank, (score, i, row) in enumerate(ranked, 1):
- label = args.labels[i] if args.labels else f"row {i}"
- tokens = row.get("tokens", "-")
- print(f"{rank:>4} {i:>3} {score:>10.6f} {tokens:>6} {label}")
-
- return 0
-
-
-if __name__ == "__main__":
- raise SystemExit(main())