aboutsummaryrefslogtreecommitdiff
path: root/tools
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-06-20 20:31:06 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-06-21 06:00:26 +0200
commitdbdf714bb57578431924e5d069089bdfd9f4c604 (patch)
tree4f5b59ef132220310e1e4b1dde64f5e38782369a /tools
parente9ecaeaf21050f39888d33d2c8f4216d595d4975 (diff)
downloadffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.tar.gz
ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.zip
devtools: add scripts for unicode generation, model inspection, embedding ranking, and more
Diffstat (limited to '')
-rwxr-xr-xdevtools/convert_model_bf16.py (renamed from tools/convert_model_bf16.py)0
-rwxr-xr-x[-rw-r--r--]devtools/gen_wordpiece_unicode.py (renamed from tools/gen_wordpiece_unicode.py)6
-rw-r--r--devtools/openapi.yaml (renamed from tools/openapi.yaml)8
-rwxr-xr-xdevtools/rank_embeddings.py (renamed from tools/rank_embeddings.py)2
-rwxr-xr-xtools/inspect_model_artifacts.py88
5 files changed, 8 insertions, 96 deletions
diff --git a/tools/convert_model_bf16.py b/devtools/convert_model_bf16.py
index d59ccf2..d59ccf2 100755
--- a/tools/convert_model_bf16.py
+++ b/devtools/convert_model_bf16.py
diff --git a/tools/gen_wordpiece_unicode.py b/devtools/gen_wordpiece_unicode.py
index 24649f1..2f8642e 100644..100755
--- a/tools/gen_wordpiece_unicode.py
+++ b/devtools/gen_wordpiece_unicode.py
@@ -1,5 +1,5 @@
#!/usr/bin/env python3
-"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece
+"""Generate unicode.h file for libtok: Unicode range tables for the WordPiece
BasicTokenizer's punctuation and control classification, matching transformers'
BertTokenizer._is_punctuation and _is_control exactly.
@@ -10,7 +10,7 @@ as sorted, merged [lo, hi] ranges for binary search in the C tokenizer.
Run when bumping the target Unicode version (the C side is data-only otherwise):
- uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h
+ uv run python devtools/gen_wordpiece_unicode.py > unicode.h
"""
import unicodedata
@@ -94,7 +94,7 @@ def main():
for f in [fold(cp)]
if f != [cp]
]
- print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata
+ print(f"""/* Generated by devtools/gen_wordpiece_unicode.py from Python unicodedata
* (Unicode {unicodedata.unidata_version}). Do not edit by hand.
*
* Tables for the BasicTokenizer, matching transformers' BertTokenizer.
diff --git a/tools/openapi.yaml b/devtools/openapi.yaml
index 6dd3818..5140186 100644
--- a/tools/openapi.yaml
+++ b/devtools/openapi.yaml
@@ -1,9 +1,9 @@
openapi: 3.0.3
info:
- title: embed.c server API
+ title: ffwd server API
version: "1.0.0"
description: >
- HTTP API served by `embed-server`. Each loaded model has an operator-chosen
+ HTTP API served by `ffwd-server`. Each loaded model has an operator-chosen
label; request `model` fields match that label. Embedding encoding follows
the serving API selected at load time (`api=openai` or `api=perplexity`).
Always compare embeddings with cosine similarity.
@@ -12,7 +12,7 @@ info:
servers:
- url: http://127.0.0.1:8000
# Auth is optional: it applies only when the server is started with --api-key
-# (or EMBED_API_KEY). The empty requirement allows unauthenticated calls.
+# (or FFWD_API_KEY). The empty requirement allows unauthenticated calls.
security:
- {}
- bearerAuth: []
@@ -86,7 +86,7 @@ components:
properties:
model:
type: string
- description: Served label chosen when starting `embed-server`.
+ description: Served label chosen when starting `ffwd-server`.
example: qwen
input:
description: A single string or an array of up to 512 strings.
diff --git a/tools/rank_embeddings.py b/devtools/rank_embeddings.py
index ddc1e42..efc8aad 100755
--- a/tools/rank_embeddings.py
+++ b/devtools/rank_embeddings.py
@@ -2,7 +2,7 @@
"""Rank stdin JSON embeddings by cosine similarity to a query row.
Usage:
- ./embed ... --stream <<'EOF' | tools/rank_embeddings.py
+ ./tools/ffwd-cli ... --stream <<'EOF' | ./devtools/rank_embeddings.py
query: what is the capital of France?
document: Paris is the capital of France.
document: Berlin is the capital of Germany.
diff --git a/tools/inspect_model_artifacts.py b/tools/inspect_model_artifacts.py
deleted file mode 100755
index 2feabe2..0000000
--- a/tools/inspect_model_artifacts.py
+++ /dev/null
@@ -1,88 +0,0 @@
-#!/usr/bin/env python3
-"""Inspect local pplx-embed snapshots."""
-
-from __future__ import annotations
-
-import argparse
-import os
-from pathlib import Path
-
-
-PRIMARY_FILES = (
- "config.json",
- "tokenizer_config.json",
- "tokenizer.json",
- "vocab.json",
- "merges.txt",
-)
-
-
-def human_bytes(size: int) -> str:
- units = ("B", "KiB", "MiB", "GiB", "TiB")
- value = float(size)
- for unit in units:
- if value < 1024.0 or unit == units[-1]:
- return f"{value:.1f} {unit}"
- value /= 1024.0
- raise AssertionError("unreachable")
-
-
-def describe(path: Path) -> str:
- if not path.exists():
- return "MISSING"
- resolved = path.resolve()
- suffix = ""
- if path.is_symlink():
- suffix = f" -> {resolved}"
- return f"{human_bytes(resolved.stat().st_size):>10}{suffix}"
-
-
-def inspect(model_dir: Path) -> bool:
- model_dir = model_dir.expanduser().resolve()
- print(f"snapshot: {model_dir}")
- ok = True
- for name in PRIMARY_FILES:
- path = model_dir / name
- print(f" {name:<30} {describe(path)}")
- ok = path.exists() and ok
-
- model = model_dir / "model.safetensors"
- index = model_dir / "model.safetensors.index.json"
- shards = sorted(model_dir.glob("model-*-of-*.safetensors"))
- if model.exists():
- print(f" {'model.safetensors':<30} {describe(model)}")
- elif index.exists() and shards:
- print(f" {'model.safetensors.index.json':<30} {describe(index)}")
- for path in shards:
- print(f" {path.name:<28} {describe(path)}")
- else:
- print(f" {'model.safetensors':<30} MISSING")
- ok = False
-
- onnx_dir = model_dir / "onnx"
- onnx_files = sorted(onnx_dir.glob("*")) if onnx_dir.is_dir() else []
- print(" onnx/")
- if not onnx_files:
- print(" MISSING")
- return False
- for path in onnx_files:
- print(f" {path.name:<28} {describe(path)}")
- ok = path.exists() and ok
- return ok
-
-
-def main() -> int:
- ap = argparse.ArgumentParser(description=__doc__)
- ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR")
- args = ap.parse_args()
-
- ok = True
- for i, name in enumerate(args.model_dirs):
- if i:
- print()
- ok = inspect(Path(os.path.expandvars(name))) and ok
- return 0 if ok else 1
-
-
-if __name__ == "__main__":
- raise SystemExit(main())