diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-06-20 20:31:06 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-06-21 06:00:26 +0200 |
| commit | dbdf714bb57578431924e5d069089bdfd9f4c604 (patch) | |
| tree | 4f5b59ef132220310e1e4b1dde64f5e38782369a /tools | |
| parent | e9ecaeaf21050f39888d33d2c8f4216d595d4975 (diff) | |
| download | ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.tar.gz ffwd-dbdf714bb57578431924e5d069089bdfd9f4c604.zip | |
devtools: add scripts for unicode generation, model inspection, embedding ranking, and more
Diffstat (limited to '')
| -rwxr-xr-x | devtools/convert_model_bf16.py (renamed from tools/convert_model_bf16.py) | 0 | ||||
| -rwxr-xr-x[-rw-r--r--] | devtools/gen_wordpiece_unicode.py (renamed from tools/gen_wordpiece_unicode.py) | 6 | ||||
| -rw-r--r-- | devtools/openapi.yaml (renamed from tools/openapi.yaml) | 8 | ||||
| -rwxr-xr-x | devtools/rank_embeddings.py (renamed from tools/rank_embeddings.py) | 2 | ||||
| -rwxr-xr-x | tools/inspect_model_artifacts.py | 88 |
5 files changed, 8 insertions, 96 deletions
diff --git a/tools/convert_model_bf16.py b/devtools/convert_model_bf16.py index d59ccf2..d59ccf2 100755 --- a/tools/convert_model_bf16.py +++ b/devtools/convert_model_bf16.py diff --git a/tools/gen_wordpiece_unicode.py b/devtools/gen_wordpiece_unicode.py index 24649f1..2f8642e 100644..100755 --- a/tools/gen_wordpiece_unicode.py +++ b/devtools/gen_wordpiece_unicode.py @@ -1,5 +1,5 @@ #!/usr/bin/env python3 -"""Generate src/tokenizer_unicode.h: Unicode range tables for the WordPiece +"""Generate unicode.h file for libtok: Unicode range tables for the WordPiece BasicTokenizer's punctuation and control classification, matching transformers' BertTokenizer._is_punctuation and _is_control exactly. @@ -10,7 +10,7 @@ as sorted, merged [lo, hi] ranges for binary search in the C tokenizer. Run when bumping the target Unicode version (the C side is data-only otherwise): - uv run python tools/gen_wordpiece_unicode.py > src/tokenizer_unicode.h + uv run python devtools/gen_wordpiece_unicode.py > unicode.h """ import unicodedata @@ -94,7 +94,7 @@ def main(): for f in [fold(cp)] if f != [cp] ] - print(f"""/* Generated by tools/gen_wordpiece_unicode.py from Python unicodedata + print(f"""/* Generated by devtools/gen_wordpiece_unicode.py from Python unicodedata * (Unicode {unicodedata.unidata_version}). Do not edit by hand. * * Tables for the BasicTokenizer, matching transformers' BertTokenizer. diff --git a/tools/openapi.yaml b/devtools/openapi.yaml index 6dd3818..5140186 100644 --- a/tools/openapi.yaml +++ b/devtools/openapi.yaml @@ -1,9 +1,9 @@ openapi: 3.0.3 info: - title: embed.c server API + title: ffwd server API version: "1.0.0" description: > - HTTP API served by `embed-server`. Each loaded model has an operator-chosen + HTTP API served by `ffwd-server`. Each loaded model has an operator-chosen label; request `model` fields match that label. Embedding encoding follows the serving API selected at load time (`api=openai` or `api=perplexity`). Always compare embeddings with cosine similarity. @@ -12,7 +12,7 @@ info: servers: - url: http://127.0.0.1:8000 # Auth is optional: it applies only when the server is started with --api-key -# (or EMBED_API_KEY). The empty requirement allows unauthenticated calls. +# (or FFWD_API_KEY). The empty requirement allows unauthenticated calls. security: - {} - bearerAuth: [] @@ -86,7 +86,7 @@ components: properties: model: type: string - description: Served label chosen when starting `embed-server`. + description: Served label chosen when starting `ffwd-server`. example: qwen input: description: A single string or an array of up to 512 strings. diff --git a/tools/rank_embeddings.py b/devtools/rank_embeddings.py index ddc1e42..efc8aad 100755 --- a/tools/rank_embeddings.py +++ b/devtools/rank_embeddings.py @@ -2,7 +2,7 @@ """Rank stdin JSON embeddings by cosine similarity to a query row. Usage: - ./embed ... --stream <<'EOF' | tools/rank_embeddings.py + ./tools/ffwd-cli ... --stream <<'EOF' | ./devtools/rank_embeddings.py query: what is the capital of France? document: Paris is the capital of France. document: Berlin is the capital of Germany. diff --git a/tools/inspect_model_artifacts.py b/tools/inspect_model_artifacts.py deleted file mode 100755 index 2feabe2..0000000 --- a/tools/inspect_model_artifacts.py +++ /dev/null @@ -1,88 +0,0 @@ -#!/usr/bin/env python3 -"""Inspect local pplx-embed snapshots.""" - -from __future__ import annotations - -import argparse -import os -from pathlib import Path - - -PRIMARY_FILES = ( - "config.json", - "tokenizer_config.json", - "tokenizer.json", - "vocab.json", - "merges.txt", -) - - -def human_bytes(size: int) -> str: - units = ("B", "KiB", "MiB", "GiB", "TiB") - value = float(size) - for unit in units: - if value < 1024.0 or unit == units[-1]: - return f"{value:.1f} {unit}" - value /= 1024.0 - raise AssertionError("unreachable") - - -def describe(path: Path) -> str: - if not path.exists(): - return "MISSING" - resolved = path.resolve() - suffix = "" - if path.is_symlink(): - suffix = f" -> {resolved}" - return f"{human_bytes(resolved.stat().st_size):>10}{suffix}" - - -def inspect(model_dir: Path) -> bool: - model_dir = model_dir.expanduser().resolve() - print(f"snapshot: {model_dir}") - ok = True - for name in PRIMARY_FILES: - path = model_dir / name - print(f" {name:<30} {describe(path)}") - ok = path.exists() and ok - - model = model_dir / "model.safetensors" - index = model_dir / "model.safetensors.index.json" - shards = sorted(model_dir.glob("model-*-of-*.safetensors")) - if model.exists(): - print(f" {'model.safetensors':<30} {describe(model)}") - elif index.exists() and shards: - print(f" {'model.safetensors.index.json':<30} {describe(index)}") - for path in shards: - print(f" {path.name:<28} {describe(path)}") - else: - print(f" {'model.safetensors':<30} MISSING") - ok = False - - onnx_dir = model_dir / "onnx" - onnx_files = sorted(onnx_dir.glob("*")) if onnx_dir.is_dir() else [] - print(" onnx/") - if not onnx_files: - print(" MISSING") - return False - for path in onnx_files: - print(f" {path.name:<28} {describe(path)}") - ok = path.exists() and ok - return ok - - -def main() -> int: - ap = argparse.ArgumentParser(description=__doc__) - ap.add_argument("model_dirs", nargs="+", metavar="MODEL_DIR") - args = ap.parse_args() - - ok = True - for i, name in enumerate(args.model_dirs): - if i: - print() - ok = inspect(Path(os.path.expandvars(name))) and ok - return 0 if ok else 1 - - -if __name__ == "__main__": - raise SystemExit(main()) |
