aboutsummaryrefslogtreecommitdiff
path: root/tests/check_e5_parity.py
blob: 37b63aece94b9aebe7faaed455691555bd81f1dd (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
#!/usr/bin/env python3
"""Compare the ffwd CLI against SentenceTransformers for an XLM-R /
SentencePiece model (default: intfloat/multilingual-e5-large-instruct).

The CLI runs the real product path: SentencePiece tokenize -> <s> ... </s> ->
xlm-roberta forward (position offset pad_id+1) -> config-driven mean pool -> L2
normalize. The reference is SentenceTransformer(dir).encode(normalize=True),
which runs the same pipeline. Cosine is scale-invariant; max_abs also checks
magnitude, so it only matches if both sides normalize.

Needs the model weights to run the forward, so it stays a manual --model-dir
check. The reference side needs torch + sentence-transformers, so run it with uv:

  uv run --python 3.12 --with-requirements requirements-reference.txt \
      tests/check_e5_parity.py --model-dir DIR
"""

import argparse
import json
import math
import subprocess
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
DEFAULT_BINARY = ROOT / "ffwd-cli"

DEFAULT_TEXTS = [
    "The capital of France is Paris.",
    "Dense retrieval maps text to a vector space for semantic search.",
    "Numbers like 12345 and dates 2020-2021 appear here.",
    "café résumé naïve coordinate punctuation, test!",
    "南瓜是一种蔬菜。",
    "Привет мир, как дела сегодня?",
    "مرحبا بالعالم هذا اختبار.",
    "a short query",
]


def ensure_cli(binary: Path) -> None:
    """Build the CLI if the default binary is missing."""
    if binary == DEFAULT_BINARY and not binary.exists():
        subprocess.run(["make", "cpu"], cwd=ROOT, check=True,
                       stdout=subprocess.DEVNULL)


def cli_encode(binary, model_dir, texts):
    cmd = [str(binary), "-d", str(model_dir), "--stream"]
    proc = subprocess.run(cmd, input="\n".join(texts) + "\n", text=True,
                          stdout=subprocess.PIPE, stderr=subprocess.PIPE)
    if proc.returncode != 0:
        sys.stderr.write(proc.stdout + proc.stderr)
        raise SystemExit(proc.returncode)
    rows = []
    for line in proc.stdout.splitlines():
        if not line.strip():
            continue
        obj = json.loads(line)
        if "error" in obj:
            raise SystemExit(f"CLI error: {obj}")
        rows.append([float(x) for x in obj["embedding"]])
    return rows


def cosine(a, b):
    dot = sum(x * y for x, y in zip(a, b))
    na = math.sqrt(sum(x * x for x in a))
    nb = math.sqrt(sum(y * y for y in b))
    return dot / (na * nb) if na and nb else 0.0


def main() -> int:
    ap = argparse.ArgumentParser()
    ap.add_argument("--model", default="intfloat/multilingual-e5-large-instruct")
    ap.add_argument("--model-dir", default=None, help="local dir; default downloads --model")
    ap.add_argument("--binary", default=str(DEFAULT_BINARY))
    ap.add_argument("--min-cos", type=float, default=0.999)
    ap.add_argument("--max-diff", type=float, default=2e-3)
    ap.add_argument("texts", nargs="*")
    args = ap.parse_args()

    from sentence_transformers import SentenceTransformer
    if args.model_dir:
        model_dir = args.model_dir
    else:
        from huggingface_hub import snapshot_download
        model_dir = snapshot_download(args.model)

    texts = list(args.texts if args.texts else DEFAULT_TEXTS)
    ref = SentenceTransformer(model_dir)
    ref_vecs = ref.encode(texts, normalize_embeddings=True).tolist()

    ensure_cli(Path(args.binary))
    c_vecs = cli_encode(args.binary, model_dir, texts)

    if len(c_vecs) != len(ref_vecs):
        print(f"row count mismatch: C={len(c_vecs)} ref={len(ref_vecs)}", file=sys.stderr)
        return 1

    worst_cos, worst_diff, fails = 1.0, 0.0, 0
    for i, (t, c, r) in enumerate(zip(texts, c_vecs, ref_vecs)):
        if len(c) != len(r):
            print(f"dim mismatch {i}: {len(c)} vs {len(r)}", file=sys.stderr)
            fails += 1
            continue
        cos = cosine(c, r)
        diff = max(abs(x - y) for x, y in zip(c, r))
        worst_cos = min(worst_cos, cos)
        worst_diff = max(worst_diff, diff)
        bad = cos < args.min_cos or diff > args.max_diff
        fails += bad
        print(f"[{i}] cos={cos:.8f} max_abs={diff:.3e}{'  <-- FAIL' if bad else ''}  {t[:42]!r}")

    print(f"worst_cos={worst_cos:.8f} worst_diff={worst_diff:.3e} dim={len(c_vecs[0])}")
    if fails:
        print(f"failed: {fails}/{len(texts)} e5 parity cases", file=sys.stderr)
        return 1
    print(f"ok: e5/XLM-R parity for {len(texts)} cases ({args.model})")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())