aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 12:43:49 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 12:43:49 +0200
commit35ab9c12ba4d396e6cba99d6ea947e26becedad4 (patch)
treeb9cbbf9d0ade9ac7421782f2630d7330fba06c07
downloadlangdet-35ab9c12ba4d396e6cba99d6ea947e26becedad4.tar.gz
langdet-35ab9c12ba4d396e6cba99d6ea947e26becedad4.zip
Init
-rw-r--r--.gitignore2
-rw-r--r--README.md100
-rw-r--r--download_model.py84
3 files changed, 186 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..99684c1
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,2 @@
+/model/
+/__pycache__/
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..efd292c
--- /dev/null
+++ b/README.md
@@ -0,0 +1,100 @@
+# XLM-R language detection: ffwd compatibility
+
+`papluca/xlm-roberta-base-language-detection` is **not supported as a language
+classifier by the inspected ffwd implementation**. Its XLM-R encoder is supported,
+but ffwd does not load or execute the classification head. Loading the checkpoint
+as an embedding model would produce 768 numbers, not 20 language scores.
+
+## Compatibility findings
+
+The reference source is `/Users/ogz/x/ffwd`.
+
+- `README.md` lists XLM-R support for text embeddings.
+- `libffwd/config.c` recognizes `xlm-roberta`, sets the correct position offset,
+ and defaults to mean pooling. It does not interpret the classification
+ architecture or `id2label`.
+- `libffwd/model.c`, in `load_bert_weights`, accepts the checkpoint's `roberta.`
+ tensor prefix and loads the encoder, but not `classifier.*`.
+- `libffwd/forward.c` implements the encoder and embedding pooling, including
+ first-token pooling (`FFWD_POOL_CLS`), but no classification head.
+- `libffwd/ffwd.h` exposes embeddings and reranking, not sequence classification.
+- `libffwd/tokenize.c` supports the checkpoint's `sentencepiece.bpe.model` format
+ and wraps text with `<s>` and `</s>`.
+
+The checkpoint declares `XLMRobertaForSequenceClassification`. Its safetensors
+header contains these F32 classifier tensors:
+
+| Tensor | Shape |
+| --- | --- |
+| `classifier.dense.weight` | `[768, 768]` |
+| `classifier.dense.bias` | `[768]` |
+| `classifier.out_proj.weight` | `[20, 768]` |
+| `classifier.out_proj.bias` | `[20]` |
+
+## Required inference work
+
+1. Reuse ffwd's CPU encoder, SentencePiece tokenizer, safetensors reader, and
+ ARM NEON kernels. On macOS ARM64, matrix multiplication uses Apple's Accelerate
+ framework; Metal and MLX are not required.
+2. Load and validate the four classifier tensors, plus `id2label` from
+ `config.json`.
+3. Take the final hidden state of the first token (`<s>`), not the mean of all
+ tokens. Do not apply L2 normalization. Compute, with column vectors:
+
+ ```text
+ h = final_hidden_state[0] # 768 values
+ z = tanh(classifier.dense.weight @ h + classifier.dense.bias)
+ logits = classifier.out_proj.weight @ z + classifier.out_proj.bias
+ probabilities = softmax(logits) # 20 values
+ language = id2label[argmax(probabilities)]
+ ```
+
+ Dropout is disabled during inference. Stable softmax subtracts the largest
+ logit before exponentiation.
+4. Add a classification runner that returns labels and scores. Keep each input
+ within 512 tokens, including both special tokens; any truncation must retain
+ the final `</s>` token.
+5. Compare token IDs, logits, and probabilities against Hugging Face Transformers
+ on CPU, including all 20 supported languages, Unicode, empty input, mixed
+ sequence lengths, and the length limit.
+
+This requires additional classification code, not retraining or a new transformer
+architecture. A configuration change alone cannot execute the missing head.
+
+## Download
+
+Python 3 and internet access are the only requirements. No Hugging Face token or
+third-party Python packages are needed for this public checkpoint.
+
+```sh
+python3 download_model.py
+# Alternatively, choose a destination:
+python3 download_model.py ./models/language-detection
+```
+
+The default destination is `model/` beside the script. The download uses revision
+`9865598389ca9d95637462f743f683b51d75b87b` and includes safetensors weights,
+configuration, tokenizer files, and the model card. It excludes duplicate PyTorch
+and TensorFlow weights. Allow about 1.13 GB of disk space for a fresh download.
+
+The script checks each file's size and hash against Hugging Face metadata. Large
+LFS files use SHA-256; ordinary files use Git blob SHA-1. Verified files are reused.
+Downloads go to `.part` files and become final files only after verification.
+Interrupted files restart on the next run; they do not resume from their previous
+byte offset.
+
+## Verification status
+
+- The host reports `Darwin arm64`.
+- All seven files were downloaded and their sizes and hashes verified. A second
+ script run verified and reused every file.
+- The local safetensors header was checked for the four classifier shapes and
+ F32 types, and its final data offset matches the file size.
+- The classification formula matches Transformers 4.12.5, the version recorded
+ in the model configuration.
+- No ffwd inference code is included or built here. No language predictions or
+ numerical comparisons against Transformers have been run. Encoder compatibility
+ is based on source and checkpoint inspection, not a runtime test.
+
+The remaining work is the classification runner and the numerical verification
+listed above. The reference ffwd source is unchanged.
diff --git a/download_model.py b/download_model.py
new file mode 100644
index 0000000..839156b
--- /dev/null
+++ b/download_model.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Download and verify a pinned XLM-R language-detection checkpoint."""
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import sys
+import urllib.error
+import urllib.request
+
+
+REPO = "papluca/xlm-roberta-base-language-detection"
+REVISION = "9865598389ca9d95637462f743f683b51d75b87b"
+FILES = (
+ "README.md",
+ "config.json",
+ "tokenizer_config.json",
+ "special_tokens_map.json",
+ "tokenizer.json",
+ "sentencepiece.bpe.model",
+ "model.safetensors",
+)
+BASE_URL = f"https://huggingface.co/{REPO}/resolve/{REVISION}"
+
+
+def verify(path, metadata):
+ if not path.is_file() or path.stat().st_size != metadata["size"]:
+ return False
+ if "lfs" in metadata:
+ digest = hashlib.sha256()
+ expected = metadata["lfs"]["oid"]
+ else:
+ digest = hashlib.sha1()
+ digest.update(f'blob {metadata["size"]}\0'.encode())
+ expected = metadata["oid"]
+ with path.open("rb") as stream:
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest() == expected
+
+
+def download(directory, metadata):
+ name = metadata["path"]
+ target = directory / name
+ if verify(target, metadata):
+ print(f"Verified {target}", flush=True)
+ return
+ partial = target.with_name(target.name + ".part")
+ print(f'Downloading {name} ({metadata["size"]:,} bytes)', flush=True)
+ with urllib.request.urlopen(f"{BASE_URL}/{name}?download=true", timeout=60) as response:
+ with partial.open("wb") as stream:
+ for chunk in iter(lambda: response.read(1024 * 1024), b""):
+ stream.write(chunk)
+ if not verify(partial, metadata):
+ raise ValueError(f"Size or checksum mismatch: {partial}")
+ partial.replace(target)
+ print(f"Verified {target}", flush=True)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "directory", nargs="?", type=Path,
+ default=Path(__file__).resolve().parent / "model",
+ help="Destination directory (default: model beside this script).",
+ )
+ args = parser.parse_args()
+ url = f"https://huggingface.co/api/models/{REPO}/tree/{REVISION}?recursive=false"
+ try:
+ with urllib.request.urlopen(url, timeout=60) as response:
+ metadata = {item["path"]: item for item in json.load(response)}
+ args.directory.mkdir(parents=True, exist_ok=True)
+ for name in FILES:
+ download(args.directory, metadata[name])
+ except (OSError, ValueError, KeyError, urllib.error.URLError) as error:
+ print(f"Download failed: {error}", file=sys.stderr)
+ return 1
+ print(f"Checkpoint ready in {args.directory} (revision {REVISION}).")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())