diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 12:43:49 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 12:43:49 +0200 |
| commit | 35ab9c12ba4d396e6cba99d6ea947e26becedad4 (patch) | |
| tree | b9cbbf9d0ade9ac7421782f2630d7330fba06c07 | |
| download | langdet-35ab9c12ba4d396e6cba99d6ea947e26becedad4.tar.gz langdet-35ab9c12ba4d396e6cba99d6ea947e26becedad4.zip | |
Init
| -rw-r--r-- | .gitignore | 2 | ||||
| -rw-r--r-- | README.md | 100 | ||||
| -rw-r--r-- | download_model.py | 84 |
3 files changed, 186 insertions, 0 deletions
diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..99684c1 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +/model/ +/__pycache__/ diff --git a/README.md b/README.md new file mode 100644 index 0000000..efd292c --- /dev/null +++ b/README.md @@ -0,0 +1,100 @@ +# XLM-R language detection: ffwd compatibility + +`papluca/xlm-roberta-base-language-detection` is **not supported as a language +classifier by the inspected ffwd implementation**. Its XLM-R encoder is supported, +but ffwd does not load or execute the classification head. Loading the checkpoint +as an embedding model would produce 768 numbers, not 20 language scores. + +## Compatibility findings + +The reference source is `/Users/ogz/x/ffwd`. + +- `README.md` lists XLM-R support for text embeddings. +- `libffwd/config.c` recognizes `xlm-roberta`, sets the correct position offset, + and defaults to mean pooling. It does not interpret the classification + architecture or `id2label`. +- `libffwd/model.c`, in `load_bert_weights`, accepts the checkpoint's `roberta.` + tensor prefix and loads the encoder, but not `classifier.*`. +- `libffwd/forward.c` implements the encoder and embedding pooling, including + first-token pooling (`FFWD_POOL_CLS`), but no classification head. +- `libffwd/ffwd.h` exposes embeddings and reranking, not sequence classification. +- `libffwd/tokenize.c` supports the checkpoint's `sentencepiece.bpe.model` format + and wraps text with `<s>` and `</s>`. + +The checkpoint declares `XLMRobertaForSequenceClassification`. Its safetensors +header contains these F32 classifier tensors: + +| Tensor | Shape | +| --- | --- | +| `classifier.dense.weight` | `[768, 768]` | +| `classifier.dense.bias` | `[768]` | +| `classifier.out_proj.weight` | `[20, 768]` | +| `classifier.out_proj.bias` | `[20]` | + +## Required inference work + +1. Reuse ffwd's CPU encoder, SentencePiece tokenizer, safetensors reader, and + ARM NEON kernels. On macOS ARM64, matrix multiplication uses Apple's Accelerate + framework; Metal and MLX are not required. +2. Load and validate the four classifier tensors, plus `id2label` from + `config.json`. +3. Take the final hidden state of the first token (`<s>`), not the mean of all + tokens. Do not apply L2 normalization. Compute, with column vectors: + + ```text + h = final_hidden_state[0] # 768 values + z = tanh(classifier.dense.weight @ h + classifier.dense.bias) + logits = classifier.out_proj.weight @ z + classifier.out_proj.bias + probabilities = softmax(logits) # 20 values + language = id2label[argmax(probabilities)] + ``` + + Dropout is disabled during inference. Stable softmax subtracts the largest + logit before exponentiation. +4. Add a classification runner that returns labels and scores. Keep each input + within 512 tokens, including both special tokens; any truncation must retain + the final `</s>` token. +5. Compare token IDs, logits, and probabilities against Hugging Face Transformers + on CPU, including all 20 supported languages, Unicode, empty input, mixed + sequence lengths, and the length limit. + +This requires additional classification code, not retraining or a new transformer +architecture. A configuration change alone cannot execute the missing head. + +## Download + +Python 3 and internet access are the only requirements. No Hugging Face token or +third-party Python packages are needed for this public checkpoint. + +```sh +python3 download_model.py +# Alternatively, choose a destination: +python3 download_model.py ./models/language-detection +``` + +The default destination is `model/` beside the script. The download uses revision +`9865598389ca9d95637462f743f683b51d75b87b` and includes safetensors weights, +configuration, tokenizer files, and the model card. It excludes duplicate PyTorch +and TensorFlow weights. Allow about 1.13 GB of disk space for a fresh download. + +The script checks each file's size and hash against Hugging Face metadata. Large +LFS files use SHA-256; ordinary files use Git blob SHA-1. Verified files are reused. +Downloads go to `.part` files and become final files only after verification. +Interrupted files restart on the next run; they do not resume from their previous +byte offset. + +## Verification status + +- The host reports `Darwin arm64`. +- All seven files were downloaded and their sizes and hashes verified. A second + script run verified and reused every file. +- The local safetensors header was checked for the four classifier shapes and + F32 types, and its final data offset matches the file size. +- The classification formula matches Transformers 4.12.5, the version recorded + in the model configuration. +- No ffwd inference code is included or built here. No language predictions or + numerical comparisons against Transformers have been run. Encoder compatibility + is based on source and checkpoint inspection, not a runtime test. + +The remaining work is the classification runner and the numerical verification +listed above. The reference ffwd source is unchanged. diff --git a/download_model.py b/download_model.py new file mode 100644 index 0000000..839156b --- /dev/null +++ b/download_model.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Download and verify a pinned XLM-R language-detection checkpoint.""" + +import argparse +import hashlib +import json +from pathlib import Path +import sys +import urllib.error +import urllib.request + + +REPO = "papluca/xlm-roberta-base-language-detection" +REVISION = "9865598389ca9d95637462f743f683b51d75b87b" +FILES = ( + "README.md", + "config.json", + "tokenizer_config.json", + "special_tokens_map.json", + "tokenizer.json", + "sentencepiece.bpe.model", + "model.safetensors", +) +BASE_URL = f"https://huggingface.co/{REPO}/resolve/{REVISION}" + + +def verify(path, metadata): + if not path.is_file() or path.stat().st_size != metadata["size"]: + return False + if "lfs" in metadata: + digest = hashlib.sha256() + expected = metadata["lfs"]["oid"] + else: + digest = hashlib.sha1() + digest.update(f'blob {metadata["size"]}\0'.encode()) + expected = metadata["oid"] + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() == expected + + +def download(directory, metadata): + name = metadata["path"] + target = directory / name + if verify(target, metadata): + print(f"Verified {target}", flush=True) + return + partial = target.with_name(target.name + ".part") + print(f'Downloading {name} ({metadata["size"]:,} bytes)', flush=True) + with urllib.request.urlopen(f"{BASE_URL}/{name}?download=true", timeout=60) as response: + with partial.open("wb") as stream: + for chunk in iter(lambda: response.read(1024 * 1024), b""): + stream.write(chunk) + if not verify(partial, metadata): + raise ValueError(f"Size or checksum mismatch: {partial}") + partial.replace(target) + print(f"Verified {target}", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "directory", nargs="?", type=Path, + default=Path(__file__).resolve().parent / "model", + help="Destination directory (default: model beside this script).", + ) + args = parser.parse_args() + url = f"https://huggingface.co/api/models/{REPO}/tree/{REVISION}?recursive=false" + try: + with urllib.request.urlopen(url, timeout=60) as response: + metadata = {item["path"]: item for item in json.load(response)} + args.directory.mkdir(parents=True, exist_ok=True) + for name in FILES: + download(args.directory, metadata[name]) + except (OSError, ValueError, KeyError, urllib.error.URLError) as error: + print(f"Download failed: {error}", file=sys.stderr) + return 1 + print(f"Checkpoint ready in {args.directory} (revision {REVISION}).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) |
