diff options
| author | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 12:43:49 +0200 |
|---|---|---|
| committer | Onur Gündüz <ogunduz@gmail.com> | 2026-09-29 12:43:49 +0200 |
| commit | 35ab9c12ba4d396e6cba99d6ea947e26becedad4 (patch) | |
| tree | b9cbbf9d0ade9ac7421782f2630d7330fba06c07 /download_model.py | |
| download | langdet-35ab9c12ba4d396e6cba99d6ea947e26becedad4.tar.gz langdet-35ab9c12ba4d396e6cba99d6ea947e26becedad4.zip | |
Init
Diffstat (limited to 'download_model.py')
| -rw-r--r-- | download_model.py | 84 |
1 files changed, 84 insertions, 0 deletions
diff --git a/download_model.py b/download_model.py new file mode 100644 index 0000000..839156b --- /dev/null +++ b/download_model.py @@ -0,0 +1,84 @@ +#!/usr/bin/env python3 +"""Download and verify a pinned XLM-R language-detection checkpoint.""" + +import argparse +import hashlib +import json +from pathlib import Path +import sys +import urllib.error +import urllib.request + + +REPO = "papluca/xlm-roberta-base-language-detection" +REVISION = "9865598389ca9d95637462f743f683b51d75b87b" +FILES = ( + "README.md", + "config.json", + "tokenizer_config.json", + "special_tokens_map.json", + "tokenizer.json", + "sentencepiece.bpe.model", + "model.safetensors", +) +BASE_URL = f"https://huggingface.co/{REPO}/resolve/{REVISION}" + + +def verify(path, metadata): + if not path.is_file() or path.stat().st_size != metadata["size"]: + return False + if "lfs" in metadata: + digest = hashlib.sha256() + expected = metadata["lfs"]["oid"] + else: + digest = hashlib.sha1() + digest.update(f'blob {metadata["size"]}\0'.encode()) + expected = metadata["oid"] + with path.open("rb") as stream: + for chunk in iter(lambda: stream.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() == expected + + +def download(directory, metadata): + name = metadata["path"] + target = directory / name + if verify(target, metadata): + print(f"Verified {target}", flush=True) + return + partial = target.with_name(target.name + ".part") + print(f'Downloading {name} ({metadata["size"]:,} bytes)', flush=True) + with urllib.request.urlopen(f"{BASE_URL}/{name}?download=true", timeout=60) as response: + with partial.open("wb") as stream: + for chunk in iter(lambda: response.read(1024 * 1024), b""): + stream.write(chunk) + if not verify(partial, metadata): + raise ValueError(f"Size or checksum mismatch: {partial}") + partial.replace(target) + print(f"Verified {target}", flush=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument( + "directory", nargs="?", type=Path, + default=Path(__file__).resolve().parent / "model", + help="Destination directory (default: model beside this script).", + ) + args = parser.parse_args() + url = f"https://huggingface.co/api/models/{REPO}/tree/{REVISION}?recursive=false" + try: + with urllib.request.urlopen(url, timeout=60) as response: + metadata = {item["path"]: item for item in json.load(response)} + args.directory.mkdir(parents=True, exist_ok=True) + for name in FILES: + download(args.directory, metadata[name]) + except (OSError, ValueError, KeyError, urllib.error.URLError) as error: + print(f"Download failed: {error}", file=sys.stderr) + return 1 + print(f"Checkpoint ready in {args.directory} (revision {REVISION}).") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) |
