#!/usr/bin/env python3 """Download and verify a pinned XLM-R language-detection checkpoint.""" import argparse import hashlib import json from pathlib import Path import sys import urllib.error import urllib.request REPO = "papluca/xlm-roberta-base-language-detection" REVISION = "9865598389ca9d95637462f743f683b51d75b87b" FILES = ( "README.md", "config.json", "tokenizer_config.json", "special_tokens_map.json", "tokenizer.json", "sentencepiece.bpe.model", "model.safetensors", ) BASE_URL = f"https://huggingface.co/{REPO}/resolve/{REVISION}" def verify(path, metadata): if not path.is_file() or path.stat().st_size != metadata["size"]: return False if "lfs" in metadata: digest = hashlib.sha256() expected = metadata["lfs"]["oid"] else: digest = hashlib.sha1() digest.update(f'blob {metadata["size"]}\0'.encode()) expected = metadata["oid"] with path.open("rb") as stream: for chunk in iter(lambda: stream.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() == expected def download(directory, metadata): name = metadata["path"] target = directory / name if verify(target, metadata): print(f"Verified {target}", flush=True) return partial = target.with_name(target.name + ".part") print(f'Downloading {name} ({metadata["size"]:,} bytes)', flush=True) with urllib.request.urlopen(f"{BASE_URL}/{name}?download=true", timeout=60) as response: with partial.open("wb") as stream: for chunk in iter(lambda: response.read(1024 * 1024), b""): stream.write(chunk) if not verify(partial, metadata): raise ValueError(f"Size or checksum mismatch: {partial}") partial.replace(target) print(f"Verified {target}", flush=True) def main(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument( "directory", nargs="?", type=Path, default=Path(__file__).resolve().parent / "model", help="Destination directory (default: model beside this script).", ) args = parser.parse_args() url = f"https://huggingface.co/api/models/{REPO}/tree/{REVISION}?recursive=false" try: with urllib.request.urlopen(url, timeout=60) as response: metadata = {item["path"]: item for item in json.load(response)} args.directory.mkdir(parents=True, exist_ok=True) for name in FILES: download(args.directory, metadata[name]) except (OSError, ValueError, KeyError, urllib.error.URLError) as error: print(f"Download failed: {error}", file=sys.stderr) return 1 print(f"Checkpoint ready in {args.directory} (revision {REVISION}).") return 0 if __name__ == "__main__": raise SystemExit(main())