aboutsummaryrefslogtreecommitdiff
path: root/download_model.py
diff options
context:
space:
mode:
authorOnur Gündüz <ogunduz@gmail.com>2026-09-29 12:43:49 +0200
committerOnur Gündüz <ogunduz@gmail.com>2026-09-29 12:43:49 +0200
commit35ab9c12ba4d396e6cba99d6ea947e26becedad4 (patch)
treeb9cbbf9d0ade9ac7421782f2630d7330fba06c07 /download_model.py
downloadlangdet-35ab9c12ba4d396e6cba99d6ea947e26becedad4.tar.gz
langdet-35ab9c12ba4d396e6cba99d6ea947e26becedad4.zip
Init
Diffstat (limited to 'download_model.py')
-rw-r--r--download_model.py84
1 files changed, 84 insertions, 0 deletions
diff --git a/download_model.py b/download_model.py
new file mode 100644
index 0000000..839156b
--- /dev/null
+++ b/download_model.py
@@ -0,0 +1,84 @@
+#!/usr/bin/env python3
+"""Download and verify a pinned XLM-R language-detection checkpoint."""
+
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import sys
+import urllib.error
+import urllib.request
+
+
+REPO = "papluca/xlm-roberta-base-language-detection"
+REVISION = "9865598389ca9d95637462f743f683b51d75b87b"
+FILES = (
+ "README.md",
+ "config.json",
+ "tokenizer_config.json",
+ "special_tokens_map.json",
+ "tokenizer.json",
+ "sentencepiece.bpe.model",
+ "model.safetensors",
+)
+BASE_URL = f"https://huggingface.co/{REPO}/resolve/{REVISION}"
+
+
+def verify(path, metadata):
+ if not path.is_file() or path.stat().st_size != metadata["size"]:
+ return False
+ if "lfs" in metadata:
+ digest = hashlib.sha256()
+ expected = metadata["lfs"]["oid"]
+ else:
+ digest = hashlib.sha1()
+ digest.update(f'blob {metadata["size"]}\0'.encode())
+ expected = metadata["oid"]
+ with path.open("rb") as stream:
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+ digest.update(chunk)
+ return digest.hexdigest() == expected
+
+
+def download(directory, metadata):
+ name = metadata["path"]
+ target = directory / name
+ if verify(target, metadata):
+ print(f"Verified {target}", flush=True)
+ return
+ partial = target.with_name(target.name + ".part")
+ print(f'Downloading {name} ({metadata["size"]:,} bytes)', flush=True)
+ with urllib.request.urlopen(f"{BASE_URL}/{name}?download=true", timeout=60) as response:
+ with partial.open("wb") as stream:
+ for chunk in iter(lambda: response.read(1024 * 1024), b""):
+ stream.write(chunk)
+ if not verify(partial, metadata):
+ raise ValueError(f"Size or checksum mismatch: {partial}")
+ partial.replace(target)
+ print(f"Verified {target}", flush=True)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument(
+ "directory", nargs="?", type=Path,
+ default=Path(__file__).resolve().parent / "model",
+ help="Destination directory (default: model beside this script).",
+ )
+ args = parser.parse_args()
+ url = f"https://huggingface.co/api/models/{REPO}/tree/{REVISION}?recursive=false"
+ try:
+ with urllib.request.urlopen(url, timeout=60) as response:
+ metadata = {item["path"]: item for item in json.load(response)}
+ args.directory.mkdir(parents=True, exist_ok=True)
+ for name in FILES:
+ download(args.directory, metadata[name])
+ except (OSError, ValueError, KeyError, urllib.error.URLError) as error:
+ print(f"Download failed: {error}", file=sys.stderr)
+ return 1
+ print(f"Checkpoint ready in {args.directory} (revision {REVISION}).")
+ return 0
+
+
+if __name__ == "__main__":
+ raise SystemExit(main())