aboutsummaryrefslogtreecommitdiff
path: root/download_model.py
blob: 839156b6f9cba17cae2aa41fb413669a8ca73fdf (plain) (blame)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
#!/usr/bin/env python3
"""Download and verify a pinned XLM-R language-detection checkpoint."""

import argparse
import hashlib
import json
from pathlib import Path
import sys
import urllib.error
import urllib.request


REPO = "papluca/xlm-roberta-base-language-detection"
REVISION = "9865598389ca9d95637462f743f683b51d75b87b"
FILES = (
    "README.md",
    "config.json",
    "tokenizer_config.json",
    "special_tokens_map.json",
    "tokenizer.json",
    "sentencepiece.bpe.model",
    "model.safetensors",
)
BASE_URL = f"https://huggingface.co/{REPO}/resolve/{REVISION}"


def verify(path, metadata):
    if not path.is_file() or path.stat().st_size != metadata["size"]:
        return False
    if "lfs" in metadata:
        digest = hashlib.sha256()
        expected = metadata["lfs"]["oid"]
    else:
        digest = hashlib.sha1()
        digest.update(f'blob {metadata["size"]}\0'.encode())
        expected = metadata["oid"]
    with path.open("rb") as stream:
        for chunk in iter(lambda: stream.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest() == expected


def download(directory, metadata):
    name = metadata["path"]
    target = directory / name
    if verify(target, metadata):
        print(f"Verified {target}", flush=True)
        return
    partial = target.with_name(target.name + ".part")
    print(f'Downloading {name} ({metadata["size"]:,} bytes)', flush=True)
    with urllib.request.urlopen(f"{BASE_URL}/{name}?download=true", timeout=60) as response:
        with partial.open("wb") as stream:
            for chunk in iter(lambda: response.read(1024 * 1024), b""):
                stream.write(chunk)
    if not verify(partial, metadata):
        raise ValueError(f"Size or checksum mismatch: {partial}")
    partial.replace(target)
    print(f"Verified {target}", flush=True)


def main():
    parser = argparse.ArgumentParser(description=__doc__)
    parser.add_argument(
        "directory", nargs="?", type=Path,
        default=Path(__file__).resolve().parent / "model",
        help="Destination directory (default: model beside this script).",
    )
    args = parser.parse_args()
    url = f"https://huggingface.co/api/models/{REPO}/tree/{REVISION}?recursive=false"
    try:
        with urllib.request.urlopen(url, timeout=60) as response:
            metadata = {item["path"]: item for item in json.load(response)}
        args.directory.mkdir(parents=True, exist_ok=True)
        for name in FILES:
            download(args.directory, metadata[name])
    except (OSError, ValueError, KeyError, urllib.error.URLError) as error:
        print(f"Download failed: {error}", file=sys.stderr)
        return 1
    print(f"Checkpoint ready in {args.directory} (revision {REVISION}).")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())