Text Generation
Transformers
Safetensors
deepseek_v4
qfs
fixture
random-init
cpu
reproducibility
random-architecture-fixture
tiny
custom_code
Instructions to use malaiwah/deepseek-v4-tiny-random-bf16 with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use malaiwah/deepseek-v4-tiny-random-bf16 with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-generation", model="malaiwah/deepseek-v4-tiny-random-bf16", trust_remote_code=True)# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForCausalLM tokenizer = AutoTokenizer.from_pretrained("malaiwah/deepseek-v4-tiny-random-bf16", trust_remote_code=True) model = AutoModelForCausalLM.from_pretrained("malaiwah/deepseek-v4-tiny-random-bf16", trust_remote_code=True, device_map="auto") - Notebooks
- Google Colab
- Kaggle
- Local Apps Settings
- vLLM
How to use malaiwah/deepseek-v4-tiny-random-bf16 with vLLM:
Install from pip and serve model
# Install vLLM from pip: pip install vllm # Start the vLLM server: vllm serve "malaiwah/deepseek-v4-tiny-random-bf16" # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:8000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "malaiwah/deepseek-v4-tiny-random-bf16", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker
docker model run hf.co/malaiwah/deepseek-v4-tiny-random-bf16
- SGLang
How to use malaiwah/deepseek-v4-tiny-random-bf16 with SGLang:
Install from pip and serve model
# Install SGLang from pip: pip install sglang # Start the SGLang server: python3 -m sglang.launch_server \ --model-path "malaiwah/deepseek-v4-tiny-random-bf16" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "malaiwah/deepseek-v4-tiny-random-bf16", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }'Use Docker images
docker run --gpus all \ --shm-size 32g \ -p 30000:30000 \ -v ~/.cache/huggingface:/root/.cache/huggingface \ --env "HF_TOKEN=<secret>" \ --ipc=host \ lmsysorg/sglang:latest \ python3 -m sglang.launch_server \ --model-path "malaiwah/deepseek-v4-tiny-random-bf16" \ --host 0.0.0.0 \ --port 30000 # Call the server using curl (OpenAI-compatible API): curl -X POST "http://localhost:30000/v1/completions" \ -H "Content-Type: application/json" \ --data '{ "model": "malaiwah/deepseek-v4-tiny-random-bf16", "prompt": "Once upon a time,", "max_tokens": 512, "temperature": 0.5 }' - Docker Model Runner
How to use malaiwah/deepseek-v4-tiny-random-bf16 with Docker Model Runner:
docker model run hf.co/malaiwah/deepseek-v4-tiny-random-bf16
Download release.py from malaiwah/deepseek-v4-tiny-random-bf16: direct link, hf CLI and curl.
- Browser
- Download file 22.5 kB
-
https://huggingface.co/malaiwah/deepseek-v4-tiny-random-bf16/resolve/main/release.py
- Command line
-
hf download hf://malaiwah/deepseek-v4-tiny-random-bf16/release.py
-
curl -L -o release.py https://huggingface.co/malaiwah/deepseek-v4-tiny-random-bf16/resolve/main/release.py
22.5 kB
| #!/usr/bin/env python3 | |
| """Stage privately by default; --publish only releases an unchanged successful stage.""" | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| from pathlib import Path | |
| import shutil | |
| import subprocess | |
| import sys | |
| import time | |
| import traceback | |
| from build_fixture import LINEAGE, VISION_LINEAGE, SCOPE, sha256, write_json | |
| HERE = Path(__file__).resolve().parent | |
| MODEL = "malaiwah/deepseek-v4-tiny-random-bf16" | |
| DATASET = "malaiwah/deepseek-v4-tiny-cpu-repro-v1" | |
| SCRIPTS = ("build_fixture.py", "build_panel.py", "verify_native.py", "verify_cpu.py", "release.py", "requirements-cpu.txt", "LICENSE", "support.json", "deepseek_v4_qfs.py", "LICENSE-APACHE-2.0") | |
| def inventory(directory): | |
| result = {} | |
| for path in sorted(directory.rglob("*")): | |
| if path.is_symlink(): | |
| raise ValueError(f"Refusing symlink {path}") | |
| if path.is_file(): | |
| relative = path.relative_to(directory) | |
| if any(part.startswith(".") or part == "__pycache__" for part in relative.parts): | |
| raise ValueError(f"Unexpected hidden/cache file {path}") | |
| result[relative.as_posix()] = {"bytes": path.stat().st_size, "sha256": sha256(path)} | |
| return result | |
| def command(name, script, arguments, work, qfs): | |
| env = os.environ.copy() | |
| env.update({k: "2" for k in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", "NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS")}) | |
| env.update({"HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "TOKENIZERS_PARALLELISM": "false", "PYTHONDONTWRITEBYTECODE": "1"}) | |
| if name == "native": | |
| env["HF_HUB_OFFLINE"] = "0" | |
| env["TRANSFORMERS_OFFLINE"] = "0" | |
| argv = [sys.executable, str(HERE / script), *map(str, arguments)] | |
| started = time.monotonic() | |
| with (work / (name + ".stdout.log")).open("xb") as out, (work / (name + ".stderr.log")).open("xb") as err: | |
| result = subprocess.run(argv, cwd=qfs, env=env, stdout=out, stderr=err, timeout=1200) | |
| write_json(work / (name + ".command.json"), {"argv": argv, "returncode": result.returncode, "elapsed_seconds": time.monotonic() - started}) | |
| if result.returncode: | |
| raise RuntimeError(f"{name} failed; inspect {work / (name + '.stderr.log')}") | |
| def verify_remote(api, repository, kind, revision, local, cache, *, anonymous): | |
| from huggingface_hub import hf_hub_download | |
| expected = inventory(local) | |
| token = False if anonymous else api.token | |
| remote_names = set(api.list_repo_files(repository, repo_type=kind, revision=revision, token=token)) | |
| if remote_names - set(expected) - {".gitattributes"} or set(expected) - remote_names: | |
| raise ValueError(f"Unexpected remote file closure for {repository}") | |
| for name, record in expected.items(): | |
| fetched = Path(hf_hub_download(repository, name, repo_type=kind, revision=revision, token=token, cache_dir=cache, force_download=True)) | |
| if fetched.stat().st_size != record["bytes"] or sha256(fetched) != record["sha256"]: | |
| raise ValueError(f"Hub bytes differ: {repository}/{name}") | |
| return {"repository": repository, "repo_type": kind, "revision": revision, "anonymous": anonymous, "files_verified": expected} | |
| def upload(api, repository, kind, directory, parent=None): | |
| return api.upload_folder(repo_id=repository, repo_type=kind, folder_path=directory, allow_patterns=list(inventory(directory)), commit_message="Complete native tiny CPU fixture and measured reproduction evidence", parent_commit=parent).oid | |
| def cards(model, evidence, pin, source): | |
| build = json.loads((model / "build-manifest.json").read_text()) | |
| native = json.loads((evidence / "native-verification.json").read_text()) | |
| measured = json.loads((evidence / "evidence-summary.json").read_text()) | |
| if native["status"] != "passed" or measured["status"] != "passed": | |
| raise ValueError("Cards require actual passed runtime evidence") | |
| comparison = measured["comparison"] | |
| support = json.loads((HERE / "support.json").read_text()) | |
| support["verified_runtime"] = { | |
| "status": "measured_passed", "native_class": native["model_class"], | |
| "versions": build["versions"], "model_pin": pin, | |
| "code_repository": MODEL, "code_revision": pin, "runtime_correction": build["runtime_correction"], | |
| "native_report_sha256": sha256(evidence / "native-verification.json"), | |
| "capture_report_sha256": sha256(evidence / "evidence-summary.json"), | |
| "measurement_scope": "Random BF16 text fixture using native primitives with explicit reviewed RMSNorm correction; not unmodified native arithmetic or published FP8/FP4, DSpark or vision.", | |
| "comparison": comparison, | |
| } | |
| for target in (model, evidence): | |
| write_json(target / "support.json", support) | |
| body = f"""# DeepSeek-V4 tiny corrected-native-primitives CPU text fixture | |
| Complete **randomly initialized**, untrained `QFSDeepseekV4ForCausalLM` text class | |
| using Transformers5.16.1 native primitives and a reviewed RMSNorm arithmetic correction. | |
| No upstream weights, paid GPU/cloud compute or useful-model claim. | |
| This is not unmodified native Transformers or the complete production release. | |
| ## Architecture and scope | |
| Text lineage: [{LINEAGE['repository']}@{LINEAGE['revision']}](https://huggingface.co/{LINEAGE['repository']}/tree/{LINEAGE['revision']}). | |
| Vision coverage research: [{VISION_LINEAGE['repository']}@{VISION_LINEAGE['revision']}](https://huggingface.co/{VISION_LINEAGE['repository']}/tree/{VISION_LINEAGE['revision']}). | |
| {SCOPE} | |
| The unmodified native RMSNorm multiplies a BF16 normalized activation by its required | |
| FP32 weight, promoting output to FP32 and causing the next BF16 linear projection to fail. | |
| The wrapper computes weighted normalization in FP32 and casts the complete result to | |
| the input activation dtype. Only this model instance's RMSNorm modules are replaced; | |
| no global/installed-library patches, autocast, forward-hook corrections or weight changes. | |
| This moves an arithmetic rounding boundary and is explicitly not native bit-equivalence. | |
| `deepseek_v4_qfs.py` and its Apache-2.0 license are retained in the immutable model pin. | |
| Observed replacements and source hashes: `{json.dumps(build['runtime_correction'])}`. | |
| Four layers use explicit native legacy schedule selectors `compress_ratios=[0,0,4,128]`, | |
| normalized to two sliding-only, one CSA, one HCA layer. Actual rates are CSA4/HCA16. | |
| All layers retain a local window8, shared K=V multi-query attention, learned sinks, | |
| low-rank query projection and grouped low-rank output (Q4/KV1, head16, Q-rank16, | |
| two output groups of rank8). Hidden64, expert intermediate32, eight routed experts, | |
| top2 plus one shared expert; three seeded frozen hash routers then one learned router | |
| use sqrt-softplus scores. mHC4 and20 Sinkhorn iterations are preserved. Native gate/up | |
| clipping limit10 is preserved. Partial interleaved RoPE uses dimension4, with separate | |
| main/compressor bases. No production YaRN/long-context scaling claim. | |
| The small real tokenizer is256 byte-level BPE tokens without merges plus16 special/reserved | |
| entries (272 total; IDs0/1/2/3 pad/BOS/EOS/unknown). The untied head is complete. | |
| The upstream flat `vision_*` fields describe a32-layer vision encoder plus aligner, | |
| not instantiated by this native text class. Upstream `dspark_*` fields and MTP are | |
| not executed. A declared `DeepseekV4ForCausalLM` architecture in Vision-Exp metadata | |
| does not establish native multimodal coverage. The upstream configs/licenses are retained. | |
| ## Actual measured evidence | |
| - Parameters: **{build['parameter_count']:,}**; semantic tensors: **{build['tensor_count']}**; seed: **{build['seed']}**. | |
| - Two independent generator subprocesses matched complete byte inventories. | |
| - Native FP32 initialization is converted by the actual BF16 loader; mandatory native FP32 | |
| retention is preserved before rounding. Exact values and dtypes pass corrected-class final reload | |
| and strict pinned QFS loading. Retained tensors: `{json.dumps(build['native_fp32_exceptions'])}`. | |
| - Native cache probes exercise CSA boundaries4, sliding eviction8, HCA16, and63/64; | |
| single- and multi-token continuation runs have exact same-schedule repeats. Cached-vs-cold | |
| differences are recorded, not asserted zero. Real cache remainder, entry count and CSA overlap | |
| states are checked. Actual hooks observe mHC row/column sums, hash/learned routes, grouped output, | |
| and CSA future-entry exclusion. Suffix perturbation checks causal prefix independence; | |
| beyond-limit activation inputs exercise the real clipped expert function. | |
| - Two fresh CPU processes, two threads, BF16, honest QFS lane `other`, sealed four64-token | |
| synthetic windows,252 shifted prediction positions; no quality benchmark/registry submission. | |
| - Strict tensor verification and forced numerical replay: metric **{comparison['metric']['value']}**, | |
| KL `{json.dumps(comparison['kl'], sort_keys=True)}`, top1 agreement **{comparison['top1_agreement']}**. | |
| The actual tokenwise artifact is252 positive float64 zeros, not just a summary claim. | |
| - Captured immutable private-stage checkpoint: `{MODEL}@{pin}`. | |
| QFS source revision `{source['revision']}`; executed file hashes accompany the evidence. | |
| Code repository is the same model repository at the same immutable pin, explicitly selected | |
| with `--trust-remote-code --code-repository --code-revision`. Capture weights/tokenizer remain | |
| local; HF network is permitted for pinned code verification. Captures are not entirely offline. | |
| See [native-verification.json](native-verification.json), [evidence-summary.json](evidence-summary.json), | |
| [build-manifest.json](build-manifest.json), [support.json](support.json), | |
| and the [evidence repository](https://huggingface.co/datasets/{DATASET}). The evidence repository root | |
| is an artifact bundle, **not a single root-format QFS dataset**; `first/` and `repeat/` are sealed roots. | |
| QFS retains its hash-proof reproduction-confirmation semantics while `--force-compute` also runs | |
| numerical replay. This proves same-stack/machine repeatability only. | |
| ## Reproduction and capture | |
| Python3.12, isolated CPU environment: `pip install -r requirements-cpu.txt`. | |
| With the recorded QFS checkout at `$QFS`, outputs must be new: | |
| ```bash | |
| python build_fixture.py --out /tmp/deepseek-v4-rebuilt | |
| python verify_native.py --model-dir /tmp/deepseek-v4-rebuilt --code-repository {MODEL} --code-revision {pin} --qfs-root "$QFS" --out /tmp/deepseek-v4-native.json | |
| python build_panel.py --model-dir /tmp/deepseek-v4-rebuilt --model-repository {MODEL} --model-revision {pin} --qfs-root "$QFS" --out /tmp/deepseek-v4-panel | |
| python verify_cpu.py --model-dir /tmp/deepseek-v4-rebuilt --model-repository {MODEL} --model-revision {pin} --code-repository {MODEL} --code-revision {pin} --panel-dir /tmp/deepseek-v4-panel --qfs-root "$QFS" --out /tmp/deepseek-v4-repro | |
| ``` | |
| `support.json` gives the real QFS capture argv for third-party authors with an explicit | |
| `--author`, including the corrected native-primitives BF16 text checkpoint path. Original releases declare dynamic | |
| FP8 e4m3/ue8m0 block128 storage plus `expert_dtype=fp4`. This BF16 fixture does **not** prove | |
| those kernels/conversions or the complete production model. `--drop-parallel-plan` addresses | |
| the HF quantizer parallel-plan rewrite bug, not unsupported packed expert formats. | |
| Never remove `quantization_config` from real upstream weights to make them load. | |
| Do not bypass missing/unexpected/mismatched/conversion-error guards. Full original-format | |
| qualification needs actual supported hardware/runtime and clean full-checkpoint loading; | |
| vision and DSpark additionally need their real implementations and modality/speculation probes. | |
| For an unchanged original config without `auto_map`, explicit `--code-repository {MODEL}` | |
| and `--code-revision {pin}` use only that pinned repository's verified `auto_map` as runtime | |
| class mapping. Wrapper config construction still uses all original checkpoint fields; | |
| no dimensions, quantization metadata or weights are replaced. This makes the runtime override | |
| explicit; it does not qualify untested original packed formats or ignored vision/DSpark modules. | |
| Publisher workflow: `release.py --stage-only --out NEW_DIR --qfs-root "$QFS"` requires authorization | |
| for the two fixed repository IDs, refuses collisions, stages privately and captures an immutable pin. | |
| Only explicit `release.py --publish --out SAME_DIR` releases an unchanged successful stage and | |
| verifies every anonymous download. It cannot overwrite a published repository. | |
| ## Licensing | |
| Independent fixture weights, authored synthetic text, and workflow: MIT ([LICENSE](LICENSE)). | |
| Upstream DeepSeek text and vision repository licenses are MIT and retained as | |
| `lineage-LICENSE` / `vision-lineage-LICENSE` with pinned configs. Native execution uses | |
| Transformers5.16.1 Apache-2.0 source as a dependency. The reviewed derived runtime | |
| `deepseek_v4_qfs.py` remains Apache-2.0 ([LICENSE-APACHE-2.0](LICENSE-APACHE-2.0)), not MIT; | |
| its prominent modification notice and installed source SHA256 values are retained. | |
| """ | |
| (model / "README.md").write_text("---\nlibrary_name: transformers\nlicense: mit\npipeline_tag: text-generation\ntags:\n- deepseek_v4\n- random-init\n- cpu\n- tiny\n---\n\n" + body) | |
| (evidence / "README.md").write_text("---\nlicense: mit\ntask_categories:\n- text-generation\ntags:\n- qfs\n- cpu\n- reproducibility\n- synthetic\n---\n\n" + body) | |
| def stage(args, api): | |
| work, qfs = args.out, args.qfs_root.resolve(strict=True) | |
| work.mkdir(parents=True, exist_ok=False) | |
| result = {"status": "running", "model_repository": MODEL, "evidence_repository": DATASET} | |
| write_json(work / "stage-result.json", result) | |
| try: | |
| for repository, kind in ((MODEL, "model"), (DATASET, "dataset")): | |
| if api.repo_exists(repository, repo_type=kind): | |
| raise ValueError(f"Refusing existing repository {repository}") | |
| command("build", "build_fixture.py", ["--out", work / "model"], work, qfs) | |
| command("regenerate", "build_fixture.py", ["--out", work / "regenerated"], work, qfs) | |
| if inventory(work / "model") != inventory(work / "regenerated"): | |
| raise ValueError("Independent regeneration bytes differ") | |
| from huggingface_hub import hf_hub_download | |
| lineage_path = Path(hf_hub_download(LINEAGE["repository"], "config.json", revision=LINEAGE["revision"], token=False, cache_dir=work / "lineage-cache")) | |
| lineage_config = json.loads(lineage_path.read_text()) | |
| lineage_license = Path(hf_hub_download(LINEAGE["repository"], "LICENSE", revision=LINEAGE["revision"], token=False, cache_dir=work / "lineage-cache")) | |
| vision_config = Path(hf_hub_download(VISION_LINEAGE["repository"], "config.json", revision=VISION_LINEAGE["revision"], token=False, cache_dir=work / "lineage-cache")) | |
| vision_license = Path(hf_hub_download(VISION_LINEAGE["repository"], "LICENSE", revision=VISION_LINEAGE["revision"], token=False, cache_dir=work / "lineage-cache")) | |
| if lineage_config.get("model_type") != "deepseek_v4": | |
| raise ValueError("Pinned lineage config is not native deepseek_v4") | |
| source = {"revision": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=qfs, text=True).strip(), "lineage": {**LINEAGE, "config_sha256": sha256(lineage_path)}, "vision_lineage": {**VISION_LINEAGE, "config_sha256": sha256(vision_config), "license_sha256": sha256(vision_license)}, "lineage_license_sha256": sha256(lineage_license), "note": "Revision identifies checkout HEAD; actual executed source file hashes are in evidence-summary and native verification."} | |
| for repository, kind in ((MODEL, "model"), (DATASET, "dataset")): | |
| api.create_repo(repository, repo_type=kind, private=True, exist_ok=False) | |
| pin = upload(api, MODEL, "model", work / "model") | |
| write_json(work / "model-pin.json", {"repository": MODEL, "revision": pin}) | |
| hub_proof = verify_remote(api, MODEL, "model", pin, work / "model", work / "private-download-cache", anonymous=False) | |
| write_json(work / "hub-checkpoint-verification.json", hub_proof) | |
| command("native", "verify_native.py", ["--model-dir", work / "model", "--code-repository", MODEL, "--code-revision", pin, "--qfs-root", qfs, "--out", work / "native-verification.json"], work, qfs) | |
| command("panel", "build_panel.py", ["--model-dir", work / "model", "--model-repository", MODEL, "--model-revision", pin, "--qfs-root", qfs, "--out", work / "panel"], work, qfs) | |
| command("cpu", "verify_cpu.py", ["--model-dir", work / "model", "--model-repository", MODEL, "--model-revision", pin, "--code-repository", MODEL, "--code-revision", pin, "--panel-dir", work / "panel", "--qfs-root", qfs, "--out", work / "reproduction"], work, qfs) | |
| measured = json.loads((work / "reproduction/evidence-summary.json").read_text()) | |
| if measured["status"] != "passed": | |
| raise ValueError("CPU reproduction did not pass") | |
| evidence = work / "evidence" | |
| shutil.copytree(work / "reproduction", evidence) | |
| shutil.copytree(work / "panel", evidence / "panel") | |
| for target in (work / "model", evidence): | |
| for name in SCRIPTS: | |
| if name == "build_fixture.py" and (target / name).exists(): | |
| continue | |
| shutil.copyfile(HERE / name, target / name) | |
| for name in ("native-verification.json", "hub-checkpoint-verification.json", "model-pin.json", "panel.binding.json", "panel.sha256.json"): | |
| shutil.copyfile(work / name, target / name) | |
| write_json(target / "qfs-source.json", source) | |
| write_json(target / "regeneration-verification.json", {"equal": True, "method": "Two fresh generator subprocesses; complete output file size/SHA256 inventories compared before adding publication metadata.", "files": inventory(work / "regenerated")}) | |
| shutil.copyfile(lineage_path, target / "lineage-config.json") | |
| shutil.copyfile(lineage_license, target / "lineage-LICENSE") | |
| shutil.copyfile(vision_config, target / "vision-lineage-config.json") | |
| shutil.copyfile(vision_license, target / "vision-lineage-LICENSE") | |
| for name in ("build", "regenerate", "native", "panel", "cpu"): | |
| for suffix in (".stdout.log", ".stderr.log", ".command.json"): | |
| shutil.copyfile(work / (name + suffix), evidence / (name + suffix)) | |
| shutil.copyfile(work / "model/build-manifest.json", evidence / "build-manifest.json") | |
| shutil.copyfile(work / "reproduction/evidence-summary.json", work / "model/evidence-summary.json") | |
| cards(work / "model", evidence, pin, source) | |
| model_revision = upload(api, MODEL, "model", work / "model", pin) | |
| evidence_revision = upload(api, DATASET, "dataset", evidence) | |
| verified = [verify_remote(api, repo, kind, revision, directory, work / "stage-download-cache", anonymous=False) for repo, kind, revision, directory in ((MODEL, "model", model_revision, work / "model"), (DATASET, "dataset", evidence_revision, evidence))] | |
| result.update({"status": "passed", "model_pin": pin, "model_revision": model_revision, "evidence_revision": evidence_revision, "model_files": inventory(work / "model"), "evidence_files": inventory(evidence), "private_download_verification": verified, "regeneration_equal": True, "native_report": "native-verification.json", "cpu_report": "reproduction/evidence-summary.json", "qfs_source": source}) | |
| except Exception as exc: | |
| result.update({"status": "failed", "error": f"{type(exc).__name__}: {exc}"}) | |
| (work / "stage-failure.log").write_text(traceback.format_exc()) | |
| raise | |
| finally: | |
| write_json(work / "stage-result.json", result) | |
| return result | |
| def publish(args, api): | |
| work = args.out | |
| if (work / "publication-result.json").exists(): | |
| raise ValueError("Refusing existing publication result") | |
| result = json.loads((work / "stage-result.json").read_text()) | |
| if result["status"] != "passed" or result["model_repository"] != MODEL or result["evidence_repository"] != DATASET: | |
| raise ValueError("No successful owned stage") | |
| entries = ((MODEL, "model", result["model_revision"], work / "model", result["model_files"]), (DATASET, "dataset", result["evidence_revision"], work / "evidence", result["evidence_files"])) | |
| for repository, kind, revision, directory, files in entries: | |
| info = api.repo_info(repository, repo_type=kind) | |
| if not info.private or info.sha != revision or inventory(directory) != files: | |
| raise ValueError(f"Staged local/remote content or privacy changed: {repository}") | |
| verify_remote(api, repository, kind, revision, directory, work / "prepublish-cache", anonymous=False) | |
| for repository, kind, *_ in entries: | |
| api.update_repo_settings(repository, repo_type=kind, private=False) | |
| from huggingface_hub import HfApi | |
| anonymous_api = HfApi(token=False) | |
| anonymous = [] | |
| for repository, kind, revision, directory, _ in entries: | |
| if anonymous_api.repo_info(repository, repo_type=kind, revision=revision, token=False).private: | |
| raise ValueError("Repository is not anonymously public") | |
| anonymous.append(verify_remote(anonymous_api, repository, kind, revision, directory, work / "anonymous-download-cache", anonymous=True)) | |
| published = {"status": "passed", "model_repository": MODEL, "model_revision": result["model_revision"], "model_pin": result["model_pin"], "evidence_repository": DATASET, "evidence_revision": result["evidence_revision"], "anonymous_download_verification": anonymous} | |
| write_json(work / "publication-result.json", published) | |
| return published | |
| def main(): | |
| parser = argparse.ArgumentParser(description=__doc__) | |
| mode = parser.add_mutually_exclusive_group() | |
| mode.add_argument("--stage-only", action="store_true") | |
| mode.add_argument("--publish", action="store_true") | |
| parser.add_argument("--out", type=Path, default=HERE / "stage") | |
| parser.add_argument("--qfs-root", type=Path, default=Path("/tmp/qfs-explorer-development")) | |
| args = parser.parse_args() | |
| args.out = args.out.absolute() | |
| from huggingface_hub import HfApi | |
| api = HfApi() | |
| result = publish(args, api) if args.publish else stage(args, api) | |
| print(json.dumps({"status": result["status"], "result": str(args.out / ("publication-result.json" if args.publish else "stage-result.json"))})) | |
| if __name__ == "__main__": | |
| main() | |