#!/usr/bin/env python3 """Stage privately by default; --publish only releases an unchanged successful stage.""" from __future__ import annotations import argparse import json import os from pathlib import Path import shutil import subprocess import sys import time import traceback from build_fixture import LINEAGE, VISION_LINEAGE, SCOPE, sha256, write_json HERE = Path(__file__).resolve().parent MODEL = "malaiwah/deepseek-v4-tiny-random-bf16" DATASET = "malaiwah/deepseek-v4-tiny-cpu-repro-v1" SCRIPTS = ("build_fixture.py", "build_panel.py", "verify_native.py", "verify_cpu.py", "release.py", "requirements-cpu.txt", "LICENSE", "support.json", "deepseek_v4_qfs.py", "LICENSE-APACHE-2.0") def inventory(directory): result = {} for path in sorted(directory.rglob("*")): if path.is_symlink(): raise ValueError(f"Refusing symlink {path}") if path.is_file(): relative = path.relative_to(directory) if any(part.startswith(".") or part == "__pycache__" for part in relative.parts): raise ValueError(f"Unexpected hidden/cache file {path}") result[relative.as_posix()] = {"bytes": path.stat().st_size, "sha256": sha256(path)} return result def command(name, script, arguments, work, qfs): env = os.environ.copy() env.update({k: "2" for k in ("OMP_NUM_THREADS", "MKL_NUM_THREADS", "OPENBLAS_NUM_THREADS", "NUMEXPR_NUM_THREADS", "VECLIB_MAXIMUM_THREADS")}) env.update({"HF_HUB_OFFLINE": "1", "TRANSFORMERS_OFFLINE": "1", "TOKENIZERS_PARALLELISM": "false", "PYTHONDONTWRITEBYTECODE": "1"}) if name == "native": env["HF_HUB_OFFLINE"] = "0" env["TRANSFORMERS_OFFLINE"] = "0" argv = [sys.executable, str(HERE / script), *map(str, arguments)] started = time.monotonic() with (work / (name + ".stdout.log")).open("xb") as out, (work / (name + ".stderr.log")).open("xb") as err: result = subprocess.run(argv, cwd=qfs, env=env, stdout=out, stderr=err, timeout=1200) write_json(work / (name + ".command.json"), {"argv": argv, "returncode": result.returncode, "elapsed_seconds": time.monotonic() - started}) if result.returncode: raise RuntimeError(f"{name} failed; inspect {work / (name + '.stderr.log')}") def verify_remote(api, repository, kind, revision, local, cache, *, anonymous): from huggingface_hub import hf_hub_download expected = inventory(local) token = False if anonymous else api.token remote_names = set(api.list_repo_files(repository, repo_type=kind, revision=revision, token=token)) if remote_names - set(expected) - {".gitattributes"} or set(expected) - remote_names: raise ValueError(f"Unexpected remote file closure for {repository}") for name, record in expected.items(): fetched = Path(hf_hub_download(repository, name, repo_type=kind, revision=revision, token=token, cache_dir=cache, force_download=True)) if fetched.stat().st_size != record["bytes"] or sha256(fetched) != record["sha256"]: raise ValueError(f"Hub bytes differ: {repository}/{name}") return {"repository": repository, "repo_type": kind, "revision": revision, "anonymous": anonymous, "files_verified": expected} def upload(api, repository, kind, directory, parent=None): return api.upload_folder(repo_id=repository, repo_type=kind, folder_path=directory, allow_patterns=list(inventory(directory)), commit_message="Complete native tiny CPU fixture and measured reproduction evidence", parent_commit=parent).oid def cards(model, evidence, pin, source): build = json.loads((model / "build-manifest.json").read_text()) native = json.loads((evidence / "native-verification.json").read_text()) measured = json.loads((evidence / "evidence-summary.json").read_text()) if native["status"] != "passed" or measured["status"] != "passed": raise ValueError("Cards require actual passed runtime evidence") comparison = measured["comparison"] support = json.loads((HERE / "support.json").read_text()) support["verified_runtime"] = { "status": "measured_passed", "native_class": native["model_class"], "versions": build["versions"], "model_pin": pin, "code_repository": MODEL, "code_revision": pin, "runtime_correction": build["runtime_correction"], "native_report_sha256": sha256(evidence / "native-verification.json"), "capture_report_sha256": sha256(evidence / "evidence-summary.json"), "measurement_scope": "Random BF16 text fixture using native primitives with explicit reviewed RMSNorm correction; not unmodified native arithmetic or published FP8/FP4, DSpark or vision.", "comparison": comparison, } for target in (model, evidence): write_json(target / "support.json", support) body = f"""# DeepSeek-V4 tiny corrected-native-primitives CPU text fixture Complete **randomly initialized**, untrained `QFSDeepseekV4ForCausalLM` text class using Transformers5.16.1 native primitives and a reviewed RMSNorm arithmetic correction. No upstream weights, paid GPU/cloud compute or useful-model claim. This is not unmodified native Transformers or the complete production release. ## Architecture and scope Text lineage: [{LINEAGE['repository']}@{LINEAGE['revision']}](https://huggingface.co/{LINEAGE['repository']}/tree/{LINEAGE['revision']}). Vision coverage research: [{VISION_LINEAGE['repository']}@{VISION_LINEAGE['revision']}](https://huggingface.co/{VISION_LINEAGE['repository']}/tree/{VISION_LINEAGE['revision']}). {SCOPE} The unmodified native RMSNorm multiplies a BF16 normalized activation by its required FP32 weight, promoting output to FP32 and causing the next BF16 linear projection to fail. The wrapper computes weighted normalization in FP32 and casts the complete result to the input activation dtype. Only this model instance's RMSNorm modules are replaced; no global/installed-library patches, autocast, forward-hook corrections or weight changes. This moves an arithmetic rounding boundary and is explicitly not native bit-equivalence. `deepseek_v4_qfs.py` and its Apache-2.0 license are retained in the immutable model pin. Observed replacements and source hashes: `{json.dumps(build['runtime_correction'])}`. Four layers use explicit native legacy schedule selectors `compress_ratios=[0,0,4,128]`, normalized to two sliding-only, one CSA, one HCA layer. Actual rates are CSA4/HCA16. All layers retain a local window8, shared K=V multi-query attention, learned sinks, low-rank query projection and grouped low-rank output (Q4/KV1, head16, Q-rank16, two output groups of rank8). Hidden64, expert intermediate32, eight routed experts, top2 plus one shared expert; three seeded frozen hash routers then one learned router use sqrt-softplus scores. mHC4 and20 Sinkhorn iterations are preserved. Native gate/up clipping limit10 is preserved. Partial interleaved RoPE uses dimension4, with separate main/compressor bases. No production YaRN/long-context scaling claim. The small real tokenizer is256 byte-level BPE tokens without merges plus16 special/reserved entries (272 total; IDs0/1/2/3 pad/BOS/EOS/unknown). The untied head is complete. The upstream flat `vision_*` fields describe a32-layer vision encoder plus aligner, not instantiated by this native text class. Upstream `dspark_*` fields and MTP are not executed. A declared `DeepseekV4ForCausalLM` architecture in Vision-Exp metadata does not establish native multimodal coverage. The upstream configs/licenses are retained. ## Actual measured evidence - Parameters: **{build['parameter_count']:,}**; semantic tensors: **{build['tensor_count']}**; seed: **{build['seed']}**. - Two independent generator subprocesses matched complete byte inventories. - Native FP32 initialization is converted by the actual BF16 loader; mandatory native FP32 retention is preserved before rounding. Exact values and dtypes pass corrected-class final reload and strict pinned QFS loading. Retained tensors: `{json.dumps(build['native_fp32_exceptions'])}`. - Native cache probes exercise CSA boundaries4, sliding eviction8, HCA16, and63/64; single- and multi-token continuation runs have exact same-schedule repeats. Cached-vs-cold differences are recorded, not asserted zero. Real cache remainder, entry count and CSA overlap states are checked. Actual hooks observe mHC row/column sums, hash/learned routes, grouped output, and CSA future-entry exclusion. Suffix perturbation checks causal prefix independence; beyond-limit activation inputs exercise the real clipped expert function. - Two fresh CPU processes, two threads, BF16, honest QFS lane `other`, sealed four64-token synthetic windows,252 shifted prediction positions; no quality benchmark/registry submission. - Strict tensor verification and forced numerical replay: metric **{comparison['metric']['value']}**, KL `{json.dumps(comparison['kl'], sort_keys=True)}`, top1 agreement **{comparison['top1_agreement']}**. The actual tokenwise artifact is252 positive float64 zeros, not just a summary claim. - Captured immutable private-stage checkpoint: `{MODEL}@{pin}`. QFS source revision `{source['revision']}`; executed file hashes accompany the evidence. Code repository is the same model repository at the same immutable pin, explicitly selected with `--trust-remote-code --code-repository --code-revision`. Capture weights/tokenizer remain local; HF network is permitted for pinned code verification. Captures are not entirely offline. See [native-verification.json](native-verification.json), [evidence-summary.json](evidence-summary.json), [build-manifest.json](build-manifest.json), [support.json](support.json), and the [evidence repository](https://huggingface.co/datasets/{DATASET}). The evidence repository root is an artifact bundle, **not a single root-format QFS dataset**; `first/` and `repeat/` are sealed roots. QFS retains its hash-proof reproduction-confirmation semantics while `--force-compute` also runs numerical replay. This proves same-stack/machine repeatability only. ## Reproduction and capture Python3.12, isolated CPU environment: `pip install -r requirements-cpu.txt`. With the recorded QFS checkout at `$QFS`, outputs must be new: ```bash python build_fixture.py --out /tmp/deepseek-v4-rebuilt python verify_native.py --model-dir /tmp/deepseek-v4-rebuilt --code-repository {MODEL} --code-revision {pin} --qfs-root "$QFS" --out /tmp/deepseek-v4-native.json python build_panel.py --model-dir /tmp/deepseek-v4-rebuilt --model-repository {MODEL} --model-revision {pin} --qfs-root "$QFS" --out /tmp/deepseek-v4-panel python verify_cpu.py --model-dir /tmp/deepseek-v4-rebuilt --model-repository {MODEL} --model-revision {pin} --code-repository {MODEL} --code-revision {pin} --panel-dir /tmp/deepseek-v4-panel --qfs-root "$QFS" --out /tmp/deepseek-v4-repro ``` `support.json` gives the real QFS capture argv for third-party authors with an explicit `--author`, including the corrected native-primitives BF16 text checkpoint path. Original releases declare dynamic FP8 e4m3/ue8m0 block128 storage plus `expert_dtype=fp4`. This BF16 fixture does **not** prove those kernels/conversions or the complete production model. `--drop-parallel-plan` addresses the HF quantizer parallel-plan rewrite bug, not unsupported packed expert formats. Never remove `quantization_config` from real upstream weights to make them load. Do not bypass missing/unexpected/mismatched/conversion-error guards. Full original-format qualification needs actual supported hardware/runtime and clean full-checkpoint loading; vision and DSpark additionally need their real implementations and modality/speculation probes. For an unchanged original config without `auto_map`, explicit `--code-repository {MODEL}` and `--code-revision {pin}` use only that pinned repository's verified `auto_map` as runtime class mapping. Wrapper config construction still uses all original checkpoint fields; no dimensions, quantization metadata or weights are replaced. This makes the runtime override explicit; it does not qualify untested original packed formats or ignored vision/DSpark modules. Publisher workflow: `release.py --stage-only --out NEW_DIR --qfs-root "$QFS"` requires authorization for the two fixed repository IDs, refuses collisions, stages privately and captures an immutable pin. Only explicit `release.py --publish --out SAME_DIR` releases an unchanged successful stage and verifies every anonymous download. It cannot overwrite a published repository. ## Licensing Independent fixture weights, authored synthetic text, and workflow: MIT ([LICENSE](LICENSE)). Upstream DeepSeek text and vision repository licenses are MIT and retained as `lineage-LICENSE` / `vision-lineage-LICENSE` with pinned configs. Native execution uses Transformers5.16.1 Apache-2.0 source as a dependency. The reviewed derived runtime `deepseek_v4_qfs.py` remains Apache-2.0 ([LICENSE-APACHE-2.0](LICENSE-APACHE-2.0)), not MIT; its prominent modification notice and installed source SHA256 values are retained. """ (model / "README.md").write_text("---\nlibrary_name: transformers\nlicense: mit\npipeline_tag: text-generation\ntags:\n- deepseek_v4\n- random-init\n- cpu\n- tiny\n---\n\n" + body) (evidence / "README.md").write_text("---\nlicense: mit\ntask_categories:\n- text-generation\ntags:\n- qfs\n- cpu\n- reproducibility\n- synthetic\n---\n\n" + body) def stage(args, api): work, qfs = args.out, args.qfs_root.resolve(strict=True) work.mkdir(parents=True, exist_ok=False) result = {"status": "running", "model_repository": MODEL, "evidence_repository": DATASET} write_json(work / "stage-result.json", result) try: for repository, kind in ((MODEL, "model"), (DATASET, "dataset")): if api.repo_exists(repository, repo_type=kind): raise ValueError(f"Refusing existing repository {repository}") command("build", "build_fixture.py", ["--out", work / "model"], work, qfs) command("regenerate", "build_fixture.py", ["--out", work / "regenerated"], work, qfs) if inventory(work / "model") != inventory(work / "regenerated"): raise ValueError("Independent regeneration bytes differ") from huggingface_hub import hf_hub_download lineage_path = Path(hf_hub_download(LINEAGE["repository"], "config.json", revision=LINEAGE["revision"], token=False, cache_dir=work / "lineage-cache")) lineage_config = json.loads(lineage_path.read_text()) lineage_license = Path(hf_hub_download(LINEAGE["repository"], "LICENSE", revision=LINEAGE["revision"], token=False, cache_dir=work / "lineage-cache")) vision_config = Path(hf_hub_download(VISION_LINEAGE["repository"], "config.json", revision=VISION_LINEAGE["revision"], token=False, cache_dir=work / "lineage-cache")) vision_license = Path(hf_hub_download(VISION_LINEAGE["repository"], "LICENSE", revision=VISION_LINEAGE["revision"], token=False, cache_dir=work / "lineage-cache")) if lineage_config.get("model_type") != "deepseek_v4": raise ValueError("Pinned lineage config is not native deepseek_v4") source = {"revision": subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=qfs, text=True).strip(), "lineage": {**LINEAGE, "config_sha256": sha256(lineage_path)}, "vision_lineage": {**VISION_LINEAGE, "config_sha256": sha256(vision_config), "license_sha256": sha256(vision_license)}, "lineage_license_sha256": sha256(lineage_license), "note": "Revision identifies checkout HEAD; actual executed source file hashes are in evidence-summary and native verification."} for repository, kind in ((MODEL, "model"), (DATASET, "dataset")): api.create_repo(repository, repo_type=kind, private=True, exist_ok=False) pin = upload(api, MODEL, "model", work / "model") write_json(work / "model-pin.json", {"repository": MODEL, "revision": pin}) hub_proof = verify_remote(api, MODEL, "model", pin, work / "model", work / "private-download-cache", anonymous=False) write_json(work / "hub-checkpoint-verification.json", hub_proof) command("native", "verify_native.py", ["--model-dir", work / "model", "--code-repository", MODEL, "--code-revision", pin, "--qfs-root", qfs, "--out", work / "native-verification.json"], work, qfs) command("panel", "build_panel.py", ["--model-dir", work / "model", "--model-repository", MODEL, "--model-revision", pin, "--qfs-root", qfs, "--out", work / "panel"], work, qfs) command("cpu", "verify_cpu.py", ["--model-dir", work / "model", "--model-repository", MODEL, "--model-revision", pin, "--code-repository", MODEL, "--code-revision", pin, "--panel-dir", work / "panel", "--qfs-root", qfs, "--out", work / "reproduction"], work, qfs) measured = json.loads((work / "reproduction/evidence-summary.json").read_text()) if measured["status"] != "passed": raise ValueError("CPU reproduction did not pass") evidence = work / "evidence" shutil.copytree(work / "reproduction", evidence) shutil.copytree(work / "panel", evidence / "panel") for target in (work / "model", evidence): for name in SCRIPTS: if name == "build_fixture.py" and (target / name).exists(): continue shutil.copyfile(HERE / name, target / name) for name in ("native-verification.json", "hub-checkpoint-verification.json", "model-pin.json", "panel.binding.json", "panel.sha256.json"): shutil.copyfile(work / name, target / name) write_json(target / "qfs-source.json", source) write_json(target / "regeneration-verification.json", {"equal": True, "method": "Two fresh generator subprocesses; complete output file size/SHA256 inventories compared before adding publication metadata.", "files": inventory(work / "regenerated")}) shutil.copyfile(lineage_path, target / "lineage-config.json") shutil.copyfile(lineage_license, target / "lineage-LICENSE") shutil.copyfile(vision_config, target / "vision-lineage-config.json") shutil.copyfile(vision_license, target / "vision-lineage-LICENSE") for name in ("build", "regenerate", "native", "panel", "cpu"): for suffix in (".stdout.log", ".stderr.log", ".command.json"): shutil.copyfile(work / (name + suffix), evidence / (name + suffix)) shutil.copyfile(work / "model/build-manifest.json", evidence / "build-manifest.json") shutil.copyfile(work / "reproduction/evidence-summary.json", work / "model/evidence-summary.json") cards(work / "model", evidence, pin, source) model_revision = upload(api, MODEL, "model", work / "model", pin) evidence_revision = upload(api, DATASET, "dataset", evidence) verified = [verify_remote(api, repo, kind, revision, directory, work / "stage-download-cache", anonymous=False) for repo, kind, revision, directory in ((MODEL, "model", model_revision, work / "model"), (DATASET, "dataset", evidence_revision, evidence))] result.update({"status": "passed", "model_pin": pin, "model_revision": model_revision, "evidence_revision": evidence_revision, "model_files": inventory(work / "model"), "evidence_files": inventory(evidence), "private_download_verification": verified, "regeneration_equal": True, "native_report": "native-verification.json", "cpu_report": "reproduction/evidence-summary.json", "qfs_source": source}) except Exception as exc: result.update({"status": "failed", "error": f"{type(exc).__name__}: {exc}"}) (work / "stage-failure.log").write_text(traceback.format_exc()) raise finally: write_json(work / "stage-result.json", result) return result def publish(args, api): work = args.out if (work / "publication-result.json").exists(): raise ValueError("Refusing existing publication result") result = json.loads((work / "stage-result.json").read_text()) if result["status"] != "passed" or result["model_repository"] != MODEL or result["evidence_repository"] != DATASET: raise ValueError("No successful owned stage") entries = ((MODEL, "model", result["model_revision"], work / "model", result["model_files"]), (DATASET, "dataset", result["evidence_revision"], work / "evidence", result["evidence_files"])) for repository, kind, revision, directory, files in entries: info = api.repo_info(repository, repo_type=kind) if not info.private or info.sha != revision or inventory(directory) != files: raise ValueError(f"Staged local/remote content or privacy changed: {repository}") verify_remote(api, repository, kind, revision, directory, work / "prepublish-cache", anonymous=False) for repository, kind, *_ in entries: api.update_repo_settings(repository, repo_type=kind, private=False) from huggingface_hub import HfApi anonymous_api = HfApi(token=False) anonymous = [] for repository, kind, revision, directory, _ in entries: if anonymous_api.repo_info(repository, repo_type=kind, revision=revision, token=False).private: raise ValueError("Repository is not anonymously public") anonymous.append(verify_remote(anonymous_api, repository, kind, revision, directory, work / "anonymous-download-cache", anonymous=True)) published = {"status": "passed", "model_repository": MODEL, "model_revision": result["model_revision"], "model_pin": result["model_pin"], "evidence_repository": DATASET, "evidence_revision": result["evidence_revision"], "anonymous_download_verification": anonymous} write_json(work / "publication-result.json", published) return published def main(): parser = argparse.ArgumentParser(description=__doc__) mode = parser.add_mutually_exclusive_group() mode.add_argument("--stage-only", action="store_true") mode.add_argument("--publish", action="store_true") parser.add_argument("--out", type=Path, default=HERE / "stage") parser.add_argument("--qfs-root", type=Path, default=Path("/tmp/qfs-explorer-development")) args = parser.parse_args() args.out = args.out.absolute() from huggingface_hub import HfApi api = HfApi() result = publish(args, api) if args.publish else stage(args, api) print(json.dumps({"status": result["status"], "result": str(args.out / ("publication-result.json" if args.publish else "stage-result.json"))})) if __name__ == "__main__": main()