Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
73 changes: 73 additions & 0 deletions c/tests/test_make_qwen38_tiny_fp8.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,73 @@
"""Regression test for replacing the source shard on Windows."""

import importlib.util
import json
import tempfile
import unittest
from pathlib import Path


try:
import torch
from safetensors.torch import load_file, save_file
except ImportError:
torch = None
load_file = save_file = None


TOOL = Path(__file__).resolve().parents[1] / "tools" / "make_qwen38_tiny.py"
REPO = TOOL.parents[2]


@unittest.skipIf(torch is None, "requires torch and safetensors")
class RewriteFp8ShardWindowsTest(unittest.TestCase):
def test_rewrite_releases_source_shard_before_unlink(self):
# On Windows this test fails with PermissionError if load_file's mapped
# tensors are still alive when _rewrite_shard_fp8 unlinks the source.
spec = importlib.util.spec_from_file_location("make_qwen38_tiny_test", TOOL)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)

with tempfile.TemporaryDirectory(dir=REPO) as tmp:
out = Path(tmp)
source = out / "model.safetensors"
gate = "model.layers.0.mlp.experts.0.gate_proj.weight"
down = "model.layers.0.mlp.experts.0.down_proj.weight"
dense = "model.embed_tokens.weight"
original = {
gate: torch.tensor([[1.0, 2.0]], dtype=torch.bfloat16),
down: torch.tensor([[3.0, 4.0]], dtype=torch.bfloat16),
dense: torch.tensor([[5.0, 6.0]], dtype=torch.bfloat16),
}
save_file(original, str(source), metadata={"format": "pt"})
(out / "config.json").write_text("{}\n", encoding="utf-8")
packed = {
gate: (torch.tensor([[1.0, 2.0]], dtype=torch.float8_e4m3fn),
torch.tensor([[0.25]], dtype=torch.bfloat16)),
down: (torch.tensor([[3.0, 4.0]], dtype=torch.float8_e4m3fn),
torch.tensor([[0.5]], dtype=torch.bfloat16)),
}

module._rewrite_shard_fp8(out, packed)

self.assertFalse(source.exists())
index = json.loads((out / "model.safetensors.index.json").read_text(encoding="utf-8"))
self.assertEqual(index["weight_map"][gate], "model-00001-of-00002.safetensors")
self.assertEqual(index["weight_map"][down], "model-00002-of-00002.safetensors")
self.assertEqual(index["weight_map"][dense], "model-00001-of-00002.safetensors")
rest = load_file(str(out / "model-00001-of-00002.safetensors"))
downs = load_file(str(out / "model-00002-of-00002.safetensors"))
self.assertEqual(rest[gate].dtype, torch.float8_e4m3fn)
self.assertTrue(torch.equal(rest[gate], packed[gate][0]))
self.assertEqual(rest[gate + "_scale_inv"].dtype, torch.bfloat16)
self.assertTrue(torch.equal(rest[gate + "_scale_inv"], packed[gate][1]))
self.assertTrue(torch.equal(rest[dense], original[dense]))
self.assertEqual(downs[down].dtype, torch.float8_e4m3fn)
self.assertTrue(torch.equal(downs[down], packed[down][0]))
self.assertEqual(downs[down + "_scale_inv"].dtype, torch.bfloat16)
self.assertTrue(torch.equal(downs[down + "_scale_inv"], packed[down][1]))
del rest, downs


if __name__ == "__main__":
unittest.main()
8 changes: 7 additions & 1 deletion c/tools/make_qwen38_tiny.py
Original file line number Diff line number Diff line change
Expand Up @@ -189,7 +189,13 @@ def _rewrite_shard_fp8(out: Path, packed):
other -- the adjacency the engine's native FP8 path requires."""
from safetensors.torch import load_file, save_file
path = out / "model.safetensors"
tensors = load_file(str(path))
source_tensors = load_file(str(path))
# load_file returns tensors backed by a mapping of the source shard. On
# Windows that mapping keeps the file open, so path.unlink() below fails.
# Copy the small fixture tensors into independent storage, then release the
# mapped tensors before replacing their source file.
tensors = {name: tensor.clone() for name, tensor in source_tensors.items()}
del source_tensors
for name, (q, scale_inv) in packed.items():
assert name in tensors, name
tensors[name] = q.contiguous()
Expand Down