Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -557,6 +557,9 @@ jobs:
# time. XDNA=1 needs nothing installed on the runner: the lane's host
# side links no XRT, and the helper is loaded at runtime on request.
make_args: "XDNA=1"
# release.yml's DeepSeek V4 (clang with LLVM's libomp, #2286, and VK=1),
# built after the default gcc one
v4_args: "CC=clang VK=1"
runs-on: ${{ matrix.os }}
defaults:
run:
Expand All @@ -574,6 +577,15 @@ jobs:
make
mingw-w64-ucrt-x86_64-gcc
mingw-w64-ucrt-x86_64-binutils
mingw-w64-ucrt-x86_64-clang
mingw-w64-ucrt-x86_64-vulkan-headers
mingw-w64-ucrt-x86_64-shaderc
# the OpenMP package under either of its names, as in release.yml
- if: matrix.os == 'windows-latest'
run: |
omp=mingw-w64-ucrt-x86_64-llvm-openmp
pacman -Si $omp > /dev/null 2>&1 || omp=mingw-w64-ucrt-x86_64-openmp
pacman -S --needed --noconfirm $omp
- if: matrix.os == 'macos-latest'
run: brew install libomp
# Keep the portable sibling list aligned with release packaging. DeepSeek
Expand All @@ -592,6 +604,12 @@ jobs:
make $t ${{ matrix.make_args }} || { echo "FAILED: $t"; rc=1; }
echo "::endgroup::"
done
# the release's DeepSeek V4 as well (the flags stamp rebuilds its units)
if [ -n "${{ matrix.v4_args }}" ]; then
echo "::group::deepseek-v4 ${{ matrix.v4_args }}"
make deepseek-v4 ${{ matrix.make_args }} ${{ matrix.v4_args }} || { echo "FAILED: deepseek-v4 ${{ matrix.v4_args }}"; rc=1; }
echo "::endgroup::"
fi
exit $rc
- if: matrix.os == 'windows-latest'
name: "The release configuration is what got built (XDNA=1)"
Expand Down
23 changes: 22 additions & 1 deletion .github/workflows/release.yml
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,9 @@ jobs:
make_args: "XDNA=1 VK=1"
vk: "1"
v4: "1"
# DeepSeek V4 with clang and LLVM's libomp, linked in: MinGW's libgomp
# never spins at a barrier, whatever the launcher's wait policy (#2286)
v4_args: "CC=clang"
# GitHub shells are format strings: 'msys2 {0}', never bare 'msys2'
# (the v1.0.0 tag build failed on exactly this). Same as ci.yml.
shell: "msys2 {0}"
Expand All @@ -84,6 +87,16 @@ jobs:
mingw-w64-ucrt-x86_64-gcc
mingw-w64-ucrt-x86_64-vulkan-headers
mingw-w64-ucrt-x86_64-shaderc
mingw-w64-ucrt-x86_64-clang

# LLVM's OpenMP runtime for the clang build of DeepSeek V4. MSYS2 renamed the
# package openmp -> llvm-openmp; the runner's package list (update: false) may
# predate that, and a name pacman does not know installs nothing (6b091239).
- if: matrix.os == 'windows-latest'
run: |
omp=mingw-w64-ucrt-x86_64-llvm-openmp
pacman -Si $omp > /dev/null 2>&1 || omp=mingw-w64-ucrt-x86_64-openmp
pacman -S --needed --noconfirm $omp

- if: matrix.os == 'macos-latest'
run: brew install libomp
Expand Down Expand Up @@ -146,7 +159,7 @@ jobs:
if: matrix.v4 == '1'
run: |
cd c
make deepseek-v4 ${{ matrix.make_args }}
make deepseek-v4 ${{ matrix.make_args }} ${{ matrix.v4_args }}
ls -lh deepseek_v4${{ matrix.ext }}

- uses: actions/setup-node@v4
Expand Down Expand Up @@ -274,6 +287,14 @@ jobs:
for e in $SIBLINGS; do
test -x $e${{ matrix.ext }} || { echo "FAIL: $e missing from archive"; ls -la; exit 1; }
done
# Windows: deepseek_v4.exe carries LLVM's libomp (#2286), linked in, so it
# must import no MinGW runtime DLL: the archive ships none.
if [ "${{ matrix.v4 }}" = "1" ] && [ "${{ matrix.ext }}" = ".exe" ]; then
grep -aq KMP_BLOCKTIME deepseek_v4.exe || { echo "FAIL: deepseek_v4.exe is not a libomp build"; exit 1; }
if objdump -p deepseek_v4.exe | grep -i 'DLL Name: lib'; then
echo "FAIL: deepseek_v4.exe imports a DLL the archive does not carry"; exit 1
fi
fi
# the TUI: present, and it runs (--version needs no terminal and no server)
test -x colibri-tui${{ matrix.ext }} || { echo "FAIL: colibri-tui missing from archive"; ls -la; exit 1; }
./colibri-tui${{ matrix.ext }} --version | grep -q "^colibri-tui " \
Expand Down
10 changes: 10 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,16 @@ Format follows [Keep a Changelog](https://keepachangelog.com/).

### Performance

- **Windows: DeepSeek V4 built with clang and LLVM's libomp** (#2286). MinGW's libgomp, which the
gcc build links, has no spin-wait: its barriers sleep on a semaphore whatever
`OMP_WAIT_POLICY` and `GOMP_SPINCOUNT` say, and every one of a token's small parallel regions
paid a thread wake-up (22 us a barrier with 4 threads on an i7-7700HQ, 0.3 us with libomp).
The release builds `deepseek_v4.exe` with `CC=clang`, libomp linked in (no new DLL), and on
Windows the launcher adds `KMP_BLOCKTIME=200` when the wait policy is active, so the team
sleeps between requests instead of spinning. Same tokens on the tiny fixture's oracle, which
decodes 3.1x faster on that laptop; the issue measured 1.5x on the full model with an LTO
clang build. With the CUDA tier the Windows team now really spins (not measured).
`c/Makefile` now recognises clang's `x86_64-w64-windows-gnu` target.
- **CPU without FMA: FP8 and int4-g64 experts read once for a prompt's rows** (#2715). On a
CPU without AVX2 or FMA (Sandy and Ivy Bridge, a build for the x86-64 baseline or v2),
quant.h's FP8 kernel decoded every weight again for each row and waited on one chain of adds:
Expand Down
13 changes: 9 additions & 4 deletions c/Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,8 @@
DETECT_CC := $(if $(filter default,$(origin CC)),gcc,$(CC))
TRIPLET := $(shell $(DETECT_CC) -dumpmachine 2>/dev/null)

MINGW := $(findstring mingw,$(TRIPLET))
# clang names the same MinGW target x86_64-w64-windows-gnu (MSYS2's clang, llvm-mingw)
MINGW := $(findstring mingw,$(TRIPLET))$(findstring windows-gnu,$(TRIPLET))
CYGWIN := $(findstring cygwin,$(TRIPLET))
DARWIN := $(findstring darwin,$(TRIPLET))
LINUX := $(findstring linux,$(TRIPLET))
Expand Down Expand Up @@ -148,11 +149,15 @@ LDFLAGS = -lm -fopenmp -static -lpsapi $(EXTRA_LDFLAGS)
# "cannot find -lgomp" (#1900, #2612). Ask the compiler where the archive is: when
# it is missing, -print-file-name just echoes "libgomp.a" back. Warn up front and
# name the package; the build still goes on, so goals that never link OpenMP are
# not blocked. LIBGOMP_CHECK=0 silences it.
# not blocked. LIBGOMP_CHECK=0 silences it. clang's -fopenmp links LLVM's libomp.a
# instead, from its own package (#2286).
ifneq ($(LIBGOMP_CHECK),0)
ifneq ($(filter-out clean%,$(or $(MAKECMDGOALS),all)),)
ifeq ($(shell $(DETECT_CC) -print-file-name=libgomp.a 2>/dev/null),libgomp.a)
$(warning $(DETECT_CC) has no libgomp.a, so the -static -fopenmp link will fail with "cannot find -lgomp". In the "MSYS2 UCRT64" shell run: pacman -S --needed mingw-w64-ucrt-x86_64-libgomp (docs/windows.md))
OMP_LIB := $(if $(findstring clang,$(notdir $(DETECT_CC))),omp,gomp)
OMP_PKG := $(if $(findstring clang,$(notdir $(DETECT_CC))),llvm-openmp,libgomp)
OMP_DOC := $(if $(findstring clang,$(notdir $(DETECT_CC))),docs/deepseek-v4.md,docs/windows.md)
ifeq ($(shell $(DETECT_CC) -print-file-name=lib$(OMP_LIB).a 2>/dev/null),lib$(OMP_LIB).a)
$(warning $(DETECT_CC) has no lib$(OMP_LIB).a, so the -static -fopenmp link will fail with "cannot find -l$(OMP_LIB)". In the "MSYS2 UCRT64" shell run: pacman -S --needed mingw-w64-ucrt-x86_64-$(OMP_PKG) ($(OMP_DOC)))
endif
endif
endif
Expand Down
24 changes: 24 additions & 0 deletions c/Makefile.deepseek-v4
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,30 @@ CFLAGS = -D_GNU_SOURCE -D_FILE_OFFSET_BITS=64 -O3 -march=$(ARCH) -fopenmp \
-DCOLI_V4_GPU_TIER \
-DCOLI_V4_EXPERIMENTAL_DUAL_EXPERT_LOADER
LDFLAGS = -lm -fopenmp -static -pthread
# CC=clang (MSYS2 UCRT64: mingw-w64-ucrt-x86_64-clang and -llvm-openmp) links LLVM's
# libomp, statically from its libomp.a. MinGW's libgomp has no spin-wait: its barriers
# sleep on a semaphore whatever OMP_WAIT_POLICY and GOMP_SPINCOUNT say, and a decode
# step is hundreds of small parallel regions (#2286). clang's driver links winpthreads
# by itself (-pthread only draws an unused-argument warning), and LTO is off unless
# asked for: GNU ld needs LLVM's linker plugin for clang's LTO objects, which the
# clang package does not install.
ifneq ($(findstring clang,$(notdir $(CC))),)
LDFLAGS = -lm -fopenmp -static
LTO ?= 0
# libomp.a defines omp_get_num_devices() only as a COFF weak symbol, and GNU ld leaves
# it unresolved with MSYS2's llvm-openmp 22.1.7 or binutils 2.46 (GitHub's runner has
# both): "undefined reference to `omp_get_num_devices'" from libomp's own kmp_alloc.
# Link one call to it the same way; only when that fails on this symbol does
# backend_loader_dsv4.c define it, with what libomp returns when no offload runtime is
# linked: 0 devices. Where it links (llvm-openmp 23.x with binutils 2.47), nothing is added.
V4_OMP_PROBE := $(shell case `$(CC) -dumpmachine 2>/dev/null` in (*mingw*|*windows-gnu*) \
echo 'int omp_get_num_devices(void); int main(void) { return omp_get_num_devices(); }' | \
$(CC) -x c - -fopenmp -static -o deepseek_v4_omp_probe.exe 2>&1; \
rm -f deepseek_v4_omp_probe.exe;; esac)
ifneq ($(findstring omp_get_num_devices,$(V4_OMP_PROBE)),)
CFLAGS += -DCOLI_OMP_NUM_DEVICES_STUB
endif
endif
V4_BINARY := deepseek_v4.exe
V4_WIN_EXTRA_OBJS = backend_loader_dsv4.o
else ifeq ($(UNAME_S),Darwin)
Expand Down
8 changes: 8 additions & 0 deletions c/backend_loader_dsv4.c
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,14 @@

#include "backend_cuda_dsv4.h"

/* Only for a clang build whose link leaves libomp.a's weak omp_get_num_devices
* unresolved (MSYS2's llvm-openmp 22.1.7 or binutils 2.46, #2286; Makefile.deepseek-v4
* probes for it). No offload runtime is linked, so 0 non-host devices: what libomp
* itself returns without one. */
#ifdef COLI_OMP_NUM_DEVICES_STUB
int omp_get_num_devices(void) { return 0; }
#endif

/* Two builds of the same source ship side by side: coli_cuda_dsv4_dg.dll
* (DeepGEMM, sm_120a tensor-core paths) and coli_cuda_dsv4.dll (portable
* generic kernels, sm_80+). The loader tries the DeepGEMM one first and asks
Expand Down
10 changes: 10 additions & 0 deletions c/coli
Original file line number Diff line number Diff line change
Expand Up @@ -583,6 +583,12 @@ def env_for_engine(a, arch, plan=None):
else:
env.setdefault("OMP_WAIT_POLICY", "active")
env.setdefault("GOMP_SPINCOUNT", "200000")
# LLVM's libomp (the Windows clang build, #2286) reads this, not GOMP_SPINCOUNT:
# with OMP_WAIT_POLICY=active alone its idle team never sleeps (#341). Only for
# active: under passive it would turn libomp's blocktime 0 into 200 ms of spinning.
# Windows only: Linux builds link libgomp, and macOS's libomp is left as it was (#707).
if sys.platform == "win32" and env["OMP_WAIT_POLICY"].strip().lower() == "active":
env.setdefault("KMP_BLOCKTIME", "200")
env.setdefault("OMP_DYNAMIC", "FALSE")
if sys.platform != "win32":
env.setdefault("OMP_PROC_BIND", "close")
Expand Down Expand Up @@ -1039,6 +1045,10 @@ def env_for(a):
("GOMP_SPINCOUNT", "200000"),
("OMP_DYNAMIC", "FALSE")):
e.setdefault(k, v)
# a clang build links LLVM's libomp, whose active team never sleeps without it
# (#341, #2286); not under passive, where it would mean 200 ms of spinning
if e["OMP_WAIT_POLICY"].strip().lower() == "active":
e.setdefault("KMP_BLOCKTIME", "200")
# Default Windows misurati sul box di riferimento (docs/tuning-9950x3d-5090.md),
# tutti lossless e tutti setdefault (un override esplicito vince sempre):
# - DIRECT=1: 10.7 GB/s O_DIRECT vs 9.0 buffered (iobench, anche a cache
Expand Down
4 changes: 0 additions & 4 deletions c/deepseek_v4.c
Original file line number Diff line number Diff line change
Expand Up @@ -13804,7 +13804,6 @@ static Dsv4CudaExpertSet *v4_moe_bank;
static int v4_moe_bank_device = -1;
static Dsv4CudaActivation *v4_moe_in, *v4_moe_out;
static int v4_moe_bank_layer = -1;
static int v4_moe_bank_hash_layer = -1;
static unsigned char v4_moe_bank_valid[256];
static int v4_moe_bank_failed;

Expand Down Expand Up @@ -13999,7 +13998,6 @@ void coli_v4_gpu_moe_batch_release(void) {
dsv4_cuda_expert_set_free(v4_moe_bank);
v4_moe_bank = NULL;
v4_moe_bank_layer = -1;
v4_moe_bank_hash_layer = -1;
memset(v4_moe_bank_valid, 0, sizeof(v4_moe_bank_valid));
}

Expand All @@ -14013,7 +14011,6 @@ int coli_v4_gpu_moe_batch_union(float *outputs,
int batch) {
#define bank v4_moe_bank
#define bank_layer v4_moe_bank_layer
#define hash_layer v4_moe_bank_hash_layer
#define bank_valid v4_moe_bank_valid
#define in_mirror v4_moe_in
#define out_mirror v4_moe_out
Expand Down Expand Up @@ -14512,7 +14509,6 @@ int coli_v4_gpu_moe_batch_union(float *outputs,
#undef out_mirror
#undef bank
#undef bank_layer
#undef hash_layer
#undef bank_valid
#undef bank_failed
}
Expand Down
6 changes: 6 additions & 0 deletions c/openai_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -6786,6 +6786,12 @@ def tune_child_env(env, arch):
# that runtime policy treat the launcher value as a user override.
env.setdefault("OMP_WAIT_POLICY", "active")
env.setdefault("GOMP_SPINCOUNT", "200000")
# LLVM's libomp (the Windows clang build, #2286) reads this, not GOMP_SPINCOUNT:
# with OMP_WAIT_POLICY=active alone its idle team never sleeps (#341). Only for
# active: under passive it would turn libomp's blocktime 0 into 200 ms of spinning.
# Windows only: Linux builds link libgomp, and macOS's libomp is left as it was (#707).
if sys.platform == "win32" and env["OMP_WAIT_POLICY"].strip().lower() == "active":
env.setdefault("KMP_BLOCKTIME", "200")
env.setdefault("OMP_DYNAMIC", "FALSE")
if sys.platform != "win32":
env.setdefault("OMP_PROC_BIND", "close")
Expand Down
22 changes: 21 additions & 1 deletion c/tests/test_cli_output.py
Original file line number Diff line number Diff line change
Expand Up @@ -415,6 +415,26 @@ def test_v4_gets_memory_bound_affinity_defaults(self):
self.assertEqual(env.get("GOMP_SPINCOUNT"), "200000")
self.assertEqual(env.get("OMP_DYNAMIC"), "FALSE")

def test_v4_libomp_team_sleeps_when_idle_on_windows(self):
# OMP_WAIT_POLICY=active alone makes LLVM's libomp spin forever (#341, #2286)
with mock.patch.dict(os.environ, {}, clear=True), \
mock.patch.object(self.coli.sys, "platform", "win32"):
env = self.coli.env_for_engine(self.args(), "deepseek_v4")
self.assertEqual(env.get("OMP_WAIT_POLICY"), "active")
self.assertEqual(env.get("KMP_BLOCKTIME"), "200")
self.assertNotIn("OMP_PROC_BIND", env)

def test_v4_blocktime_only_for_an_active_team_on_windows(self):
# libomp gives passive its blocktime 0 only while KMP_BLOCKTIME is unset;
# macOS's libomp is left as it was (#707)
for platform, environ in (("win32", {"OMP_WAIT_POLICY": " Passive"}),
("darwin", {}), ("linux", {})):
with self.subTest(platform=platform), \
mock.patch.dict(os.environ, environ, clear=True), \
mock.patch.object(self.coli.sys, "platform", platform):
env = self.coli.env_for_engine(self.args(), "deepseek_v4")
self.assertNotIn("KMP_BLOCKTIME", env)

def test_explicit_setting_still_wins(self):
with mock.patch.dict(os.environ, {"OMP_NUM_THREADS": "3"}, clear=True), \
mock.patch("resource_plan.physical_cpu_count",
Expand All @@ -425,7 +445,7 @@ def test_explicit_setting_still_wins(self):
def test_kill_switch_is_honoured(self):
with mock.patch.dict(os.environ, {"COLI_NO_OMP_TUNE": "1"}, clear=True):
env = self.coli.env_for_engine(self.args(), "deepseek_v4")
for key in ("OMP_NUM_THREADS", "OMP_WAIT_POLICY", "GOMP_SPINCOUNT",
for key in ("OMP_NUM_THREADS", "OMP_WAIT_POLICY", "GOMP_SPINCOUNT", "KMP_BLOCKTIME",
"OMP_DYNAMIC", "OMP_PROC_BIND", "OMP_PLACES"):
self.assertNotIn(key, env)

Expand Down
9 changes: 8 additions & 1 deletion c/tests/test_env_defaults.py
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,14 @@ def test_win32_sets_measured_defaults(self):
self.assertEqual(e["PIPE"], "1")
self.assertEqual(e["PILOT_REAL"], "1")
self.assertEqual(e["OMP_WAIT_POLICY"], "active")
self.assertEqual(e["KMP_BLOCKTIME"], "200") # libomp (clang): no endless idle spin
self.assertNotIn("OMP_PROC_BIND", e) # MinGW libgomp: niente affinity

def test_passive_keeps_libomp_blocktime_zero(self):
# libomp gives passive its blocktime 0 only while KMP_BLOCKTIME is unset
e = self.env_for_with({"OMP_WAIT_POLICY": "passive"}, "win32")
self.assertNotIn("KMP_BLOCKTIME", e)

def test_explicit_override_wins(self):
e = self.env_for_with({"DIRECT": "0", "PIPE": "0"}, "win32")
self.assertEqual(e["DIRECT"], "0")
Expand All @@ -70,13 +76,14 @@ def test_explicit_override_wins(self):
def test_kill_switch_scope_is_omp_only(self):
e = self.env_for_with({"COLI_NO_OMP_TUNE": "1"}, "win32")
self.assertNotIn("OMP_WAIT_POLICY", e)
self.assertNotIn("KMP_BLOCKTIME", e)
self.assertNotIn("OMP_NUM_THREADS", e)
self.assertEqual(e["DIRECT"], "1") # i default I/O restano attivi
self.assertEqual(e["PIPE"], "1")

def test_non_windows_untouched(self):
e = self.env_for_with({}, "linux")
for k in ("DIRECT", "PIPE", "PILOT_REAL", "OMP_WAIT_POLICY"):
for k in ("DIRECT", "PIPE", "PILOT_REAL", "OMP_WAIT_POLICY", "KMP_BLOCKTIME"):
self.assertNotIn(k, e)


Expand Down
Loading
Loading