flint CPU opt: IQ4_XS AVX512-VNNI vec_dot, Q5_K mask merge, rebased on de5122d; re-measured benchmarks and KLD

README.md

@@ -1,15 +1,15 @@
 # flint CPU optimization
 
-CPU-optimized build of [unsloth/flint-GGUF](https://huggingface.co/unsloth/flint-GGUF) (flint-UD-Q4_K_XL.gguf) using a llama.cpp CPU patch in the style of [HelloSun/Qwen3.6-35B-A3B-GGUF_CPU_OPT](https://huggingface.co/HelloSun/Qwen3.6-35B-A3B-GGUF_CPU_OPT).
+`flint-UD-Q4_K_XL.gguf` is the unmodified file from [unsloth/flint-GGUF](https://huggingface.co/unsloth/flint-GGUF).
+All changes are in `patches/flint-cpu-opt.patch` (llama.cpp commit in `patches/BASE_COMMIT`), in the style of
+[HelloSun/Qwen3.6-35B-A3B-GGUF_CPU_OPT](https://huggingface.co/HelloSun/Qwen3.6-35B-A3B-GGUF_CPU_OPT).
 
-## Apply
-```bash
-git clone https://github.com/ggerganov/llama.cpp
-cd llama.cpp
-git checkout $(cat patches/BASE_COMMIT)
-git apply patches/flint-cpu-opt.patch
+```
 ./build.sh
+taskset -c 0-15 llama.cpp/build/bin/llama-cli -m flint-UD-Q4_K_XL.gguf -t 16 -fa on -lm mlock
 ```
 
-## Results
-See benchmarks/summary.md. Mean KLD vs stock: 0.2058.
+Measured on Xeon Platinum 8559C, 16 threads (median of 3 interleaved rounds, vs stock llama.cpp):
+pp256 231 -> 296 t/s (1.28x), pp2048 274 -> 307 (1.12x), tg128 38.2 -> 42.8 (1.12x).
+PPL 18.28 vs 18.36 stock (no degradation), but **KLD vs stock is 0.067** (top-1 agreement ~92 %), much larger than on Qwen3.6.
+Details, caveats and the KLD bisect: `benchmarks/summary.md`. Validated only on this CPU.

benchmarks/ab-interleaved-flint.csv

@@ -0,0 +1,27 @@
+stock,1,0,245.145577
+stock,1,0,268.315708
+stock,1,0,37.025341
+qwen,1,0,264.423995
+qwen,1,0,307.766206
+qwen,1,0,42.127365
+opt,1,0,274.398648
+opt,1,0,306.705280
+opt,1,0,43.836806
+stock,2,0,229.210797
+stock,2,0,276.811500
+stock,2,0,38.154299
+qwen,2,0,265.295236
+qwen,2,0,308.702421
+qwen,2,0,42.523910
+opt,2,0,295.754988
+opt,2,0,301.992873
+opt,2,0,42.808636
+stock,3,0,231.043869
+stock,3,0,274.222148
+stock,3,0,39.321185
+qwen,3,0,261.775600
+qwen,3,0,304.829067
+qwen,3,0,39.530870
+opt,3,0,299.816432
+opt,3,0,318.658592
+opt,3,0,41.833254

benchmarks/kld-bisect.log

@@ -0,0 +1,5 @@
+no-VNNI Q4K : Mean    KLD:   0.067003 ±   0.005035
+no-VNNI Q5K : Mean    KLD:   0.052438 ±   0.003954
+no-VNNI Q6K : Mean    KLD:   0.068454 ±   0.005053
+no-VNNI IQ4XS : Mean    KLD:   0.066395 ±   0.004854
+ALLDONE

benchmarks/kld_vs_stock.log

@@ -0,0 +1,17 @@
+== base (stock)
+1.31.737.501 I Final estimate: PPL = 18.3627 +/- 0.79499
+== opt vs stock
+0.13.393.993 I 
+chunk             PPL               ln(PPL(Q)/PPL(base))          KL Divergence              Δp RMS            Same top p
+   6      24.3841 ±    2.5972       0.06191 ±    0.02147       0.06779 ±    0.00974     4.503 ±  0.393 %    91.699 ±  0.706 %
+  19      23.9972 ±    1.4052       0.03615 ±    0.00982       0.06924 ±    0.00674     4.672 ±  0.267 %    91.889 ±  0.392 %
+  27      17.8997 ±    0.8389       0.02832 ±    0.00775       0.06868 ±    0.00558     5.388 ±  0.294 %    92.433 ±  0.319 %
+Mean PPL(Q)                   :  18.277320 ±   0.787208
+Mean PPL(base)                :  17.797758 ±   0.744380
+Cor(ln(PPL(Q)), ln(PPL(base))):  98.68%
+Mean ln(PPL(Q)/PPL(base))     :   0.026588 ±   0.007011
+Mean PPL(Q)/PPL(base)         :   1.026945 ±   0.007200
+Mean PPL(Q)-PPL(base)         :   0.479562 ±   0.131595
+Mean    KLD:   0.066583 ±   0.004897
+99.9%   KLD:   7.426280
+99.0%   KLD:   1.112220

benchmarks/profile-decode.txt

@@ -0,0 +1,55 @@
+| model                          |       size |     params | backend    | threads |  fa |         lm |            test |                  t/s |
+| ------------------------------ | ---------: | ---------: | ---------- | ------: | --: | ---------: | --------------: | -------------------: |
+| flint ?B Q4_K - Medium         |  23.33 GiB |    39.79 B | CPU        |      16 |   1 |      mlock |            tg64 |         37.72 ± 0.00 |
+
+[GGML_PROF] total_us=1678629
+[GGML_PROF] GET_ROWS             f32   <q8_0   [  2048,    1,   1]      0.45 ms   0.03% calls=65 us/call=6.9
+[GGML_PROF] RMS_NORM             f32   <f32    [  2048,    1,   1]     18.00 ms   1.07% calls=6435 us/call=2.8
+[GGML_PROF] MUL_MAT              f32   <q8_0   [  4096,    1,   1]    171.92 ms  10.24% calls=2730 us/call=63.0
+[GGML_PROF] RMS_NORM             f32   <f32    [   128,   32,   1]      6.32 ms   0.38% calls=3185 us/call=2.0
+[GGML_PROF] MUL_MAT              f32   <q8_0   [   512,    1,   1]     62.51 ms   3.72% calls=5460 us/call=11.4
+[GGML_PROF] RMS_NORM             f32   <f32    [   128,    4,   1]      7.42 ms   0.44% calls=3185 us/call=2.3
+[GGML_PROF] SET_ROWS             f16   <f32    [   512,  256,   1]      7.33 ms   0.44% calls=6370 us/call=1.2
+[GGML_PROF] FLASH_ATTN_EXT       f32   <f32    [   128,   32,   1]     29.72 ms   1.77% calls=3185 us/call=9.3
+[GGML_PROF] MUL_MAT              f32   <q8_0   [  2048,    1,   1]    218.00 ms  12.99% calls=5070 us/call=43.0
+[GGML_PROF] ADD                  f32   <f32    [  2048,    1,   1]     34.85 ms   2.08% calls=24700 us/call=1.4
+[GGML_PROF] MUL_MAT              f32   <q8_0   [  6144,    1,   1]     22.79 ms   1.36% calls=260 us/call=87.6
+[GGML_PROF] GLU                  f32   <f32    [  6144,    1,   1]      0.63 ms   0.04% calls=130 us/call=4.8
+[GGML_PROF] ROPE                 f32   <f32    [   128,   32,   1]      5.86 ms   0.35% calls=2275 us/call=2.6
+[GGML_PROF] ROPE                 f32   <f32    [   128,    4,   1]      2.98 ms   0.18% calls=2275 us/call=1.3
+[GGML_PROF] MUL_MAT              f32   <bf16   [   256,    1,   1]     43.35 ms   2.58% calls=3055 us/call=14.2
+[GGML_PROF] UNARY                f32   <f32    [   256,    1,   1]      4.08 ms   0.24% calls=3055 us/call=1.3
+[GGML_PROF] ARGSORT              i32   <f32    [   256,    1,   1]     36.86 ms   2.20% calls=3055 us/call=12.1
+[GGML_PROF] GET_ROWS             f32   <f32    [     1,    6,   1]      4.56 ms   0.27% calls=3055 us/call=1.5
+[GGML_PROF] SUM_ROWS             f32   <f32    [     1,    1,   1]      3.27 ms   0.19% calls=3055 us/call=1.1
+[GGML_PROF] CLAMP                f32   <f32    [     1,    1,   1]      3.43 ms   0.20% calls=3055 us/call=1.1
+[GGML_PROF] DIV                  f32   <f32    [     6,    1,   1]      3.41 ms   0.20% calls=3055 us/call=1.1
+[GGML_PROF] MUL_MAT_ID           f32   <iq3_s  [   512,    6,   1]     18.62 ms   1.11% calls=130 us/call=143.2
+[GGML_PROF] GLU                  f32   <f32    [   512,    6,   1]      4.30 ms   0.26% calls=3055 us/call=1.4
+[GGML_PROF] UNARY                f32   <f32    [   512,    6,   1]      3.62 ms   0.22% calls=3055 us/call=1.2
+[GGML_PROF] POOL_1D              f32   <f32    [     1,    6,   1]     13.12 ms   0.78% calls=3055 us/call=4.3
+[GGML_PROF] SCALE                f32   <f32    [     1,    6,   1]     10.41 ms   0.62% calls=9165 us/call=1.1
+[GGML_PROF] LOG                  f32   <f32    [     1,    6,   1]      3.64 ms   0.22% calls=3055 us/call=1.2
+[GGML_PROF] UNARY                f32   <f32    [     1,    6,   1]      7.11 ms   0.42% calls=6110 us/call=1.2
+[GGML_PROF] CLAMP                f32   <f32    [     1,    6,   1]      3.29 ms   0.20% calls=3055 us/call=1.1
+[GGML_PROF] DIV                  f32   <f32    [   512,    6,   1]      5.20 ms   0.31% calls=3055 us/call=1.7
+[GGML_PROF] MUL_MAT_ID           f32   <q5_K   [  2048,    6,   1]    141.87 ms   8.45% calls=1885 us/call=75.3
+[GGML_PROF] MUL                  f32   <f32    [  2048,    6,   1]     30.55 ms   1.82% calls=6110 us/call=5.0
+[GGML_PROF] MUL_MAT              f32   <q8_0   [  1024,    1,   1]     82.51 ms   4.92% calls=4420 us/call=18.7
+[GGML_PROF] GLU                  f32   <f32    [  1024,    1,   1]      3.47 ms   0.21% calls=3055 us/call=1.1
+[GGML_PROF] MUL_MAT_ID           f32   <q5_K   [   512,    6,   1]    220.48 ms  13.13% calls=2340 us/call=94.2
+[GGML_PROF] MUL_MAT_ID           f32   <q8_0   [  2048,    6,   1]     12.57 ms   0.75% calls=65 us/call=193.4
+[GGML_PROF] MUL_MAT_ID           f32   <q6_K   [  2048,    6,   1]     10.04 ms   0.60% calls=65 us/call=154.5
+[GGML_PROF] MUL_MAT              f32   <q5_K   [  4096,    1,   1]     33.90 ms   2.02% calls=455 us/call=74.5
+[GGML_PROF] MUL_MAT              f32   <q5_K   [   512,    1,   1]     18.08 ms   1.08% calls=910 us/call=19.9
+[GGML_PROF] MUL_MAT              f32   <q5_K   [  2048,    1,   1]     52.65 ms   3.14% calls=1235 us/call=42.6
+[GGML_PROF] MUL_MAT_ID           f32   <iq4_xs [   512,    6,   1]    102.81 ms   6.12% calls=3250 us/call=31.6
+[GGML_PROF] MUL_MAT_ID           f32   <iq4_xs [  2048,    6,   1]     35.32 ms   2.10% calls=1040 us/call=34.0
+[GGML_PROF] MUL_MAT_ID           f32   <q4_K   [   512,    6,   1]     12.47 ms   0.74% calls=390 us/call=32.0
+[GGML_PROF] MUL_MAT              f32   <q5_K   [  1024,    1,   1]     36.01 ms   2.14% calls=1560 us/call=23.1
+[GGML_PROF] MUL_MAT              f32   <iq4_xs [  1024,    1,   1]      2.38 ms   0.14% calls=130 us/call=18.3
+[GGML_PROF] MUL_MAT              f32   <iq4_xs [  2048,    1,   1]      1.02 ms   0.06% calls=65 us/call=15.7
+[GGML_PROF] GET_ROWS             f32   <f32    [  2048,    1,   1]      0.19 ms   0.01% calls=130 us/call=1.5
+[GGML_PROF] MUL_MAT              f32   <q8_0   [150144,    1,   1]    125.28 ms   7.46% calls=65 us/call=1927.3
+
+build: de5122d87 (1)

benchmarks/profile-prefill.txt

@@ -0,0 +1,79 @@
+| model                          |       size |     params | backend    | threads | n_batch | n_ubatch |  fa |         lm |            test |                  t/s |
+| ------------------------------ | ---------: | ---------: | ---------- | ------: | ------: | -------: | --: | ---------: | --------------: | -------------------: |
+| flint ?B Q4_K - Medium         |  23.33 GiB |    39.79 B | CPU        |      16 |    1024 |     1024 |   1 |      mlock |          pp2048 |        309.71 ± 0.00 |
+
+[GGML_PROF] total_us=13581874
+[GGML_PROF] GET_ROWS             f32   <q8_0   [  2048, 1024,   1]      5.90 ms   0.04% calls=4 us/call=1476.0
+[GGML_PROF] RMS_NORM             f32   <f32    [  2048, 1024,   1]    125.01 ms   0.92% calls=388 us/call=322.2
+[GGML_PROF] MUL_MAT              f32   <q8_0   [  4096, 1024,   1]    732.99 ms   5.40% calls=168 us/call=4363.0
+[GGML_PROF] RMS_NORM             f32   <f32    [   128,   32,1024]    139.90 ms   1.03% calls=196 us/call=713.8
+[GGML_PROF] MUL_MAT              f32   <q8_0   [   512, 1024,   1]    245.74 ms   1.81% calls=336 us/call=731.4
+[GGML_PROF] RMS_NORM             f32   <f32    [   128,    4,1024]     50.24 ms   0.37% calls=196 us/call=256.3
+[GGML_PROF] SET_ROWS             f16   <f32    [   512, 2048,   1]     19.08 ms   0.14% calls=112 us/call=170.4
+[GGML_PROF] FLASH_ATTN_EXT       f32   <f32    [   128,   32,1024]   3199.22 ms  23.56% calls=196 us/call=16322.6
+[GGML_PROF] MUL_MAT              f32   <q8_0   [  2048, 1024,   1]    948.76 ms   6.99% calls=308 us/call=3080.4
+[GGML_PROF] ADD                  f32   <f32    [  2048, 1024,   1]     88.97 ms   0.66% calls=1488 us/call=59.8
+[GGML_PROF] MUL_MAT              f32   <q8_0   [  6144, 1024,   1]    138.57 ms   1.02% calls=16 us/call=8660.9
+[GGML_PROF] GLU                  f32   <f32    [  6144, 1024,   1]     12.83 ms   0.09% calls=8 us/call=1603.5
+[GGML_PROF] ROPE                 f32   <f32    [   128,   32,1024]     42.53 ms   0.31% calls=140 us/call=303.8
+[GGML_PROF] ROPE                 f32   <f32    [   128,    4,1024]     10.87 ms   0.08% calls=140 us/call=77.6
+[GGML_PROF] SET_ROWS             f16   <f32    [   512, 1792,   1]      3.89 ms   0.03% calls=280 us/call=13.9
+[GGML_PROF] MUL_MAT              f32   <bf16   [   256, 1024,   1]    197.14 ms   1.45% calls=184 us/call=1071.4
+[GGML_PROF] UNARY                f32   <f32    [   256, 1024,   1]     11.94 ms   0.09% calls=184 us/call=64.9
+[GGML_PROF] ARGSORT              i32   <f32    [   256, 1024,   1]    150.49 ms   1.11% calls=184 us/call=817.9
+[GGML_PROF] GET_ROWS             f32   <f32    [     1,    6,1024]      2.21 ms   0.02% calls=184 us/call=12.0
+[GGML_PROF] SUM_ROWS             f32   <f32    [     1, 1024,   1]      1.02 ms   0.01% calls=184 us/call=5.6
+[GGML_PROF] CLAMP                f32   <f32    [     1, 1024,   1]      1.94 ms   0.01% calls=184 us/call=10.5
+[GGML_PROF] DIV                  f32   <f32    [     6, 1024,   1]      0.65 ms   0.00% calls=184 us/call=3.5
+[GGML_PROF] MUL_MAT_ID           f32   <iq3_s  [   512,    6,1024]    111.13 ms   0.82% calls=8 us/call=13891.1
+[GGML_PROF] GLU                  f32   <f32    [   512,    6,1024]     41.05 ms   0.30% calls=184 us/call=223.1
+[GGML_PROF] UNARY                f32   <f32    [   512,    6,1024]     14.72 ms   0.11% calls=184 us/call=80.0
+[GGML_PROF] POOL_1D              f32   <f32    [     1,    6,1024]    717.08 ms   5.28% calls=184 us/call=3897.2
+[GGML_PROF] SCALE                f32   <f32    [     1,    6,1024]      6.38 ms   0.05% calls=552 us/call=11.6
+[GGML_PROF] LOG                  f32   <f32    [     1,    6,1024]      1.32 ms   0.01% calls=184 us/call=7.2
+[GGML_PROF] UNARY                f32   <f32    [     1,    6,1024]      1.79 ms   0.01% calls=368 us/call=4.9
+[GGML_PROF] CLAMP                f32   <f32    [     1,    6,1024]      8.17 ms   0.06% calls=184 us/call=44.4
+[GGML_PROF] DIV                  f32   <f32    [   512,    6,1024]     48.76 ms   0.36% calls=184 us/call=265.0
+[GGML_PROF] MUL_MAT_ID           f32   <q5_K   [  2048,    6,1024]   1450.95 ms  10.68% calls=116 us/call=12508.2
+[GGML_PROF] MUL                  f32   <f32    [  2048,    6,1024]    388.11 ms   2.86% calls=368 us/call=1054.7
+[GGML_PROF] MUL_MAT              f32   <q8_0   [  1024, 1024,   1]    292.28 ms   2.15% calls=264 us/call=1107.1
+[GGML_PROF] GLU                  f32   <f32    [  1024, 1024,   1]      9.64 ms   0.07% calls=184 us/call=52.4
+[GGML_PROF] MUL_MAT_ID           f32   <q5_K   [   512,    6,1024]   1415.40 ms  10.42% calls=144 us/call=9829.2
+[GGML_PROF] MUL_MAT_ID           f32   <q8_0   [  2048,    6,1024]    105.06 ms   0.77% calls=4 us/call=26264.0
+[GGML_PROF] MUL_MAT_ID           f32   <q6_K   [  2048,    6,1024]     62.41 ms   0.46% calls=4 us/call=15601.8
+[GGML_PROF] MUL_MAT              f32   <q5_K   [  4096, 1024,   1]    140.26 ms   1.03% calls=28 us/call=5009.4
+[GGML_PROF] MUL_MAT              f32   <q5_K   [   512, 1024,   1]     48.46 ms   0.36% calls=56 us/call=865.3
+[GGML_PROF] MUL_MAT              f32   <q5_K   [  2048, 1024,   1]    215.82 ms   1.59% calls=76 us/call=2839.8
+[GGML_PROF] MUL_MAT_ID           f32   <iq4_xs [   512,    6,1024]   1376.81 ms  10.14% calls=192 us/call=7170.9
+[GGML_PROF] MUL_MAT_ID           f32   <iq4_xs [  2048,    6,1024]    631.64 ms   4.65% calls=60 us/call=10527.4
+[GGML_PROF] MUL_MAT_ID           f32   <q4_K   [   512,    6,1024]    194.15 ms   1.43% calls=24 us/call=8089.7
+[GGML_PROF] MUL_MAT              f32   <q5_K   [  1024, 1024,   1]    146.41 ms   1.08% calls=96 us/call=1525.1
+[GGML_PROF] MUL_MAT              f32   <iq4_xs [  1024, 1024,   1]     10.24 ms   0.08% calls=8 us/call=1280.6
+[GGML_PROF] MUL_MAT              f32   <iq4_xs [  2048, 1024,   1]      4.50 ms   0.03% calls=4 us/call=1126.0
+[GGML_PROF] GET_ROWS             f32   <f32    [  2048,    1,   1]      0.02 ms   0.00% calls=8 us/call=2.1
+[GGML_PROF] ADD                  f32   <f32    [  2048,    1,   1]      0.05 ms   0.00% calls=32 us/call=1.5
+[GGML_PROF] RMS_NORM             f32   <f32    [  2048,    1,   1]      0.04 ms   0.00% calls=8 us/call=4.5
+[GGML_PROF] MUL_MAT              f32   <bf16   [   256,    1,   1]      0.07 ms   0.00% calls=4 us/call=16.8
+[GGML_PROF] UNARY                f32   <f32    [   256,    1,   1]      0.01 ms   0.00% calls=4 us/call=2.5
+[GGML_PROF] ARGSORT              i32   <f32    [   256,    1,   1]      0.06 ms   0.00% calls=4 us/call=13.8
+[GGML_PROF] GET_ROWS             f32   <f32    [     1,    6,   1]      0.01 ms   0.00% calls=4 us/call=1.2
+[GGML_PROF] SUM_ROWS             f32   <f32    [     1,    1,   1]      0.01 ms   0.00% calls=4 us/call=1.5
+[GGML_PROF] CLAMP                f32   <f32    [     1,    1,   1]      0.00 ms   0.00% calls=4 us/call=1.0
+[GGML_PROF] DIV                  f32   <f32    [     6,    1,   1]      0.00 ms   0.00% calls=4 us/call=1.0
+[GGML_PROF] MUL_MAT_ID           f32   <iq4_xs [   512,    6,   1]      0.25 ms   0.00% calls=8 us/call=30.9
+[GGML_PROF] GLU                  f32   <f32    [   512,    6,   1]      0.01 ms   0.00% calls=4 us/call=2.2
+[GGML_PROF] UNARY                f32   <f32    [   512,    6,   1]      0.01 ms   0.00% calls=4 us/call=1.5
+[GGML_PROF] POOL_1D              f32   <f32    [     1,    6,   1]      0.02 ms   0.00% calls=4 us/call=5.8
+[GGML_PROF] SCALE                f32   <f32    [     1,    6,   1]      0.01 ms   0.00% calls=12 us/call=1.1
+[GGML_PROF] LOG                  f32   <f32    [     1,    6,   1]      0.01 ms   0.00% calls=4 us/call=1.8
+[GGML_PROF] UNARY                f32   <f32    [     1,    6,   1]      0.01 ms   0.00% calls=8 us/call=1.4
+[GGML_PROF] CLAMP                f32   <f32    [     1,    6,   1]      0.00 ms   0.00% calls=4 us/call=1.0
+[GGML_PROF] DIV                  f32   <f32    [   512,    6,   1]      0.01 ms   0.00% calls=4 us/call=1.2
+[GGML_PROF] MUL_MAT_ID           f32   <iq4_xs [  2048,    6,   1]      0.14 ms   0.00% calls=4 us/call=35.0
+[GGML_PROF] MUL                  f32   <f32    [  2048,    6,   1]      0.03 ms   0.00% calls=8 us/call=3.5
+[GGML_PROF] MUL_MAT              f32   <q8_0   [  1024,    1,   1]      0.15 ms   0.00% calls=8 us/call=18.6
+[GGML_PROF] GLU                  f32   <f32    [  1024,    1,   1]      0.01 ms   0.00% calls=4 us/call=2.0
+[GGML_PROF] MUL_MAT              f32   <q8_0   [  2048,    1,   1]      0.07 ms   0.00% calls=4 us/call=17.5
+[GGML_PROF] MUL_MAT              f32   <q8_0   [150144,    1,   1]      8.43 ms   0.06% calls=4 us/call=2106.2
+
+build: de5122d87 (1)

benchmarks/summary.md

@@ -1,14 +1,42 @@
-# flint CPU optimization benchmarks
+# flint-UD-Q4_K_XL CPU optimization (Xeon Platinum 8559C, 16 threads)
 
-Model: unsloth/flint-GGUF flint-UD-Q4_K_XL.gguf (25.05 GiB)
-Hardware: Intel Xeon Platinum 8559C, 16 threads, taskset -c 0-15, -fa 1
+`llama-bench -t 16 -fa on -lm mlock -b 1024 -ub 1024`, 3 interleaved rounds (stock / Qwen3.6 patch / this patch per round), median t/s.
+Raw: `ab-interleaved-flint.csv`.
 
-| metric | stock (t/s) | optimized (t/s) | speedup |
+| variant | pp256 | pp2048 | tg128 |
 |---|---|---|---|
-| pp256 | 221.5 | 260.9 | 1.15x |
-| tg128 | 32.6  | 35.4  | 1.07x |
-| pp2048 | 215.4 | 263.4 | 1.22x |
-| tg128 (pp2048 run) | 32.9 | 37.3 | 1.13x |
+| stock llama.cpp (de5122d) | 231.0 | 274.2 | 38.2 |
+| Qwen3.6 CPU-OPT patch, unchanged | 264.4 | 307.8 | 42.1 |
+| **this patch** | **295.8** | **306.7** | **42.8** |
 
-Mean KLD vs stock: 0.2058 ± 0.0123 (prefill+decode, 32 chunks x 512)
-Mean PPL(Q)/PPL(base): 1.0136
+Versus stock: pp256 1.28x, pp2048 1.12x, tg128 1.12x. Versus the Qwen patch alone, only pp256 (+12%) is clearly better;
+decode (+2%) and pp2048 are within noise. Most of the gain comes from the Qwen3.6 patch's kernels also applying to flint.
+
+## What is new (flint-specific)
+flint-UD-Q4_K_XL is Q5_K 11.6 GiB + IQ4_XS 9.0 GiB + Q8_0 2.0 GiB (+ small Q4_K/Q6_K/IQ3_S).
+1. IQ4_XS x Q8_K `vec_dot`: AVX512-VNNI (the x86 path had only AVX2). Unsigned LUT (+128) with `vpdpbusd`, bias removed
+   through `bsums`, L2 prefetch (`GGML_IQ4XS_PF`, default 40). Decode MUL_MAT_ID iq4_xs reaches ~97 GB/s.
+2. Q5_K x Q8_K `vec_dot`: 5th bit merged with `vptestmb` + masked add instead of shift/and/or.
+   Bit-identical results to the previous VNNI version (KLD unchanged on Qwen3.6 as well).
+Other contents are the Qwen3.6 patch (Q4_K/Q5_K/Q6_K VNNI, AMX tweaks, tiled MoE group 32). Its Gated-DeltaNet changes
+do not apply to flint.
+
+## Accuracy (wikitext-2 test, 32 x 512, vs stock logits)
+| | KLD | same top-1 | PPL |
+|---|---|---|---|
+| stock vs stock (-t 16 vs -t 8) | 0.0000 | 100 % | 18.363 |
+| Qwen3.6 patch unchanged | 0.0664 +- 0.0049 | 92.2 % | - |
+| this patch | 0.0666 +- 0.0049 | ~92 % | **18.277** +- 0.787 |
+
+PPL is not worse than stock (18.363 +- 0.795), but KLD is far larger than on Qwen3.6 (0.003-0.006), so this patch is
+**not** numerically close to stock on flint. Bisect (`kld-bisect.log`, KLD when one VNNI kernel is disabled):
+Q4_K 0.0670, Q5_K 0.0524, Q6_K 0.0685, IQ4_XS 0.0664; removing all of `quants.c` gives 0.0162.
+So the Q4_K/Q5_K/Q6_K VNNI kernels carry most of the deviation and the new IQ4_XS kernel only ~0.0002.
+I did not find a defect in them (official `test-quantize-fns` dot-product errors equal stock); my unverified guess is that
+flint's 256-expert top-6 routing amplifies tiny numeric differences. The earlier flint_cpu_opt README reported KLD 0.2058.
+
+## Profile (decode, 64 tokens)
+MUL_MAT q8_0 41 % (attention, shared expert, lm_head; close to bandwidth bound), MUL_MAT_ID q5_K 22 %,
+MUL_MAT q5_K 8 %, MUL_MAT_ID iq4_xs 8 %. Prefill: FLASH_ATTN_EXT 24 %, MUL_MAT_ID q5_K 21 %, q8_0 17 %, iq4_xs 15 %.
+Prefetch distance, thread count and `--poll` sweeps showed no gain beyond noise.
+Not done: flash-attention kernel work, Q8_0 decode kernel.

build.sh

@@ -1,4 +1,9 @@
-#!/usr/bin/env bash
+#!/bin/bash
+# Build llama.cpp at patches/BASE_COMMIT with the flint CPU patch (Xeon with AVX512-VNNI + AMX).
 set -e
+git clone https://github.com/ggml-org/llama.cpp && cd llama.cpp
+git fetch --depth 1 origin $(cat ../patches/BASE_COMMIT) && git checkout $(cat ../patches/BASE_COMMIT)
+git apply ../patches/flint-cpu-opt.patch
 cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_NATIVE=ON -DGGML_CUDA=OFF -DLLAMA_CURL=OFF
-cmake --build build -j$(nproc) --target llama-bench llama-cli llama-perplexity
+cmake --build build -j$(nproc) --target llama-cli llama-server llama-bench llama-perplexity
+# run: taskset -c 0-15 build/bin/llama-cli -m flint-UD-Q4_K_XL.gguf -t 16 -fa on -lm mlock

machine.txt

@@ -1,6 +1,5 @@
-CPU: Intel Xeon Platinum 8559C (Emerald Rapids)
-Threads used: 16 (cgroup cpu.max 1600000/100000)
-Memory cap: ~104 GiB (cgroup memory.max)
-Build: GCC 14.2.0, cmake 3.31.6, -march=native (AVX512, VNNI, AMX-INT8)
-llama.cpp base commit: 338fce651 (b11942)
-AMX buffer: 1716 MiB (Q8_0 2D tensors); CPU_REPACK: 864 MiB (Q4_K 8x8)
+Model name:                              Intel(R) Xeon(R) Platinum 8559C
+Threads: 16 (taskset -c 0-15, cgroup cpu.max 1600000/100000)
+Build: GCC 14.2.0, cmake 3.31.6, -DGGML_NATIVE=ON (AVX512, VNNI, AMX-INT8)
+llama.cpp base commit: de5122d87
+Host was shared; single runs vary by roughly +-8%.

patches/BASE_COMMIT

@@ -1 +1 @@
-338fce6511c744893cc0d3d958b275f40552c7a4
+de5122d879d00d2895c36be7e0cee94105aeeabd

patches/flint-cpu-opt.patch

@@ -1,3 +1,33 @@
+diff --git a/ggml/include/ggml.h b/ggml/include/ggml.h
+index d62a63104..0cc5ed9cb 100644
+--- a/ggml/include/ggml.h
++++ b/ggml/include/ggml.h
+@@ -2678,6 +2678,25 @@ extern "C" {
+             struct ggml_tensor  * state,
+             int64_t               K);
+ 
++    // Cache variant of ggml_gated_delta_net (single-snapshot, K == 1): instead of a gathered copy of the
++    // initial state, the state of sequence i is read straight from row ids[i] of the state cache, and the
++    // final state is written in place to row i of state_dst (a view of consecutive cache rows), so the caller
++    // needs no get_rows before and no cpy after the op.
++    //   state_cache : [S_v*S_v*H_v, n_cells] f32, contiguous rows
++    //   ids         : [n_seqs] i32, source cell of every sequence
++    //   state_dst   : [S_v*S_v*H_v, n_seqs] f32 view into the cache (destination cells)
++    // The packed output keeps the layout of ggml_gated_delta_net; its snapshot part is left unwritten.
++    GGML_API struct ggml_tensor * ggml_gated_delta_net_cache(
++            struct ggml_context * ctx,
++            struct ggml_tensor  * q,
++            struct ggml_tensor  * k,
++            struct ggml_tensor  * v,
++            struct ggml_tensor  * g,
++            struct ggml_tensor  * beta,
++            struct ggml_tensor  * state_cache,
++            struct ggml_tensor  * ids,
++            struct ggml_tensor  * state_dst);
++
+     // DSA lightning indexer
+     //
+     // q:       [n_embd_idx, n_head_idx, n_batch, ne3 ]
 diff --git a/ggml/src/ggml-cpu/amx/amx.cpp b/ggml/src/ggml-cpu/amx/amx.cpp
 index e90c2797f..e8b61a2eb 100644
 --- a/ggml/src/ggml-cpu/amx/amx.cpp
@@ -142,7 +172,7 @@ index 9f3a744b5..d23e4d971 100644
  
          parallel_for_ggml(params, n_batch * NB, [&](int begin, int end) {
 diff --git a/ggml/src/ggml-cpu/arch/x86/quants.c b/ggml/src/ggml-cpu/arch/x86/quants.c
-index 21cd6312a..0e8aafa76 100644
+index 21cd6312a..9b65eebb2 100644
 --- a/ggml/src/ggml-cpu/arch/x86/quants.c
 +++ b/ggml/src/ggml-cpu/arch/x86/quants.c
 @@ -2118,7 +2118,89 @@ void ggml_vec_dot_q4_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
@@ -236,7 +266,7 @@ index 21cd6312a..0e8aafa76 100644
  
      const __m256i m4 = _mm256_set1_epi8(0xF);
  
-@@ -2296,7 +2378,97 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
+@@ -2296,7 +2378,110 @@ void ggml_vec_dot_q5_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
  
      uint32_t utmp[4];
  
@@ -260,6 +290,13 @@ index 21cd6312a..0e8aafa76 100644
 +    const __m512i cr2 = K5_CNT(0, 2);   // sub-blocks 4,6 : right shift s-4
 +    const __m512i cr3 = K5_CNT(1, 3);   // sub-blocks 5,7
 +    #undef K5_CNT
++    // per-vector bit selectors: low 32 bytes test bit s, high 32 bytes test bit s+2 (sub-block pairs (0,2),(1,3),(4,6),(5,7))
++    #define K5_BM(a, b) _mm512_inserti64x4(_mm512_castsi256_si512(_mm256_set1_epi8((char)(1 << (a)))), _mm256_set1_epi8((char)(1 << (b))), 1)
++    const __m512i bm0 = K5_BM(0, 2);
++    const __m512i bm1 = K5_BM(1, 3);
++    const __m512i bm2 = K5_BM(4, 6);
++    const __m512i bm3 = K5_BM(5, 7);
++    #undef K5_BM
 +
 +    static int k5pf = -1;
 +    if (k5pf < 0) { const char * e = getenv("GGML_K5_PF"); k5pf = e ? atoi(e) : 48; }
@@ -307,10 +344,16 @@ index 21cd6312a..0e8aafa76 100644
 +        const __m512i Q2 = _mm512_loadu_si512((const void *)(q8 + 128));
 +        const __m512i Q3 = _mm512_loadu_si512((const void *)(q8 + 192));
 +
-+        const __m512i a0 = _mm512_or_si512(_mm512_and_si512(v0, m4),                       _mm512_and_si512(_mm512_sllv_epi16(qhv, cl0), m10));
-+        const __m512i a1 = _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi16(v0, 4), m4), _mm512_and_si512(_mm512_sllv_epi16(qhv, cl1), m10));
-+        const __m512i a2 = _mm512_or_si512(_mm512_and_si512(v1, m4),                       _mm512_and_si512(_mm512_srlv_epi16(qhv, cr2), m10));
-+        const __m512i a3 = _mm512_or_si512(_mm512_and_si512(_mm512_srli_epi16(v1, 4), m4), _mm512_and_si512(_mm512_srlv_epi16(qhv, cr3), m10));
++        // a_k = nibble + 16 * bit(qh[l], sub-block). qhv holds qh (32 B) duplicated in both 256-bit halves;
++        // lanes 0-31 of a vector are sub-block s (low half) and 32-63 are sub-block s+2 (high half) -> per-half bit masks.
++        const __m512i n0 = _mm512_and_si512(v0, m4);
++        const __m512i a0 = _mm512_mask_add_epi8(n0, _mm512_test_epi8_mask(qhv, bm0), n0, m10);
++        const __m512i n1 = _mm512_and_si512(_mm512_srli_epi16(v0, 4), m4);
++        const __m512i a1 = _mm512_mask_add_epi8(n1, _mm512_test_epi8_mask(qhv, bm1), n1, m10);
++        const __m512i n2 = _mm512_and_si512(v1, m4);
++        const __m512i a2 = _mm512_mask_add_epi8(n2, _mm512_test_epi8_mask(qhv, bm2), n2, m10);
++        const __m512i n3 = _mm512_and_si512(_mm512_srli_epi16(v1, 4), m4);
++        const __m512i a3 = _mm512_mask_add_epi8(n3, _mm512_test_epi8_mask(qhv, bm3), n3, m10);
 +
 +        const __m512i b0 = _mm512_shuffle_i64x2(Q0, Q1, 0x44);
 +        const __m512i b1 = _mm512_shuffle_i64x2(Q0, Q1, 0xEE);
@@ -335,7 +378,7 @@ index 21cd6312a..0e8aafa76 100644
  
      const __m256i m4 = _mm256_set1_epi8(0xF);
      const __m128i mzero = _mm_setzero_si128();
-@@ -2500,7 +2672,81 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
+@@ -2500,7 +2685,81 @@ void ggml_vec_dot_q6_K_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const voi
  
      const int nb = n / QK_K;
  
@@ -418,84 +461,249 @@ index 21cd6312a..0e8aafa76 100644
  
      const __m256i m3 = _mm256_set1_epi8(3);
      const __m256i m15 = _mm256_set1_epi8(15);
-diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
-index 53d39d249..02889fd24 100644
---- a/ggml/src/ggml-cpu/ggml-cpu.c
-+++ b/ggml/src/ggml-cpu/ggml-cpu.c
-@@ -35,6 +35,49 @@
- #include <limits.h>
- #include <stdarg.h>
- #include <signal.h>
-+
-+// ---- lightweight per-op profiling (GGML_PROF=1), diagnostic only ----
-+static int g_prof = -1;
-+struct ggml_prof_entry { const char * op; const char * type; const char * ty0; int64_t ne0, ne1, ne2; int64_t us; int64_t calls; };
-+static struct ggml_prof_entry g_prof_tab[1024];
-+static int g_prof_n = 0;
-+static int64_t g_prof_total = 0;
-+
-+static void ggml_prof_dump(void) {
-+    if (g_prof_n == 0) return;
-+    fprintf(stderr, "\n[GGML_PROF] total_us=%" PRId64 "\n", g_prof_total);
-+    for (int i = 0; i < g_prof_n; ++i) {
-+        fprintf(stderr, "[GGML_PROF] %-20s %-6s<%-6s [%6" PRId64 ",%5" PRId64 ",%4" PRId64 "] %9.2f ms %6.2f%% calls=%" PRId64 " us/call=%.1f\n",
-+            g_prof_tab[i].op, g_prof_tab[i].type, g_prof_tab[i].ty0, g_prof_tab[i].ne0, g_prof_tab[i].ne1, g_prof_tab[i].ne2,
-+            g_prof_tab[i].us/1000.0,
-+            100.0*g_prof_tab[i].us/(g_prof_total?g_prof_total:1),
-+            g_prof_tab[i].calls,
-+            (double)g_prof_tab[i].us/(g_prof_tab[i].calls?g_prof_tab[i].calls:1));
-+    }
-+    fflush(stderr);
-+}
+@@ -4078,7 +4337,75 @@ void ggml_vec_dot_iq4_xs_q8_K(int n, float * GGML_RESTRICT s, size_t bs, const v
+ 
+     const int nb = n / QK_K;
+ 
+-#if defined __AVX2__
++#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VL__) && defined(__AVX512VNNI__)
++    // AVX512-VNNI path (IQ4_XS x Q8_K). 64 qs bytes = 4 sub-blocks (16 B each, low nibble -> values 0..15, high
++    // nibble -> values 16..31 of the same sub-block). The kvalues LUT is stored as unsigned (v + 128) so that
++    // vpdpbusd can be used with 32-bit accumulation; the +128 bias is removed once per super-block through bsums.
++    const __m512i lut = _mm512_broadcast_i32x4(_mm_xor_si128(_mm_loadu_si128((const __m128i *)kvalues_iq4nl), _mm_set1_epi8((char)0x80)));
++    const __m512i m4  = _mm512_set1_epi8(0xF);
++    const __m512i idxA = _mm512_setr_epi32(0,0,0,0, 1,1,1,1, 2,2,2,2, 3,3,3,3);
++    const __m512i idxB = _mm512_setr_epi32(4,4,4,4, 5,5,5,5, 6,6,6,6, 7,7,7,7);
++    const __m128i hsh  = _mm_setr_epi16(0, 2, 4, 6, 8, 10, 12, 14);
 +
-+static void ggml_prof_add(const struct ggml_tensor * node, int64_t us) {
-+    g_prof_total += us;
-+    const char * op = ggml_op_name(node->op);
-+    const char * ty = ggml_type_name(node->type);
-+    const char * ty0 = node->src[0] ? ggml_type_name(node->src[0]->type) : "-";
-+    const int64_t ne0 = node->ne[0], ne1 = node->ne[1], ne2 = node->ne[2];
-+    for (int i = 0; i < g_prof_n; ++i) {
-+        if (strcmp(g_prof_tab[i].op, op) == 0 && strcmp(g_prof_tab[i].type, ty) == 0 && strcmp(g_prof_tab[i].ty0, ty0) == 0 &&
-+                g_prof_tab[i].ne0 == ne0 && g_prof_tab[i].ne1 == ne1 && g_prof_tab[i].ne2 == ne2) {
-+            g_prof_tab[i].us += us; g_prof_tab[i].calls++;
-+            return;
++    static int ipf = -1;
++    if (ipf < 0) { const char * e = getenv("GGML_IQ4XS_PF"); ipf = e ? atoi(e) : 40; }
++
++    __m512 acc = _mm512_setzero_ps();
++
++    for (int ibl = 0; ibl < nb; ++ibl) {
++        {
++            const char * pf = (const char *)(x + ibl + ipf);
++            _mm_prefetch(pf,       _MM_HINT_T1);
++            _mm_prefetch(pf +  64, _MM_HINT_T1);
++            _mm_prefetch(pf + 128, _MM_HINT_T1);
 +        }
++
++        // 6-bit sub-block scales (-32), 8 per super-block
++        uint32_t sl; memcpy(&sl, x[ibl].scales_l, 4);
++        const __m128i xl  = _mm_cvtsi32_si128((int)sl);
++        const __m128i l8  = _mm_unpacklo_epi8(_mm_and_si128(xl, _mm_set1_epi8(0x0f)),
++                                              _mm_and_si128(_mm_srli_epi16(xl, 4), _mm_set1_epi8(0x0f)));
++        const __m128i h16 = _mm_slli_epi16(_mm_and_si128(_mm_srlv_epi16(_mm_set1_epi16((short)x[ibl].scales_h), hsh), _mm_set1_epi16(3)), 4);
++        const __m128i ls16 = _mm_sub_epi16(_mm_or_si128(_mm_cvtepu8_epi16(l8), h16), _mm_set1_epi16(32));
++        const __m512i ls512 = _mm512_castsi256_si512(_mm256_cvtepi16_epi32(ls16));
++        const __m512i sc0 = _mm512_permutexvar_epi32(idxA, ls512);
++        const __m512i sc1 = _mm512_permutexvar_epi32(idxB, ls512);
++
++        const uint8_t * qs = x[ibl].qs;
++        const int8_t  * q8 = y[ibl].qs;
++
++        const __m512i q0 = _mm512_loadu_si512((const void *)(qs));
++        const __m512i q1 = _mm512_loadu_si512((const void *)(qs + 64));
++        const __m512i Y0 = _mm512_loadu_si512((const void *)(q8));
++        const __m512i Y1 = _mm512_loadu_si512((const void *)(q8 + 64));
++        const __m512i Y2 = _mm512_loadu_si512((const void *)(q8 + 128));
++        const __m512i Y3 = _mm512_loadu_si512((const void *)(q8 + 192));
++
++        const __m512i xl0 = _mm512_shuffle_epi8(lut, _mm512_and_si512(q0, m4));
++        const __m512i xh0 = _mm512_shuffle_epi8(lut, _mm512_and_si512(_mm512_srli_epi16(q0, 4), m4));
++        const __m512i xl1 = _mm512_shuffle_epi8(lut, _mm512_and_si512(q1, m4));
++        const __m512i xh1 = _mm512_shuffle_epi8(lut, _mm512_and_si512(_mm512_srli_epi16(q1, 4), m4));
++
++        __m512i p0 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), xl0, _mm512_shuffle_i64x2(Y0, Y1, 0x88));
++        p0         = _mm512_dpbusd_epi32(p0,                     xh0, _mm512_shuffle_i64x2(Y0, Y1, 0xDD));
++        __m512i p1 = _mm512_dpbusd_epi32(_mm512_setzero_si512(), xl1, _mm512_shuffle_i64x2(Y2, Y3, 0x88));
++        p1         = _mm512_dpbusd_epi32(p1,                     xh1, _mm512_shuffle_i64x2(Y2, Y3, 0xDD));
++
++        __m512i sumi = _mm512_add_epi32(_mm512_mullo_epi32(p0, sc0), _mm512_mullo_epi32(p1, sc1));
++
++        // remove the +128 bias: 128 * sum_k ls_k * (bsums[2k] + bsums[2k+1])
++        const __m256i bs   = _mm256_loadu_si256((const __m256i *)y[ibl].bsums);
++        const __m256i lsd  = _mm256_set_m128i(_mm_unpackhi_epi16(ls16, ls16), _mm_unpacklo_epi16(ls16, ls16));
++        const __m512i corr = _mm512_slli_epi32(_mm512_zextsi256_si512(_mm256_madd_epi16(bs, lsd)), 7);
++        sumi = _mm512_sub_epi32(sumi, corr);
++
++        const float d = GGML_CPU_FP16_TO_FP32(x[ibl].d) * y[ibl].d;
++        acc = _mm512_fmadd_ps(_mm512_set1_ps(d), _mm512_cvtepi32_ps(sumi), acc);
 +    }
-+    if (g_prof_n < 1024) {
-+        g_prof_tab[g_prof_n].op = op; g_prof_tab[g_prof_n].type = ty; g_prof_tab[g_prof_n].ty0 = ty0;
-+        g_prof_tab[g_prof_n].ne0 = ne0; g_prof_tab[g_prof_n].ne1 = ne1; g_prof_tab[g_prof_n].ne2 = ne2;
-+        g_prof_tab[g_prof_n].us = us; g_prof_tab[g_prof_n].calls = 1;
-+        g_prof_n++;
-+    }
-+}
-+// ----------------------------------------------------------------
- #if defined(__gnu_linux__)
- #include <syscall.h>
- #endif
-@@ -3148,6 +3191,13 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
-             continue;
-         }
++
++    *s = _mm512_reduce_add_ps(acc);
++
++#elif defined __AVX2__
+ 
+     const __m128i values128 = _mm_loadu_si128((const __m128i*)kvalues_iq4nl);
+     const __m128i m4b  = _mm_set1_epi8(0x0f);
+diff --git a/ggml/src/ggml-cpu/ops.cpp b/ggml/src/ggml-cpu/ops.cpp
+index 85e17018c..119d3c997 100644
+--- a/ggml/src/ggml-cpu/ops.cpp
++++ b/ggml/src/ggml-cpu/ops.cpp
+@@ -5316,6 +5316,29 @@ static void ggml_compute_forward_get_rows_f32(
+     const int ith = params->ith;
+     const int nth = params->nth;
  
-+        if (g_prof < 0) {
-+            const char * e = getenv("GGML_PROF");
-+            g_prof = e && e[0] == '1' ? 1 : 0;
-+            if (g_prof) atexit(ggml_prof_dump);
++    // few rows: parallelize across the row instead of across rows so that a single
++    // large gather (e.g. the recurrent state of a single sequence) uses all threads
++    if (nr < nth && nc >= 4096) {
++        const int64_t per = (nc + nth - 1)/nth;
++        const int64_t c0  = per*ith;
++        const int64_t c1  = MIN(c0 + per, nc);
++        if (c0 < c1) {
++            for (int64_t i = 0; i < nr; ++i) {
++                const int64_t i12 = i/(ne11*ne10);
++                const int64_t i11 = (i - i12*ne11*ne10)/ne10;
++                const int64_t i10 = (i - i12*ne11*ne10 - i11*ne10);
++                const int64_t i01 = *(int32_t *) ((char *) src1->data + i10*nb10 + i11*nb11 + i12*nb12);
++
++                GGML_ASSERT(i01 >= 0 && i01 < ne01);
++
++                memcpy((char *) dst->data + i10*nb1 + i11*nb2 + i12*nb3 + c0*sizeof(float),
++                       (char *) src0->data + i01*nb01 + i11*nb02 + i12*nb03 + c0*sizeof(float),
++                       (c1 - c0)*sizeof(float));
++            }
 +        }
-+        const int64_t prof_t0 = (g_prof && state->ith == 0) ? ggml_time_us() : 0;
-+
-         // TODO: move fused-op detection into ggml_graph_plan so fusion decisions are made once at planning time
-         // Try fused ops, fall back to normal compute
-         const int n_fused = ggml_cpu_try_fuse_ops(cgraph, node_n, &params, cplan);
-@@ -3166,6 +3216,9 @@ static thread_ret_t ggml_graph_compute_thread(void * data) {
-         if (node_n + 1 < cgraph->n_nodes) {
-             ggml_barrier(state->threadpool);
-         }
-+        if (g_prof && state->ith == 0) {
-+            ggml_prof_add(node, ggml_time_us() - prof_t0);
++        return;
++    }
++
+     // rows per thread
+     const int dr = (nr + nth - 1)/nth;
+ 
+@@ -11108,6 +11131,9 @@ static void ggml_compute_forward_gated_delta_net_one_chunk(
+     ggml_tensor * src_g     = dst->src[3];
+     ggml_tensor * src_beta  = dst->src[4];
+     ggml_tensor * src_state = dst->src[5];
++    // cache mode (see ggml_gated_delta_net_cache): src[6] = source cell ids, src[7] = in-place destination view
++    const ggml_tensor * src_ids  = dst->src[6];
++    const ggml_tensor * src_sdst = dst->src[7];
+ 
+     const int64_t S_v      = src_v->ne[0];
+     const int64_t H        = src_v->ne[1];
+@@ -11119,7 +11145,7 @@ static void ggml_compute_forward_gated_delta_net_one_chunk(
+     GGML_ASSERT(ggml_is_contiguous_rows(src_v));
+     GGML_ASSERT(ggml_is_contiguous(src_g));
+     GGML_ASSERT(ggml_is_contiguous(src_beta));
+-    GGML_ASSERT(ggml_is_contiguous(src_state));
++    GGML_ASSERT(src_ids ? src_state->nb[1] == (size_t) (S_v * S_v * H) * sizeof(float) : ggml_is_contiguous(src_state));
+ 
+     GGML_ASSERT(src_g->ne[0] == 1 || src_g->ne[0] == S_v);
+     GGML_ASSERT(src_beta->ne[0] == 1);
+@@ -11139,6 +11165,7 @@ static void ggml_compute_forward_gated_delta_net_one_chunk(
+     // K (snapshot slot count) is an op param; state holds s0 only [S_v, S_v, H, n_seqs].
+     const int64_t K = ggml_get_op_params_i32(dst, 0);
+     GGML_ASSERT(K >= 1);
++    GGML_ASSERT(!src_ids || K == 1);
+     // per-seq stride in floats (seq s starts at state + s * seq_stride)
+     const int64_t state_seq_stride = src_state->nb[3] / sizeof(float);
+ 
+@@ -11182,13 +11209,18 @@ static void ggml_compute_forward_gated_delta_net_one_chunk(
+         // For K>1, work in scratch and copy out per-token when the slot is in range.
+         float * s_out = (K > 1)
+             ? state_work
+-            : state_out_base + (iv3 * H + iv1) * S_v * S_v;
++            : (src_sdst
++                ? (float *) src_sdst->data + iv3 * (src_sdst->nb[1] / sizeof(float)) + iv1 * S_v * S_v
++                : state_out_base + (iv3 * H + iv1) * S_v * S_v);
+ 
+         // copy input state into the working buffer and operate in-place
+         // state layout [S_v, S_v, H, n_seqs]: seq iv3 starts at iv3 * state_seq_stride.
+-        const float * s_in = state_in_base + iv3 * state_seq_stride + iv1 * S_v * S_v;
+-        memcpy(s_out, s_in, S_v * S_v * sizeof(float));
+-
++        const float * s_in = src_ids
++            ? state_in_base + (int64_t) ((const int32_t *) src_ids->data)[iv3] * (src_state->nb[1] / sizeof(float)) + iv1 * S_v * S_v
++            : state_in_base + iv3 * state_seq_stride + iv1 * S_v * S_v;
++        if (s_out != s_in) {
++            memcpy(s_out, s_in, S_v * S_v * sizeof(float));
 +        }
-     }
+         // attn output pointer for first token of this (head, seq)
+         float * attn_data = attn_out_base + (iv3 * n_tokens * H + iv1) * S_v;
  
- #ifdef GGML_USE_OPENMP
+@@ -11203,6 +11235,56 @@ static void ggml_compute_forward_gated_delta_net_one_chunk(
+             // state is stored transposed: s_out[j*S_v + i] = S[i][j]
+             // so row j of s_out = column j of S (contiguous access)
+ 
++            bool gdn_done = false;
++#if defined(__AVX512F__)
++            if (!kda && S_v == 128) {
++                // Fused 2-pass update for the scalar-gate case (non-KDA), S_v == 128:
++                //   delta[j] = (v[j] - g * dot(S_j, k)) * beta
++                //   S_j     <- g * S_j + delta[j] * k ;  out[j] = dot(S_j, q) * scale
++                // Same element-wise fma as the reference path; only the dot rounding order differs.
++                const float g = expf(g_d[0]);
++                __m512 kv[8], qv[8];
++                for (int i = 0; i < 8; ++i) {
++                    kv[i] = _mm512_loadu_ps(k_d + 16 * i);
++                    qv[i] = _mm512_loadu_ps(q_d + 16 * i);
++                }
++                for (int64_t j = 0; j < 128; j += 4) {
++                    float sums[4];
++                    for (int r = 0; r < 4; ++r) {
++                        const float * row = s_out + (j + r) * 128;
++                        __m512 a0 = _mm512_mul_ps(_mm512_loadu_ps(row),      kv[0]);
++                        __m512 a1 = _mm512_mul_ps(_mm512_loadu_ps(row + 16), kv[1]);
++                        a0 = _mm512_fmadd_ps(_mm512_loadu_ps(row + 32),  kv[2], a0);
++                        a1 = _mm512_fmadd_ps(_mm512_loadu_ps(row + 48),  kv[3], a1);
++                        a0 = _mm512_fmadd_ps(_mm512_loadu_ps(row + 64),  kv[4], a0);
++                        a1 = _mm512_fmadd_ps(_mm512_loadu_ps(row + 80),  kv[5], a1);
++                        a0 = _mm512_fmadd_ps(_mm512_loadu_ps(row + 96),  kv[6], a0);
++                        a1 = _mm512_fmadd_ps(_mm512_loadu_ps(row + 112), kv[7], a1);
++                        sums[r] = _mm512_reduce_add_ps(_mm512_add_ps(a0, a1));
++                    }
++                    for (int r = 0; r < 4; ++r) {
++                        delta[j + r] = (v_d[j + r] - g * sums[r]) * beta_val;
++                    }
++                }
++                const __m512 gv = _mm512_set1_ps(g);
++                for (int64_t j = 0; j < 128; ++j) {
++                    float * row = s_out + j * 128;
++                    const __m512 dv = _mm512_set1_ps(delta[j]);
++                    __m512 aq0 = _mm512_setzero_ps(), aq1 = _mm512_setzero_ps();
++                    for (int i = 0; i < 8; i += 2) {
++                        const __m512 s0 = _mm512_fmadd_ps(dv, kv[i],     _mm512_mul_ps(gv, _mm512_loadu_ps(row + 16 * i)));
++                        const __m512 s1 = _mm512_fmadd_ps(dv, kv[i + 1], _mm512_mul_ps(gv, _mm512_loadu_ps(row + 16 * (i + 1))));
++                        _mm512_storeu_ps(row + 16 * i,       s0);
++                        _mm512_storeu_ps(row + 16 * (i + 1), s1);
++                        aq0 = _mm512_fmadd_ps(s0, qv[i],     aq0);
++                        aq1 = _mm512_fmadd_ps(s1, qv[i + 1], aq1);
++                    }
++                    attn_data[j] = _mm512_reduce_add_ps(_mm512_add_ps(aq0, aq1)) * scale;
++                }
++                gdn_done = true;
++            }
++#endif
++            if (!gdn_done) {
+             if (kda) {
+                 // precompute exp(g) into delta scratch (reused below)
+                 for (int64_t i = 0; i < S_v; ++i) {
+@@ -11235,6 +11317,8 @@ static void ggml_compute_forward_gated_delta_net_one_chunk(
+                 attn_data[j] = sum * scale;
+             }
+ 
++            }
++
+             attn_data += S_v * H; // advance to next token
+ 
+             if (K > 1) {
+diff --git a/ggml/src/ggml-cpu/repack.cpp b/ggml/src/ggml-cpu/repack.cpp
+index 9282ca299..0431b7ada 100644
+--- a/ggml/src/ggml-cpu/repack.cpp
++++ b/ggml/src/ggml-cpu/repack.cpp
+@@ -5009,7 +5009,10 @@ static const ggml::cpu::tensor_traits * ggml_repack_get_optimal_repack_type(cons
+             #endif
+         }
+     } else if (cur->type == GGML_TYPE_Q4_K) {
+-        if (ggml_cpu_has_avx2()) {
++        const char * rp_q4k = getenv("GGML_REPACK_Q4K");
++        // Xeon 8559C: the AVX512-VNNI Q4_K vec_dot beats the AVX2 8x8 repack on decode+prefill; opt in with GGML_REPACK_Q4K=1
++        const bool q4k_repack = (rp_q4k && rp_q4k[0] == '1');
++        if (q4k_repack && ggml_cpu_has_avx2()) {
+             if (cur->ne[1] % 8 == 0) {
+                 return &q4_K_8x8_q8_K;
+             }
 diff --git a/ggml/src/ggml-cpu/tiled/tiled.cpp b/ggml/src/ggml-cpu/tiled/tiled.cpp
 index 77e1a9945..06fd9a2d5 100644
 --- a/ggml/src/ggml-cpu/tiled/tiled.cpp
@@ -511,3 +719,208 @@ index 77e1a9945..06fd9a2d5 100644
  
  // like tiled_store_window, but the dst columns are not contiguous: col_ptrs[j] points at the
  // start of dst column j, whose rows are contiguous (dim 0, nb0 == 4 bytes).
+diff --git a/ggml/src/ggml.c b/ggml/src/ggml.c
+index 5d8449a88..203f15e43 100644
+--- a/ggml/src/ggml.c
++++ b/ggml/src/ggml.c
+@@ -6416,6 +6416,59 @@ struct ggml_tensor * ggml_solve_tri(
+ 
+ // ggml_gated_delta_net
+ 
++struct ggml_tensor * ggml_gated_delta_net_cache(
++        struct ggml_context * ctx,
++        struct ggml_tensor  * q,
++        struct ggml_tensor  * k,
++        struct ggml_tensor  * v,
++        struct ggml_tensor  * g,
++        struct ggml_tensor  * beta,
++        struct ggml_tensor  * state_cache,
++        struct ggml_tensor  * ids,
++        struct ggml_tensor  * state_dst) {
++    GGML_ASSERT(ggml_is_contiguous_rows(q));
++    GGML_ASSERT(ggml_is_contiguous_rows(k));
++    GGML_ASSERT(ggml_is_contiguous_rows(v));
++    GGML_ASSERT(ggml_is_contiguous(g));
++    GGML_ASSERT(ggml_is_contiguous(beta));
++    GGML_ASSERT(q->type == GGML_TYPE_F32 && k->type == GGML_TYPE_F32 && v->type == GGML_TYPE_F32);
++    GGML_ASSERT(g->type == GGML_TYPE_F32 && beta->type == GGML_TYPE_F32);
++
++    const int64_t S_v      = v->ne[0];
++    const int64_t H        = v->ne[1];
++    const int64_t n_tokens = v->ne[2];
++    const int64_t n_seqs   = v->ne[3];
++
++    GGML_ASSERT(g->ne[0] == 1 || g->ne[0] == S_v);
++    GGML_ASSERT(beta->ne[0] == 1);
++
++    GGML_ASSERT(state_cache->type == GGML_TYPE_F32);
++    GGML_ASSERT(state_cache->ne[0] == S_v * S_v * H);
++    GGML_ASSERT(state_cache->nb[1] == (size_t) (S_v * S_v * H) * sizeof(float));
++    GGML_ASSERT(ids->type == GGML_TYPE_I32);
++    GGML_ASSERT(ids->ne[0] == n_seqs);
++    GGML_ASSERT(state_dst->type == GGML_TYPE_F32);
++    GGML_ASSERT(state_dst->ne[0] == S_v * S_v * H && state_dst->ne[1] == n_seqs);
++    GGML_ASSERT(state_dst->nb[1] == state_cache->nb[1]);
++
++    const int64_t ne[4] = { S_v * H, n_tokens * n_seqs + S_v * n_seqs, 1, 1 };
++    struct ggml_tensor * result = ggml_new_tensor(ctx, GGML_TYPE_F32, 4, ne);
++
++    ggml_set_op_params_i32(result, 0, 1); // K == 1
++
++    result->op     = GGML_OP_GATED_DELTA_NET;
++    result->src[0] = q;
++    result->src[1] = k;
++    result->src[2] = v;
++    result->src[3] = g;
++    result->src[4] = beta;
++    result->src[5] = state_cache;
++    result->src[6] = ids;
++    result->src[7] = state_dst;
++
++    return result;
++}
++
+ struct ggml_tensor * ggml_gated_delta_net(
+         struct ggml_context * ctx,
+         struct ggml_tensor  * q,
+diff --git a/src/llama-graph.cpp b/src/llama-graph.cpp
+index 7309705dc..de5403dbe 100644
+--- a/src/llama-graph.cpp
++++ b/src/llama-graph.cpp
+@@ -3915,6 +3915,13 @@ ggml_tensor * llm_graph_context::build_rs(
+     // {state_size, rs_size} -> {state_size, n_rs - i0}
+     const int64_t i0 = n_rs - state_copy->ne[0];
+ 
++    // custom getter + no extra states: nothing to gather or copy
++    if (get_state_rows && n_rs == (uint32_t) n_seqs) {
++        ggml_tensor * output_states = get_state_rows(ctx0, states, state_copy_main);
++        ggml_build_forward_expand(gf, output_states);
++        return output_states;
++    }
++
+     ggml_tensor * states_all = ggml_get_rows(ctx0, states, state_copy);
+ 
+     ggml_tensor * output_states = get_state_rows ?
+diff --git a/src/models/delta-net-base.cpp b/src/models/delta-net-base.cpp
+index 2e15777ee..efc8c0e8e 100644
+--- a/src/models/delta-net-base.cpp
++++ b/src/models/delta-net-base.cpp
+@@ -521,6 +521,46 @@ ggml_tensor * llm_build_delta_net_base::build_conv_state(
+     return conv_input;
+ }
+ 
++ggml_tensor * llm_build_delta_net_base::build_recurrent_attn_cache(
++        llm_graph_input_rs * inp,
++        ggml_tensor *        ssm_states_all,
++        ggml_tensor *        state_cache,
++        ggml_tensor *        q,
++        ggml_tensor *        k,
++        ggml_tensor *        v,
++        ggml_tensor *        g,
++        ggml_tensor *        b,
++        int                  il) {
++    const auto * mctx_cur = inp->mctx;
++    const auto   kv_head  = mctx_cur->get_head();
++
++    const int64_t S_v          = v->ne[0];
++    const int64_t H_v          = v->ne[1];
++    const int64_t n_seq_tokens = q->ne[2];
++    const int64_t n_seqs       = q->ne[3];
++
++    GGML_ASSERT(n_seqs == 1 && cparams.n_rs_seq == 0);
++
++    ggml_tensor * state_dst = ggml_view_2d(ctx0, ssm_states_all, hparams.n_embd_s(), n_seqs, ssm_states_all->nb[1],
++            kv_head * hparams.n_embd_s() * ggml_element_size(ssm_states_all));
++
++    ggml_tensor * result = ggml_gated_delta_net_cache(ctx0, q, k, v, g, b, state_cache, inp->s_copy_main, state_dst);
++    if (n_seq_tokens == 1) {
++        res->add_fused_node({LLM_FUSED_OP_GDN_AR, result, il});
++    } else {
++        res->add_fused_node({LLM_FUSED_OP_GDN_CH, result, il});
++    }
++
++    ggml_tensor * output = ggml_view_4d(ctx0, result,
++            S_v, H_v, n_seq_tokens, n_seqs,
++            ggml_row_size(result->type, S_v),
++            ggml_row_size(result->type, S_v * H_v),
++            ggml_row_size(result->type, S_v * H_v * n_seq_tokens), 0);
++    cb(output, "attn_output", il);
++
++    return output;
++}
++
+ ggml_tensor * llm_build_delta_net_base::build_recurrent_attn(
+         llm_graph_input_rs * inp,
+         ggml_tensor *        ssm_states_all,
+diff --git a/src/models/models.h b/src/models/models.h
+index 5d9b6077a..d0ba683fc 100644
+--- a/src/models/models.h
++++ b/src/models/models.h
+@@ -88,6 +88,19 @@ struct llm_build_delta_net_base : public llm_graph_context {
+ 
+     // run delta-net attention and write the new recurrent state(s) back to ssm_states_all
+     // s: (head_v_dim, head_v_dim, num_v_heads, n_seqs); returns output: (head_v_dim, num_v_heads, n_seq_tokens, n_seqs)
++    // single-sequence fast path: reads the state straight from the cache and updates it in place
++    // (no get_rows before, no cpy after); only valid for n_rs_seq == 0, n_seqs == 1 and the fused GDN ops
++    ggml_tensor * build_recurrent_attn_cache(
++            llm_graph_input_rs * inp,
++            ggml_tensor *        ssm_states_all,
++            ggml_tensor *        state_cache,
++            ggml_tensor *        q,
++            ggml_tensor *        k,
++            ggml_tensor *        v,
++            ggml_tensor *        g,
++            ggml_tensor *        b,
++            int                  il);
++
+     ggml_tensor * build_recurrent_attn(
+             llm_graph_input_rs * inp,
+             ggml_tensor *        ssm_states_all,
+diff --git a/src/models/qwen35moe.cpp b/src/models/qwen35moe.cpp
+index 939f10f7f..ffb396c1c 100644
+--- a/src/models/qwen35moe.cpp
++++ b/src/models/qwen35moe.cpp
+@@ -1,3 +1,4 @@
++#include <cstdlib>
+ #include "models.h"
+ #include "llama-memory-recurrent.h"
+ 
+@@ -406,9 +407,24 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear(
+ 
+     ggml_tensor * conv_input = build_conv_state(inp, conv_states_all, qkv_mixed, conv_kernel_size, conv_channels, il);
+ 
+-    ggml_tensor * state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs);
+-    state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs);
+-    cb(state, "state_predelta", il);
++    // Single-sequence decode/prefill fast path (CPU GDN): read the recurrent state straight from the cache and
++    // update it in place, instead of get_rows + gdn + cpy. Disable with LLAMA_GDN_NO_INPLACE=1.
++    static const bool gdn_inplace_allowed = getenv("LLAMA_GDN_NO_INPLACE") == nullptr;
++    const bool gdn_inplace = gdn_inplace_allowed && n_seqs == 1 && cparams.n_rs_seq == 0 &&
++            (n_seq_tokens == 1 ? cparams.fused_gdn_ar : cparams.fused_gdn_ch) &&
++            mctx_cur->get_n_rs() == 1;
++
++    ggml_tensor * state       = nullptr;
++    ggml_tensor * state_cache = nullptr;
++    if (gdn_inplace) {
++        // the custom getter hands back the cache itself; build_rs still orders the optional state clearing
++        state_cache = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs,
++                [](ggml_context *, ggml_tensor * states, ggml_tensor *) { return states; });
++    } else {
++        state = build_rs(inp, ssm_states_all, hparams.n_embd_s(), n_seqs);
++        state = ggml_reshape_4d(ctx0, state, head_v_dim, head_v_dim, num_v_heads, n_seqs);
++        cb(state, "state_predelta", il);
++    }
+ 
+     ggml_tensor * conv_output_proper = ggml_ssm_conv(ctx0, conv_input, conv_kernel);
+     cb(conv_output_proper, "conv_output_raw", il);
+@@ -467,7 +483,9 @@ ggml_tensor * llama_model_qwen35moe::graph::build_layer_attn_linear(
+     cb(k_conv, "k_conv_predelta", il);
+     cb(v_conv, "v_conv_predelta", il);
+ 
+-    ggml_tensor * output = build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il);
++    ggml_tensor * output = gdn_inplace
++        ? build_recurrent_attn_cache(inp, ssm_states_all, state_cache, q_conv, k_conv, v_conv, gate, beta, il)
++        : build_recurrent_attn(inp, ssm_states_all, q_conv, k_conv, v_conv, gate, beta, state, il);
+ 
+     // z: [head_dim, n_heads, n_tokens, n_seqs] -> [n_heads * n_tokens * n_seqs, head_dim]
+     ggml_tensor * z_2d = ggml_reshape_4d(ctx0, z, head_v_dim, num_v_heads, n_seq_tokens, n_seqs);

patches/optional-profiler.patch

@@ -1,5 +1,5 @@
 diff --git a/ggml/src/ggml-cpu/ggml-cpu.c b/ggml/src/ggml-cpu/ggml-cpu.c
-index 53d39d249..02889fd24 100644
+index 53d39d2..02889fd 100644
 --- a/ggml/src/ggml-cpu/ggml-cpu.c
 +++ b/ggml/src/ggml-cpu/ggml-cpu.c
 @@ -35,6 +35,49 @@