From 4c353eb4dcdb041a1e51696114e538d938f384be Mon Sep 17 00:00:00 2001 From: cyclic Date: Thu, 9 Oct 2025 22:45:46 +0000 Subject: [PATCH] Update results.md --- results.md | 180 ++++++++++++++++++++++++++++++++++++++++++++++------- 1 file changed, 156 insertions(+), 24 deletions(-) diff --git a/results.md b/results.md index bbdac32..31a23e7 100644 --- a/results.md +++ b/results.md @@ -2,29 +2,40 @@ |---:|---|---:|---:|---:| | 1 | gpt-oss:20b | 89.1% | 49 | 733 | | 2 | hf.co/BasedBase/Qwen3-Coder-30B-A3B-Instruct-480B-Distill-V2:Q4_K_M | 88.2% | 49 | 726 | -| 3 | hf.co/bartowski/NousResearch_Hermes-4-14B-GGUF:Q4_K_M | 73.6% | 49 | 606 | -| 4 | hf.co/unsloth/gemma-3n-E4B-it-GGUF:Q8_0 | 66.0% | 49 | 543 | -| 5 | hf.co/unsloth/Phi-4-mini-reasoning-GGUF:Q8_0 | 65.5% | 49 | 539 | -| 6 | hf.co/unsloth/gemma-3n-E2B-it-GGUF:Q8_0 | 49.9% | 49 | 411 | -| 7 | hf.co/unsloth/Qwen3-0.6B-GGUF:BF16 | 45.8% | 49 | 377 | -| 8 | hf.co/unsloth/gemma-3-4b-it-GGUF:Q8_0 | 44.2% | 49 | 364 | -| 9 | hf.co/bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0 | 42.2% | 49 | 347 | -| 10 | hf.co/unsloth/gemma-3-1b-it-GGUF:BF16 | 35.0% | 49 | 288 | +| 3 | hf.co/unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF:Q4_K_S | 85.9% | 49 | 707 | +| 4 | hf.co/mradermacher/aquif-3.5-8B-Think-GGUF:Q6_K | 82.5% | 49 | 679 | +| 5 | hf.co/mradermacher/aquif-3.5-7B-GGUF:Q8_0 | 79.2% | 49 | 652 | +| 6 | hf.co/unsloth/Qwen3-4B-Instruct-2507-GGUF:Q8_0 | 78.9% | 49 | 649 | +| 7 | hf.co/unsloth/Qwen2.5-Coder-7B-Instruct-GGUF:Q8_0 | 76.7% | 49 | 631 | +| 8 | hf.co/unsloth/Qwen3-4B-Instruct-2507-GGUF:F16 | 75.0% | 49 | 617 | +| 9 | hf.co/TorpedoSoftware/Luau-Devstral-24B-Instruct-v0.1:Q4_K_M | 73.8% | 49 | 607 | +| 10 | hf.co/bartowski/NousResearch_Hermes-4-14B-GGUF:Q4_K_M | 73.6% | 49 | 606 | +| 11 | hf.co/mradermacher/aquif-3-moe-17b-a2.8b-thinking-GGUF:Q4_K_M | 71.9% | 49 | 592 | +| 12 | hf.co/unsloth/gemma-3n-E4B-it-GGUF:Q8_0 | 66.0% | 49 | 543 | +| 13 | hf.co/unsloth/Phi-4-mini-reasoning-GGUF:Q8_0 | 65.5% | 49 | 539 | +| 14 | hf.co/mradermacher/aquif-3.5-3B-GGUF:F16 | 60.6% | 49 | 499 | +| 15 | hf.co/unsloth/gemma-3-12b-it-GGUF:Q4_K_M | 56.3% | 49 | 463 | +| 16 | hf.co/unsloth/gemma-3n-E2B-it-GGUF:Q8_0 | 49.9% | 49 | 411 | +| 17 | hf.co/unsloth/Qwen3-0.6B-GGUF:BF16 | 45.8% | 49 | 377 | +| 18 | hf.co/unsloth/gemma-3-4b-it-GGUF:Q8_0 | 44.2% | 49 | 364 | +| 19 | hf.co/bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0 | 42.2% | 49 | 347 | +| 20 | hf.co/unsloth/gemma-3-1b-it-GGUF:BF16 | 35.0% | 49 | 288 | +| 21 | hf.co/mradermacher/Gemma-3-1B-Roblox-Luau-GGUF:F16 | 19.7% | 49 | 162 | --- -## Per-category stats +## Per-Category Stats ### 1) gpt-oss:20b - **Aggregated:** 89.1% - **Count:** 49 - **Sum Total:** 733 | Category | % | Count | Total | Max | |---|---:|---:|---:|---:| | syntax | 100.0% | 8 | 95 | 95 | -| types | 95.5% | 11 | 189 | 198 | -| advanced | 93.0% | 8 | 107 | 115 | -| internals | 88.4% | 8 | 190 | 215 | -| runtimes | 78.9% | 6 | 75 | 95 | -| compatibility | 66.7% | 8 | 70 | 105 | +| types | 97.5% | 11 | 193 | 198 | +| compatibility | 95.2% | 8 | 100 | 105 | +| advanced | 91.3% | 8 | 105 | 115 | +| internals | 86.0% | 8 | 185 | 215 | +| runtimes | 57.9% | 6 | 55 | 95 | ### 2) hf.co/BasedBase/Qwen3-Coder-30B-A3B-Instruct-480B-Distill-V2:Q4_K_M - **Aggregated:** 88.2% - **Count:** 49 - **Sum Total:** 726 @@ -37,7 +48,84 @@ | runtimes | 78.9% | 6 | 75 | 95 | | compatibility | 66.7% | 8 | 70 | 105 | -### 3) hf.co/bartowski/NousResearch_Hermes-4-14B-GGUF:Q4_K_M +### 3) hf.co/unsloth/Qwen3-Coder-30B-A3B-Instruct-GGUF:Q4_K_S +- **Aggregated:** 85.9% - **Count:** 49 - **Sum Total:** 707 +| Category | % | Count | Total | Max | +|---|---:|---:|---:|---:| +| syntax | 100.0% | 8 | 95 | 95 | +| types | 90.4% | 11 | 179 | 198 | +| compatibility | 88.6% | 8 | 93 | 105 | +| advanced | 87.0% | 8 | 100 | 115 | +| internals | 83.7% | 8 | 180 | 215 | +| runtimes | 63.2% | 6 | 60 | 95 | + +### 4) hf.co/mradermacher/aquif-3.5-8B-Think-GGUF:Q6_K +- **Aggregated:** 82.5% - **Count:** 49 - **Sum Total:** 679 +| Category | % | Count | Total | Max | +|---|---:|---:|---:|---:| +| advanced | 91.3% | 8 | 105 | 115 | +| internals | 88.4% | 8 | 190 | 215 | +| types | 83.3% | 11 | 165 | 198 | +| syntax | 80.0% | 8 | 76 | 95 | +| compatibility | 79.0% | 8 | 83 | 105 | +| runtimes | 63.2% | 6 | 60 | 95 | + +### 5) hf.co/mradermacher/aquif-3.5-7B-GGUF:Q8_0 +- **Aggregated:** 79.2% - **Count:** 49 - **Sum Total:** 652 +| Category | % | Count | Total | Max | +|---|---:|---:|---:|---:| +| internals | 85.1% | 8 | 183 | 215 | +| types | 83.8% | 11 | 166 | 198 | +| advanced | 82.6% | 8 | 95 | 115 | +| syntax | 82.1% | 8 | 78 | 95 | +| compatibility | 76.2% | 8 | 80 | 105 | +| runtimes | 52.6% | 6 | 50 | 95 | + +### 6) hf.co/unsloth/Qwen3-4B-Instruct-2507-GGUF:Q8_0 +- **Aggregated:** 78.9% - **Count:** 49 - **Sum Total:** 649 +| Category | % | Count | Total | Max | +|---|---:|---:|---:|---:| +| syntax | 89.5% | 8 | 85 | 95 | +| internals | 87.4% | 8 | 188 | 215 | +| advanced | 82.6% | 8 | 95 | 115 | +| types | 78.8% | 11 | 156 | 198 | +| compatibility | 71.4% | 8 | 75 | 105 | +| runtimes | 52.6% | 6 | 50 | 95 | + +### 7) hf.co/unsloth/Qwen2.5-Coder-7B-Instruct-GGUF:Q8_0 +- **Aggregated:** 76.7% - **Count:** 49 - **Sum Total:** 631 +| Category | % | Count | Total | Max | +|---|---:|---:|---:|---:| +| syntax | 94.7% | 8 | 90 | 95 | +| advanced | 93.9% | 8 | 108 | 115 | +| types | 84.3% | 11 | 167 | 198 | +| compatibility | 71.4% | 8 | 75 | 105 | +| internals | 70.2% | 8 | 151 | 215 | +| runtimes | 42.1% | 6 | 40 | 95 | + +### 8) hf.co/unsloth/Qwen3-4B-Instruct-2507-GGUF:F16 +- **Aggregated:** 75.0% - **Count:** 49 - **Sum Total:** 617 +| Category | % | Count | Total | Max | +|---|---:|---:|---:|---:| +| syntax | 94.7% | 8 | 90 | 95 | +| advanced | 82.6% | 8 | 95 | 115 | +| internals | 79.1% | 8 | 170 | 215 | +| compatibility | 71.4% | 8 | 75 | 105 | +| types | 69.2% | 11 | 137 | 198 | +| runtimes | 52.6% | 6 | 50 | 95 | + +### 9) hf.co/TorpedoSoftware/Luau-Devstral-24B-Instruct-v0.1:Q4_K_M +- **Aggregated:** 73.8% - **Count:** 49 - **Sum Total:** 607 +| Category | % | Count | Total | Max | +|---|---:|---:|---:|---:| +| compatibility | 83.8% | 8 | 88 | 105 | +| advanced | 78.3% | 8 | 90 | 115 | +| types | 77.8% | 11 | 154 | 198 | +| runtimes | 74.7% | 6 | 71 | 95 | +| syntax | 74.7% | 8 | 71 | 95 | +| internals | 61.9% | 8 | 133 | 215 | + +### 10) hf.co/bartowski/NousResearch_Hermes-4-14B-GGUF:Q4_K_M - **Aggregated:** 73.6% - **Count:** 49 - **Sum Total:** 606 | Category | % | Count | Total | Max | |---|---:|---:|---:|---:| @@ -48,7 +136,18 @@ | runtimes | 57.9% | 6 | 55 | 95 | | compatibility | 57.1% | 8 | 60 | 105 | -### 4) hf.co/unsloth/gemma-3n-E4B-it-GGUF:Q8_0 +### 11) hf.co/mradermacher/aquif-3-moe-17b-a2.8b-thinking-GGUF:Q4_K_M +- **Aggregated:** 71.9% - **Count:** 49 - **Sum Total:** 592 +| Category | % | Count | Total | Max | +|---|---:|---:|---:|---:| +| compatibility | 81.0% | 8 | 85 | 105 | +| internals | 75.3% | 8 | 162 | 215 | +| syntax | 74.7% | 8 | 71 | 95 | +| advanced | 73.9% | 8 | 85 | 115 | +| types | 68.7% | 11 | 136 | 198 | +| runtimes | 55.8% | 6 | 53 | 95 | + +### 12) hf.co/unsloth/gemma-3n-E4B-it-GGUF:Q8_0 - **Aggregated:** 66.0% - **Count:** 49 - **Sum Total:** 543 | Category | % | Count | Total | Max | |---|---:|---:|---:|---:| @@ -59,7 +158,7 @@ | runtimes | 55.8% | 6 | 53 | 95 | | compatibility | 55.2% | 8 | 58 | 105 | -### 5) hf.co/unsloth/Phi-4-mini-reasoning-GGUF:Q8_0 +### 13) hf.co/unsloth/Phi-4-mini-reasoning-GGUF:Q8_0 - **Aggregated:** 65.5% - **Count:** 49 - **Sum Total:** 539 | Category | % | Count | Total | Max | |---|---:|---:|---:|---:| @@ -70,7 +169,29 @@ | advanced | 50.4% | 8 | 58 | 115 | | compatibility | 47.6% | 8 | 50 | 105 | -### 6) hf.co/unsloth/gemma-3n-E2B-it-GGUF:Q8_0 +### 14) hf.co/mradermacher/aquif-3.5-3B-GGUF:F16 +- **Aggregated:** 60.6% - **Count:** 49 - **Sum Total:** 499 +| Category | % | Count | Total | Max | +|---|---:|---:|---:|---:| +| advanced | 85.2% | 8 | 98 | 115 | +| compatibility | 60.0% | 8 | 63 | 105 | +| syntax | 63.2% | 8 | 60 | 95 | +| types | 63.1% | 11 | 125 | 198 | +| runtimes | 50.5% | 6 | 48 | 95 | +| internals | 48.8% | 8 | 105 | 215 | + +### 15) hf.co/unsloth/gemma-3-12b-it-GGUF:Q4_K_M +- **Aggregated:** 56.3% - **Count:** 49 - **Sum Total:** 463 +| Category | % | Count | Total | Max | +|---|---:|---:|---:|---:| +| syntax | 88.4% | 8 | 84 | 95 | +| types | 61.1% | 11 | 121 | 198 | +| internals | 60.5% | 8 | 130 | 215 | +| advanced | 53.0% | 8 | 61 | 115 | +| compatibility | 40.0% | 8 | 42 | 105 | +| runtimes | 26.3% | 6 | 25 | 95 | + +### 16) hf.co/unsloth/gemma-3n-E2B-it-GGUF:Q8_0 - **Aggregated:** 49.9% - **Count:** 49 - **Sum Total:** 411 | Category | % | Count | Total | Max | |---|---:|---:|---:|---:| @@ -81,7 +202,7 @@ | compatibility | 44.8% | 8 | 47 | 105 | | runtimes | 40.0% | 6 | 38 | 95 | -### 7) hf.co/unsloth/Qwen3-0.6B-GGUF:BF16 +### 17) hf.co/unsloth/Qwen3-0.6B-GGUF:BF16 - **Aggregated:** 45.8% - **Count:** 49 - **Sum Total:** 377 | Category | % | Count | Total | Max | |---|---:|---:|---:|---:| @@ -92,7 +213,7 @@ | runtimes | 36.8% | 6 | 35 | 95 | | syntax | 34.7% | 8 | 33 | 95 | -### 8) hf.co/unsloth/gemma-3-4b-it-GGUF:Q8_0 +### 18) hf.co/unsloth/gemma-3-4b-it-GGUF:Q8_0 - **Aggregated:** 44.2% - **Count:** 49 - **Sum Total:** 364 | Category | % | Count | Total | Max | |---|---:|---:|---:|---:| @@ -103,7 +224,7 @@ | compatibility | 31.4% | 8 | 33 | 105 | | runtimes | 26.3% | 6 | 25 | 95 | -### 9) hf.co/bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0 +### 19) hf.co/bartowski/Llama-3.2-3B-Instruct-GGUF:Q8_0 - **Aggregated:** 42.2% - **Count:** 49 - **Sum Total:** 347 | Category | % | Count | Total | Max | |---|---:|---:|---:|---:| @@ -111,10 +232,10 @@ | syntax | 46.3% | 8 | 44 | 95 | | internals | 40.5% | 8 | 87 | 215 | | runtimes | 37.9% | 6 | 36 | 95 | -| types | 35.4% | 11 | 70 | 198 | +| types | 35. suspension% | 11 | 70 | 198 | | compatibility | 33.3% | 8 | 35 | 105 | -### 10) hf.co/unsloth/gemma-3-1b-it-GGUF:BF16 +### 20) hf.co/unsloth/gemma-3-1b-it-GGUF:BF16 - **Aggregated:** 35.0% - **Count:** 49 - **Sum Total:** 288 | Category | % | Count | Total | Max | |---|---:|---:|---:|---:| @@ -123,4 +244,15 @@ | internals | 37.2% | 8 | 80 | 215 | | types | 33.3% | 11 | 66 | 198 | | syntax | 31.6% | 8 | 30 | 95 | -| compatibility | 23.8% | 8 | 25 | 105 | \ No newline at end of file +| compatibility | 23.8% | 8 | 25 | 105 | + +### 21) hf.co/mradermacher/Gemma-3-1B-Roblox-Luau-GGUF:F16 +- **Aggregated:** 19.7% - **Count:** 49 - **Sum Total:** 162 +| Category | % | Count | Total | Max | +|---|---:|---:|---:|---:| +| advanced | 38.3% | 8 | 44 | 115 | +| syntax | 26.3% | 8 | 25 | 95 | +| runtimes | 21.1% | 6 | 20 | 95 | +| compatibility | 19.0% | 8 | 20 | 105 | +| types | 15.7% | 11 | 31 | 198 | +| internals | 10.2% | 8 | 22 | 215 | \ No newline at end of file