diff --git a/systems/x86_64-linux/halo/models.ini b/systems/x86_64-linux/halo/models.ini index 9d23606..83b160b 100644 --- a/systems/x86_64-linux/halo/models.ini +++ b/systems/x86_64-linux/halo/models.ini @@ -2,6 +2,7 @@ version = 1 [*] flash-attn = on +parallel = 1 jinja = true n-gpu-layers = 99 threads = 8 @@ -20,13 +21,15 @@ spec-type = draft-mtp [Qwen3.6-35B-A3B] hf = unsloth/Qwen3.6-35B-A3B-MTP-GGUF:UD-Q8_K_XL -spec-draft-n-max = 3 +spec-draft-n-max = 2 parallel = 1 -c = 262144 +c = 131072 +load-on-startup = true [Qwen3.6-27B] hf = unsloth/Qwen3.6-27B-MTP-GGUF:Q6_K spec-draft-n-max = 6 parallel = 2 c = 524288 +load-on-startup = true