harald/nixcfg
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix(halo): only one model

Browse source
This commit is contained in:
Harald Hoyer 2026-05-20 14:23:20 +02:00
parent 5ee2f65337
commit 6c5ce8742c
2 changed files with 2 additions and 2 deletions
Download patch file Download diff file Expand all files Collapse all files

2
systems/x86_64-linux/halo/llama-server.nix
View file

@ -29,7 +29,7 @@
"--host 0.0.0.0" "--host 0.0.0.0"
"--port 8000" "--port 8000"
"--models-preset ${./models.ini}" "--models-preset ${./models.ini}"
"--models-max 2" "--models-max 1"
]; ];
Restart = "on-failure"; Restart = "on-failure";
RestartSec = 10; RestartSec = 10;

2
systems/x86_64-linux/halo/models.ini
View file

@ -11,7 +11,6 @@ cache-type-k = bf16
cache-type-v = bf16 cache-type-v = bf16
mmap = false mmap = false
no-context-shift = true no-context-shift = true
chat-template-kwargs = {"preserve_thinking": true}
fit = on fit = on
c = 131072 c = 131072
@ -27,3 +26,4 @@ min-p = 0.0
parallel = 2 parallel = 2
c = 524288 c = 524288
load-on-startup = true load-on-startup = true
chat-template-kwargs = {"preserve_thinking": true}