harald/nixcfg
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

chore(halo): max models

Browse source
This commit is contained in:
Harald Hoyer 2026-05-22 10:37:00 +02:00
parent c845f7c385
commit 87dfe74daa
2 changed files with 2 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

2
systems/x86_64-linux/halo/llama-server.nix
View file

@ -29,7 +29,7 @@
"--host 0.0.0.0" "--host 0.0.0.0"
"--port 8000" "--port 8000"
"--models-preset ${./models.ini}" "--models-preset ${./models.ini}"
"--models-max 2" "--models-max 3"
]; ];
Restart = "on-failure"; Restart = "on-failure";
RestartSec = 10; RestartSec = 10;

1
systems/x86_64-linux/halo/models.ini
View file

@ -54,5 +54,6 @@ min-p = 0.0
presence-penalty = 1.5 presence-penalty = 1.5
parallel = 1 parallel = 1
c = 131072 c = 131072
load-on-startup = true
chat-template-kwargs = {"preserve_thinking": true} chat-template-kwargs = {"preserve_thinking": true}