harald/nixcfg
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat(halo): activate qwen 27b

Browse source
This commit is contained in:
Harald Hoyer 2026-05-10 20:43:33 +02:00
parent bef528e26a
commit 689cdec28d
2 changed files with 3 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

2
systems/x86_64-linux/halo/default.nix
View file

@ -10,7 +10,7 @@ with lib.metacfg;
./hardware-configuration.nix ./hardware-configuration.nix
#./xremap.nix #./xremap.nix
./wyoming.nix ./wyoming.nix
./llama-server.nix ./llama-server-27B.nix
]; ];
boot.lanzaboote.pkiBundle = "/var/lib/sbctl"; boot.lanzaboote.pkiBundle = "/var/lib/sbctl";

4
systems/x86_64-linux/halo/llama-server.nix
View file

@ -29,7 +29,7 @@
ExecStart = lib.concatStringsSep " " [ ExecStart = lib.concatStringsSep " " [
"${pkgs.llama-cpp-rocm}/bin/llama-server" "${pkgs.llama-cpp-rocm}/bin/llama-server"
"--flash-attn on" "--flash-attn on"
"--parallel 2" "--parallel 1"
"--jinja" "--jinja"
"--host 0.0.0.0" "--host 0.0.0.0"
"--port 8000" "--port 8000"
@ -43,7 +43,7 @@
"--temp 0.6 --top-p 0.95 --top-k 20 --min-p 0.00" "--temp 0.6 --top-p 0.95 --top-k 20 --min-p 0.00"
"--no-context-shift" "--no-context-shift"
''--chat-template-kwargs '{"preserve_thinking": true}' '' ''--chat-template-kwargs '{"preserve_thinking": true}' ''
"-c 524288" "-c 262144"
"--fit on" "--fit on"
"--slot-save-path %C/llama-server/kv-slots" "--slot-save-path %C/llama-server/kv-slots"
]; ];