nixcfg/systems/x86_64-linux/sgx/firefly.nix
Harald Hoyer 491a7b38e4 sgx/firefly: switch Firefly III backend from sqlite to postgres
SQLite was slow under btrfs CoW, and the no-CoW migration path turned
out to be fragile (WAL deletion without checkpoint = data loss). Move
to PostgreSQL on Unix-socket peer auth — no password needed for the
local-host setup, NixOS provisions the database+user declaratively.

Drop the now-unused +C tmpfiles rule on the sqlite directory; the
leftover database.sqlite* files at /var/lib/firefly-iii/storage/database/
are harmless and can be removed manually after switch is verified.

Migration of existing Firefly III data is not preserved by this
commit — fresh-start path: re-register admin, re-issue PAT, re-POST
the bulk CSV through the importer.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-01 21:49:08 +02:00

205 lines
6.4 KiB
Nix

{ config, pkgs, ... }:
let
domain = "firefly.hoyer.world";
importDomain = "firefly-import.hoyer.world";
importerHome = "/var/lib/firefly-iii-data-importer";
inbox = "${importerHome}/inbox";
configFile = "${importerHome}/sparda-config.json";
bankCode = "55090500";
userId = "5987838198";
giroAccountId = "3";
# aqbanking 6.8.2 ships only an "import" profile and a "full" export
# profile that renders amounts as fractions ("-499/100"). Firefly's CSV
# importer needs decimal amounts and benefits from localIban/remoteIban
# columns, so derive a profile that combines "full"'s columns with
# decimal value formatting.
fireflyCsvProfile = pkgs.runCommand "aqbanking-csv-firefly-profile" { } ''
sed 's/name="full"/name="firefly"/; s/valueFormat="rational"/valueFormat="float"/' \
${pkgs.aqbanking}/share/aqbanking/imexporters/csv/profiles/full.conf > $out
'';
vhostBase = {
enableACME = false;
useACMEHost = "internal.hoyer.world";
forceSSL = true;
};
in
{
sops.secrets = {
"firefly/app_key" = {
sopsFile = ../../../.secrets/sgx/firefly.yaml;
owner = "firefly-iii";
};
"firefly/sparda_pin" = {
sopsFile = ../../../.secrets/sgx/firefly.yaml;
owner = "firefly-iii-data-importer";
};
"firefly/auto_import_secret" = {
sopsFile = ../../../.secrets/sgx/firefly.yaml;
owner = "firefly-iii-data-importer";
};
"firefly/access_token" = {
sopsFile = ../../../.secrets/sgx/firefly.yaml;
owner = "firefly-iii-data-importer";
};
};
environment.systemPackages = [ pkgs.aqbanking ];
systemd = {
tmpfiles.rules = [
"d ${inbox} 0700 firefly-iii-data-importer nginx -"
"d ${importerHome}/.aqbanking/imexporters/csv/profiles 0700 firefly-iii-data-importer nginx -"
"L+ ${importerHome}/.aqbanking/imexporters/csv/profiles/firefly.conf - - - - ${fireflyCsvProfile}"
];
services.firefly-sparda-fetch = {
description = "Fetch Sparda transactions via FinTS and trigger Firefly auto-import";
after = [
"network-online.target"
"phpfpm-firefly-iii-data-importer.service"
];
wants = [ "network-online.target" ];
path = with pkgs; [
aqbanking
curl
coreutils
];
serviceConfig = {
Type = "oneshot";
User = "firefly-iii-data-importer";
Group = "nginx";
RuntimeDirectory = "firefly-sparda-fetch";
LoadCredential = [
"pin:${config.sops.secrets."firefly/sparda_pin".path}"
"secret:${config.sops.secrets."firefly/auto_import_secret".path}"
];
ProtectSystem = "strict";
ReadWritePaths = [ importerHome ];
ProtectHome = true;
PrivateTmp = true;
NoNewPrivileges = true;
TimeoutStartSec = "3min";
};
script = ''
set -euo pipefail
pinfile=$RUNTIME_DIRECTORY/pinfile
umask 077
printf 'PIN_%s_%s = "%s"\n' "${bankCode}" "${userId}" \
"$(<"$CREDENTIALS_DIRECTORY/pin")" >"$pinfile"
ts=$(date +%Y%m%d-%H%M%S)
ctx=$RUNTIME_DIRECTORY/ctx-$ts.aqb
out=${inbox}/sparda-$ts.csv
# Refresh SEPA account list Atruvia/Sparda rejects HKCAZ
# ("Mussfeld 9160") if this metadata isn't fresh in the dialog.
aqhbci-tool4 -n -A -P "$pinfile" getaccsepa -u ${giroAccountId}
fromdate=$(date --date='35 days ago' +%Y%m%d)
aqbanking-cli -n -A -P "$pinfile" request \
--transactions --fromdate="$fromdate" \
--aid=${giroAccountId} -c "$ctx"
aqbanking-cli export \
--exporter=csv --profile=firefly \
-c "$ctx" -o "$out"
secret=$(<"$CREDENTIALS_DIRECTORY/secret")
curl -fsS -X POST \
"https://${importDomain}/autoupload?secret=$secret" \
-F "json=@${configFile}" \
-F "importable=@$out"
'';
};
# Sparda online-banking PIN must contain only [A-Za-z0-9] — special
# chars (`:`, `+`, `'`, `?`, `@`, `%`, `*`) get mangled by aqbanking
# 6.8.2's pinfile path and the bank locks the access after a few
# rejected attempts (3 soft / 9 hard). Same applies if the secret in
# sops is rotated.
timers.firefly-sparda-fetch = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "daily";
Persistent = true;
RandomizedDelaySec = "1h";
};
};
};
services = {
postgresql = {
enable = true;
ensureDatabases = [ "firefly-iii" ];
ensureUsers = [
{
name = "firefly-iii";
ensureDBOwnership = true;
}
];
};
firefly-iii = {
enable = true;
enableNginx = true;
virtualHost = domain;
settings = {
APP_ENV = "production";
APP_KEY_FILE = config.sops.secrets."firefly/app_key".path;
SITE_OWNER = "harald.hoyer@gmail.com";
TZ = "Europe/Berlin";
DEFAULT_LANGUAGE = "de_DE";
DEFAULT_LOCALE = "de_DE";
TRUSTED_PROXIES = "**";
LOG_CHANNEL = "stack";
# PostgreSQL via Unix socket peer auth — no password needed.
DB_CONNECTION = "pgsql";
DB_HOST = "/run/postgresql";
DB_DATABASE = "firefly-iii";
DB_USERNAME = "firefly-iii";
};
};
firefly-iii-data-importer = {
enable = true;
enableNginx = true;
virtualHost = importDomain;
settings = {
FIREFLY_III_URL = "https://${domain}";
VANITY_URL = "https://${importDomain}";
TZ = "Europe/Berlin";
CAN_POST_FILES = "true";
CAN_POST_AUTOIMPORT = "true";
IMPORT_DIR_ALLOWLIST = inbox;
AUTO_IMPORT_SECRET_FILE = config.sops.secrets."firefly/auto_import_secret".path;
FIREFLY_III_ACCESS_TOKEN_FILE = config.sops.secrets."firefly/access_token".path;
};
};
nginx.virtualHosts = {
${domain} = vhostBase;
# Importer's autoupload endpoint blocks until the entire batch
# finishes — POSTing 100+ transactions takes minutes. Default 60s
# fastcgi timeout makes nginx 504 even though PHP-FPM keeps going.
${importDomain} = vhostBase // {
extraConfig = ''
fastcgi_read_timeout 600s;
'';
};
};
# PHP's stock max_execution_time = 30s aborts large bulk imports
# mid-stream. Match the nginx fastcgi_read_timeout above.
phpfpm.pools.firefly-iii-data-importer.settings = {
"php_admin_value[max_execution_time]" = "600";
"php_admin_value[memory_limit]" = "512M";
};
};
}