nixcfg/systems/x86_64-linux/sgx/firefly.nix
Harald Hoyer e0d2a2f50d feat(sgx): finish firefly-sparda-fetch — headless FinTS import
End-to-end verified: aqbanking-cli fetches Sparda Südwest transactions
via FinTS PIN/TAN + SecureGo+, exports CSV using a custom decimal-amount
profile, POSTs to firefly-iii-data-importer's autoupload endpoint, which
creates transactions in Firefly III via API.

Changes vs. previous WIP commit:
- firefly/access_token sops slot for the importer's Firefly III API auth
  (FIREFLY_III_ACCESS_TOKEN_FILE — was the missing piece causing 401s
  from the API after the autoupload secret authenticated)
- nginx fastcgi_read_timeout=600s on the importer vhost (prevents 504
  while PHP-FPM is still processing the batch)
- PHP-FPM max_execution_time=600s + memory_limit=512M on the importer
  pool (PHP's stock 30s aborts mid-import for batches > ~50 transactions)
- timer re-enabled, wantedBy=[timers.target]

Caveats baked into a code comment:
- Sparda online-banking PIN must be [A-Za-z0-9] only. aqbanking 6.8.2's
  -P pinfile mangles `:`, `+`, `'`, `?`, `@`, `%`, `*`; bank locks the
  access (3 soft / 9 hard strikes) on rejected attempts. Same applies
  whenever the sops secret is rotated.
- Bulk historical imports beyond the PSD2 90-day window need interactive
  SCA approval per ~30-day chunk and cannot run from the timer; the
  daily 35-day rolling window stays inside the no-SCA region.

Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
2026-05-01 18:58:45 +02:00

189 lines
6 KiB
Nix

{ config, pkgs, ... }:
let
domain = "firefly.hoyer.world";
importDomain = "firefly-import.hoyer.world";
importerHome = "/var/lib/firefly-iii-data-importer";
inbox = "${importerHome}/inbox";
configFile = "${importerHome}/sparda-config.json";
bankCode = "55090500";
userId = "5987838198";
giroAccountId = "3";
# aqbanking 6.8.2 ships only an "import" profile and a "full" export
# profile that renders amounts as fractions ("-499/100"). Firefly's CSV
# importer needs decimal amounts and benefits from localIban/remoteIban
# columns, so derive a profile that combines "full"'s columns with
# decimal value formatting.
fireflyCsvProfile = pkgs.runCommand "aqbanking-csv-firefly-profile" { } ''
sed 's/name="full"/name="firefly"/; s/valueFormat="rational"/valueFormat="float"/' \
${pkgs.aqbanking}/share/aqbanking/imexporters/csv/profiles/full.conf > $out
'';
vhostBase = {
enableACME = false;
useACMEHost = "internal.hoyer.world";
forceSSL = true;
};
in
{
sops.secrets = {
"firefly/app_key" = {
sopsFile = ../../../.secrets/sgx/firefly.yaml;
owner = "firefly-iii";
};
"firefly/sparda_pin" = {
sopsFile = ../../../.secrets/sgx/firefly.yaml;
owner = "firefly-iii-data-importer";
};
"firefly/auto_import_secret" = {
sopsFile = ../../../.secrets/sgx/firefly.yaml;
owner = "firefly-iii-data-importer";
};
"firefly/access_token" = {
sopsFile = ../../../.secrets/sgx/firefly.yaml;
owner = "firefly-iii-data-importer";
};
};
environment.systemPackages = [ pkgs.aqbanking ];
systemd = {
tmpfiles.rules = [
"d ${inbox} 0700 firefly-iii-data-importer nginx -"
"d ${importerHome}/.aqbanking/imexporters/csv/profiles 0700 firefly-iii-data-importer nginx -"
"L+ ${importerHome}/.aqbanking/imexporters/csv/profiles/firefly.conf - - - - ${fireflyCsvProfile}"
];
services.firefly-sparda-fetch = {
description = "Fetch Sparda transactions via FinTS and trigger Firefly auto-import";
after = [
"network-online.target"
"phpfpm-firefly-iii-data-importer.service"
];
wants = [ "network-online.target" ];
path = with pkgs; [
aqbanking
curl
coreutils
];
serviceConfig = {
Type = "oneshot";
User = "firefly-iii-data-importer";
Group = "nginx";
RuntimeDirectory = "firefly-sparda-fetch";
LoadCredential = [
"pin:${config.sops.secrets."firefly/sparda_pin".path}"
"secret:${config.sops.secrets."firefly/auto_import_secret".path}"
];
ProtectSystem = "strict";
ReadWritePaths = [ importerHome ];
ProtectHome = true;
PrivateTmp = true;
NoNewPrivileges = true;
TimeoutStartSec = "3min";
};
script = ''
set -euo pipefail
pinfile=$RUNTIME_DIRECTORY/pinfile
umask 077
printf 'PIN_%s_%s = "%s"\n' "${bankCode}" "${userId}" \
"$(<"$CREDENTIALS_DIRECTORY/pin")" >"$pinfile"
ts=$(date +%Y%m%d-%H%M%S)
ctx=$RUNTIME_DIRECTORY/ctx-$ts.aqb
out=${inbox}/sparda-$ts.csv
# Refresh SEPA account list Atruvia/Sparda rejects HKCAZ
# ("Mussfeld 9160") if this metadata isn't fresh in the dialog.
aqhbci-tool4 -n -A -P "$pinfile" getaccsepa -u ${giroAccountId}
fromdate=$(date --date='35 days ago' +%Y%m%d)
aqbanking-cli -n -A -P "$pinfile" request \
--transactions --fromdate="$fromdate" \
--aid=${giroAccountId} -c "$ctx"
aqbanking-cli export \
--exporter=csv --profile=firefly \
-c "$ctx" -o "$out"
secret=$(<"$CREDENTIALS_DIRECTORY/secret")
curl -fsS -X POST \
"https://${importDomain}/autoupload?secret=$secret" \
-F "json=@${configFile}" \
-F "importable=@$out"
'';
};
# Sparda online-banking PIN must contain only [A-Za-z0-9] — special
# chars (`:`, `+`, `'`, `?`, `@`, `%`, `*`) get mangled by aqbanking
# 6.8.2's pinfile path and the bank locks the access after a few
# rejected attempts (3 soft / 9 hard). Same applies if the secret in
# sops is rotated.
timers.firefly-sparda-fetch = {
wantedBy = [ "timers.target" ];
timerConfig = {
OnCalendar = "daily";
Persistent = true;
RandomizedDelaySec = "1h";
};
};
};
services = {
firefly-iii = {
enable = true;
enableNginx = true;
virtualHost = domain;
settings = {
APP_ENV = "production";
APP_KEY_FILE = config.sops.secrets."firefly/app_key".path;
SITE_OWNER = "harald.hoyer@gmail.com";
TZ = "Europe/Berlin";
DEFAULT_LANGUAGE = "de_DE";
DEFAULT_LOCALE = "de_DE";
TRUSTED_PROXIES = "**";
LOG_CHANNEL = "stack";
};
};
firefly-iii-data-importer = {
enable = true;
enableNginx = true;
virtualHost = importDomain;
settings = {
FIREFLY_III_URL = "https://${domain}";
VANITY_URL = "https://${importDomain}";
TZ = "Europe/Berlin";
CAN_POST_FILES = "true";
CAN_POST_AUTOIMPORT = "true";
IMPORT_DIR_ALLOWLIST = inbox;
AUTO_IMPORT_SECRET_FILE = config.sops.secrets."firefly/auto_import_secret".path;
FIREFLY_III_ACCESS_TOKEN_FILE = config.sops.secrets."firefly/access_token".path;
};
};
nginx.virtualHosts = {
${domain} = vhostBase;
# Importer's autoupload endpoint blocks until the entire batch
# finishes — POSTing 100+ transactions takes minutes. Default 60s
# fastcgi timeout makes nginx 504 even though PHP-FPM keeps going.
${importDomain} = vhostBase // {
extraConfig = ''
fastcgi_read_timeout 600s;
'';
};
};
# PHP's stock max_execution_time = 30s aborts large bulk imports
# mid-stream. Match the nginx fastcgi_read_timeout above.
phpfpm.pools.firefly-iii-data-importer.settings = {
"php_admin_value[max_execution_time]" = "600";
"php_admin_value[memory_limit]" = "512M";
};
};
}