refactor(tdx_google): simplify service configurations

- Replaced hardcoded metadata-fetching logic with shared metadata service.
- Removed custom pre-start scripts and refactored environment handling.
- Updated Vector configuration to include custom field transformations.
- Streamlined container startup process and ensured proper cleanup.

Signed-off-by: Harald Hoyer <harald@matterlabs.dev>

packages/tdx_google/configuration.nix

@@ -6,6 +6,9 @@
   imports = [
     "${toString modulesPath}/profiles/minimal.nix"
     "${toString modulesPath}/profiles/qemu-guest.nix"
+    ./metadata.nix
+    ./vector.nix
+    ./container.nix
   ];
 
   services.journald.console = "/dev/ttyS0";

packages/tdx_google/container.nix

@@ -8,46 +8,29 @@
   systemd.services.docker_start_container = {
     description = "The main application container";
     wantedBy = [ "multi-user.target" ];
-    after = [ "network-online.target" "docker.service" "vector.service" "chronyd.service" ];
+    after = [ "network-online.target" "docker.service" "vector.service" "chronyd.service" "metadata.service" ];
-    requires = [ "network-online.target" "docker.service" "vector.service" ];
+    requires = [ "network-online.target" "docker.service" "vector.service" "metadata.service" ];
     serviceConfig = {
       Type = "exec";
       User = "root";
-      EnvironmentFile = "-/run/container/env";
+      EnvironmentFile = "-/run/env/env";
-      ExecStartPre = "+" + toString (
-        pkgs.writeShellScript "container-start-pre" ''
-          set -eu -o pipefail
-          : "''${CONTAINER_IMAGE:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_image" -H "Metadata-Flavor: Google")}"
-          : "''${CONTAINER_HUB:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_hub" -H "Metadata-Flavor: Google")}"
-          : "''${CONTAINER_USER:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_user" -H "Metadata-Flavor: Google")}"
-          : "''${CONTAINER_TOKEN:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_token" -H "Metadata-Flavor: Google")}"
-
-          : "''${CONTAINER_IMAGE:?Error: Missing CONTAINER_IMAGE}"
-          : "''${CONTAINER_HUB:?Error: Missing CONTAINER_HUB}"
-
-          mkdir -p /run/container
-          cat >/run/container/env <<EOF
-          CONTAINER_IMAGE="''${CONTAINER_IMAGE}"
-          CONTAINER_HUB="''${CONTAINER_HUB}"
-          CONTAINER_USER="''${CONTAINER_USER}"
-          CONTAINER_TOKEN="''${CONTAINER_TOKEN}"
-          EOF
-        ''
-      );
     };
-    path = [ pkgs.curl pkgs.docker pkgs.teepot.teepot.tdx_extend pkgs.coreutils ];
+    path = [ pkgs.docker pkgs.teepot.teepot.tdx_extend ];
     script = ''
       set -eu -o pipefail
-      if [[ $CONTAINER_USER ]] && [[ $CONTAINER_TOKEN ]]; then
-        docker login -u "$CONTAINER_USER" -p "$CONTAINER_TOKEN" "$CONTAINER_HUB"
-      fi
 
-      docker pull "''${CONTAINER_HUB}/''${CONTAINER_IMAGE}"
+      DIGEST=''${CONTAINER_DIGEST#sha256:}
-      DIGEST=$(docker inspect --format '{{.Id}}' "''${CONTAINER_HUB}/''${CONTAINER_IMAGE}")
-      DIGEST=''${DIGEST#sha256:}
       echo "Measuring $DIGEST" >&2
       test -c /dev/tdx_guest && tdx-extend --digest "$DIGEST" --rtmr 3
-      exec docker run --env "GOOGLE_METADATA=1" --network=host --init --privileged "sha256:$DIGEST"
+
+      docker run -d --rm \
+        --name tdx_container \
+        --env "GOOGLE_METADATA=1" \
+        --network=host \
+        --init \
+        --privileged \
+        "sha256:$DIGEST"
+      exec docker wait tdx_container
     '';
 
     postStop = lib.mkDefault ''

packages/tdx_google/metadata.nix

@@ -0,0 +1,53 @@
+{ lib
+, modulesPath
+, pkgs
+, ...
+}: {
+  systemd.services.metadata = {
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+    };
+    enable = true;
+    path = [ pkgs.curl pkgs.docker pkgs.teepot.teepot.tdx_extend pkgs.coreutils ];
+    wantedBy = [ "default.target" ];
+    after = [ "network-online.target" "docker.service" ];
+    requires = [ "network-online.target" "docker.service" ];
+    script = ''
+      set -eu -o pipefail
+      : "''${CONTAINER_HUB:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_hub" -H "Metadata-Flavor: Google")}"
+      : "''${CONTAINER_IMAGE:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_image" -H "Metadata-Flavor: Google")}"
+      : "''${CONTAINER_TOKEN:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_token" -H "Metadata-Flavor: Google")}"
+      : "''${CONTAINER_USER:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_user" -H "Metadata-Flavor: Google")}"
+      : "''${HOST_ID:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/id" -H "Metadata-Flavor: Google")}"
+      : "''${HOST_IMAGE:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/image" -H "Metadata-Flavor: Google")}"
+      : "''${HOST_NAME:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/hostname" -H "Metadata-Flavor: Google")}"
+      : "''${KAFKA_TOPIC:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/kafka_topic" -H "Metadata-Flavor: Google")}"
+      : "''${KAFKA_URLS:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/kafka_urls" -H "Metadata-Flavor: Google")}"
+
+      : "''${CONTAINER_IMAGE:?Error: Missing CONTAINER_IMAGE}"
+      : "''${CONTAINER_HUB:?Error: Missing CONTAINER_HUB}"
+
+      if [[ $CONTAINER_USER ]] && [[ $CONTAINER_TOKEN ]]; then
+        docker login -u "$CONTAINER_USER" -p "$CONTAINER_TOKEN" "$CONTAINER_HUB"
+      fi
+
+      docker pull "''${CONTAINER_HUB}/''${CONTAINER_IMAGE}"
+      CONTAINER_DIGEST=$(docker inspect --format '{{.Id}}' "''${CONTAINER_HUB}/''${CONTAINER_IMAGE}")
+
+      mkdir -p /run/env
+      cat >/run/env/env <<EOF
+      CONTAINER_HUB="''${CONTAINER_HUB}"
+      CONTAINER_IMAGE="''${CONTAINER_IMAGE}"
+      CONTAINER_TOKEN="''${CONTAINER_TOKEN}"
+      CONTAINER_USER="''${CONTAINER_USER}"
+      CONTAINER_DIGEST="''${CONTAINER_DIGEST}"
+      HOST_ID="''${HOST_ID}"
+      HOST_IMAGE="''${HOST_IMAGE}"
+      HOST_NAME="''${HOST_NAME}"
+      KAFKA_TOPIC="''${KAFKA_TOPIC}"
+      KAFKA_URLS="''${KAFKA_URLS}"
+      EOF
+    '';
+  };
+}

packages/tdx_google/vector.nix

@@ -19,16 +19,36 @@
         };
       };
     };
+    transforms = {
+      add_custom_fields = {
+        type = "remap";
+        inputs = [ "otlp.logs" ];
+        source = ''
+          # Create resources if it doesn't exist
+          if !exists(.resources) {
+              .resources = {}
+          }
+          # https://opentelemetry.io/docs/specs/semconv/resource/host/
+          .resources.host.name            = "''${HOST_NAME:-hostname}"
+          .resources.host.id              = "''${HOST_ID:-hostid}"
+          .resources.host.image.name      = "''${HOST_IMAGE:-host_image}"
+
+          # https://opentelemetry.io/docs/specs/semconv/resource/container/
+          .resources.container.image.name = "''${CONTAINER_HUB:-container_hub}/''${CONTAINER_IMAGE:-container_image}"
+          .resources.container.image.id   = "''${CONTAINER_DIGEST:-container_digest}"
+        '';
+      };
+    };
     sinks = {
       console = {
-        inputs = [ "otlp.logs" ];
+        inputs = [ "add_custom_fields" ];
         target = "stdout";
         type = "console";
         encoding = { codec = "json"; };
       };
       kafka = {
         type = "kafka";
-        inputs = [ "otlp.logs" ];
+        inputs = [ "add_custom_fields" ];
         bootstrap_servers = "\${KAFKA_URLS:-127.0.0.1:0}";
         topic = "\${KAFKA_TOPIC:-tdx-google}";
         encoding = {
@@ -38,23 +58,10 @@
       };
     };
   };
-  systemd.services.vector.path = [ pkgs.curl pkgs.coreutils ];
+  systemd.services.vector = {
-  # `-` means, that the file can be missing, so that `ExecStartPre` can execute and create it
+    after = [ "network-online.target" "metadata.service" ];
-  systemd.services.vector.serviceConfig.EnvironmentFile = "-/run/vector/env";
+    requires = [ "network-online.target" "metadata.service" ];
-  # `+` means, that the process has access to all files, to be able to write to `/run`
+    path = [ pkgs.curl pkgs.coreutils ];
-  systemd.services.vector.serviceConfig.ExecStartPre = "+" + toString (
+    serviceConfig.EnvironmentFile = "-/run/env/env";
-    pkgs.writeShellScript "vector-start-pre" ''
+  };
-      set -eu -o pipefail
-      : "''${KAFKA_URLS:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/kafka_urls" -H "Metadata-Flavor: Google")}"
-      : "''${KAFKA_TOPIC:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/kafka_topic" -H "Metadata-Flavor: Google")}"
-
-      KAFKA_TOPIC="''${KAFKA_TOPIC:-tdx-google}"
-
-      mkdir -p /run/vector
-      cat >/run/vector/env <<EOF
-      KAFKA_URLS="''${KAFKA_URLS}"
-      KAFKA_TOPIC="''${KAFKA_TOPIC}"
-      EOF
-    ''
-  );
 }