From 5d2ad57cfdd97ea9b26e223be48f6e2dce97829c Mon Sep 17 00:00:00 2001 From: Harald Hoyer Date: Wed, 19 Feb 2025 10:57:46 +0100 Subject: [PATCH 1/3] refactor(tdx_google): modularize tdx_google configuration - Split `tdx_google/configuration.nix` into smaller modules: `vector.nix`, and `container.nix`. - Simplified the main configuration by leveraging modular imports for better readability and maintainability. Signed-off-by: Harald Hoyer # Conflicts: # packages/tdx_google/configuration.nix --- packages/tdx_google/configuration.nix | 107 -------------------------- packages/tdx_google/container.nix | 57 ++++++++++++++ packages/tdx_google/vector.nix | 60 +++++++++++++++ 3 files changed, 117 insertions(+), 107 deletions(-) create mode 100644 packages/tdx_google/container.nix create mode 100644 packages/tdx_google/vector.nix diff --git a/packages/tdx_google/configuration.nix b/packages/tdx_google/configuration.nix index 89973a5..cba2e4f 100644 --- a/packages/tdx_google/configuration.nix +++ b/packages/tdx_google/configuration.nix @@ -8,61 +8,6 @@ "${toString modulesPath}/profiles/qemu-guest.nix" ]; - services.vector.enable = true; - services.vector.settings = { - api.enabled = false; - sources = { - otlp = { - type = "opentelemetry"; - grpc = { address = "127.0.0.1:4317"; }; - http = { - address = "127.0.0.1:4318"; - keepalive = { - max_connection_age_jitter_factor = 0.1; - max_connection_age_secs = 300; - }; - }; - }; - }; - sinks = { - console = { - inputs = [ "otlp.logs" ]; - target = "stdout"; - type = "console"; - encoding = { codec = "json"; }; - }; - kafka = { - type = "kafka"; - inputs = [ "otlp.logs" ]; - bootstrap_servers = "\${KAFKA_URLS:-127.0.0.1:0}"; - topic = "\${KAFKA_TOPIC:-tdx-google}"; - encoding = { - codec = "json"; - compression = "lz4"; - }; - }; - }; - }; - systemd.services.vector.path = [ pkgs.curl pkgs.coreutils ]; - # `-` means, that the file can be missing, so that `ExecStartPre` can execute and create it - systemd.services.vector.serviceConfig.EnvironmentFile = "-/run/vector/env"; - # `+` means, that the process has access to all files, to be able to write to `/run` - systemd.services.vector.serviceConfig.ExecStartPre = "+" + toString ( - pkgs.writeShellScript "vector-start-pre" '' - set -eu -o pipefail - : "''${KAFKA_URLS:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/kafka_urls" -H "Metadata-Flavor: Google")}" - : "''${KAFKA_TOPIC:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/kafka_topic" -H "Metadata-Flavor: Google")}" - - KAFKA_TOPIC="''${KAFKA_TOPIC:-tdx-google}" - - mkdir -p /run/vector - cat >/run/vector/env </run/container/env <&2 - test -c /dev/tdx_guest && tdx-extend --digest "$DIGEST" --rtmr 3 - exec docker run --env "GOOGLE_METADATA=1" --network=host --init --privileged "sha256:$DIGEST" - ''; - - postStop = lib.mkDefault '' - shutdown --reboot +5 - ''; - }; - services.prometheus.exporters.node = { enable = true; port = 9100; diff --git a/packages/tdx_google/container.nix b/packages/tdx_google/container.nix new file mode 100644 index 0000000..19ced88 --- /dev/null +++ b/packages/tdx_google/container.nix @@ -0,0 +1,57 @@ +{ lib +, modulesPath +, pkgs +, ... +}: { + virtualisation.docker.enable = true; + + systemd.services.docker_start_container = { + description = "The main application container"; + wantedBy = [ "multi-user.target" ]; + after = [ "network-online.target" "docker.service" "vector.service" "chronyd.service" ]; + requires = [ "network-online.target" "docker.service" "vector.service" ]; + serviceConfig = { + Type = "exec"; + User = "root"; + EnvironmentFile = "-/run/container/env"; + ExecStartPre = "+" + toString ( + pkgs.writeShellScript "container-start-pre" '' + set -eu -o pipefail + : "''${CONTAINER_IMAGE:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_image" -H "Metadata-Flavor: Google")}" + : "''${CONTAINER_HUB:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_hub" -H "Metadata-Flavor: Google")}" + : "''${CONTAINER_USER:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_user" -H "Metadata-Flavor: Google")}" + : "''${CONTAINER_TOKEN:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_token" -H "Metadata-Flavor: Google")}" + + : "''${CONTAINER_IMAGE:?Error: Missing CONTAINER_IMAGE}" + : "''${CONTAINER_HUB:?Error: Missing CONTAINER_HUB}" + + mkdir -p /run/container + cat >/run/container/env <&2 + test -c /dev/tdx_guest && tdx-extend --digest "$DIGEST" --rtmr 3 + exec docker run --env "GOOGLE_METADATA=1" --network=host --init --privileged "sha256:$DIGEST" + ''; + + postStop = lib.mkDefault '' + shutdown --reboot +5 + ''; + }; +} diff --git a/packages/tdx_google/vector.nix b/packages/tdx_google/vector.nix new file mode 100644 index 0000000..63ada3c --- /dev/null +++ b/packages/tdx_google/vector.nix @@ -0,0 +1,60 @@ +{ lib +, modulesPath +, pkgs +, ... +}: { + services.vector.enable = true; + services.vector.settings = { + api.enabled = false; + sources = { + otlp = { + type = "opentelemetry"; + grpc = { address = "127.0.0.1:4317"; }; + http = { + address = "127.0.0.1:4318"; + keepalive = { + max_connection_age_jitter_factor = 0.1; + max_connection_age_secs = 300; + }; + }; + }; + }; + sinks = { + console = { + inputs = [ "otlp.logs" ]; + target = "stdout"; + type = "console"; + encoding = { codec = "json"; }; + }; + kafka = { + type = "kafka"; + inputs = [ "otlp.logs" ]; + bootstrap_servers = "\${KAFKA_URLS:-127.0.0.1:0}"; + topic = "\${KAFKA_TOPIC:-tdx-google}"; + encoding = { + codec = "json"; + compression = "lz4"; + }; + }; + }; + }; + systemd.services.vector.path = [ pkgs.curl pkgs.coreutils ]; + # `-` means, that the file can be missing, so that `ExecStartPre` can execute and create it + systemd.services.vector.serviceConfig.EnvironmentFile = "-/run/vector/env"; + # `+` means, that the process has access to all files, to be able to write to `/run` + systemd.services.vector.serviceConfig.ExecStartPre = "+" + toString ( + pkgs.writeShellScript "vector-start-pre" '' + set -eu -o pipefail + : "''${KAFKA_URLS:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/kafka_urls" -H "Metadata-Flavor: Google")}" + : "''${KAFKA_TOPIC:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/kafka_topic" -H "Metadata-Flavor: Google")}" + + KAFKA_TOPIC="''${KAFKA_TOPIC:-tdx-google}" + + mkdir -p /run/vector + cat >/run/vector/env < Date: Wed, 19 Feb 2025 11:01:25 +0100 Subject: [PATCH 2/3] refactor(tdx_google): simplify service configurations - Replaced hardcoded metadata-fetching logic with shared metadata service. - Removed custom pre-start scripts and refactored environment handling. - Updated Vector configuration to include custom field transformations. - Streamlined container startup process and ensured proper cleanup. Signed-off-by: Harald Hoyer --- packages/tdx_google/configuration.nix | 3 ++ packages/tdx_google/container.nix | 45 +++++++---------------- packages/tdx_google/metadata.nix | 53 +++++++++++++++++++++++++++ packages/tdx_google/vector.nix | 49 ++++++++++++++----------- 4 files changed, 98 insertions(+), 52 deletions(-) create mode 100644 packages/tdx_google/metadata.nix diff --git a/packages/tdx_google/configuration.nix b/packages/tdx_google/configuration.nix index cba2e4f..0fee234 100644 --- a/packages/tdx_google/configuration.nix +++ b/packages/tdx_google/configuration.nix @@ -6,6 +6,9 @@ imports = [ "${toString modulesPath}/profiles/minimal.nix" "${toString modulesPath}/profiles/qemu-guest.nix" + ./metadata.nix + ./vector.nix + ./container.nix ]; services.journald.console = "/dev/ttyS0"; diff --git a/packages/tdx_google/container.nix b/packages/tdx_google/container.nix index 19ced88..07be8df 100644 --- a/packages/tdx_google/container.nix +++ b/packages/tdx_google/container.nix @@ -8,46 +8,29 @@ systemd.services.docker_start_container = { description = "The main application container"; wantedBy = [ "multi-user.target" ]; - after = [ "network-online.target" "docker.service" "vector.service" "chronyd.service" ]; - requires = [ "network-online.target" "docker.service" "vector.service" ]; + after = [ "network-online.target" "docker.service" "vector.service" "chronyd.service" "metadata.service" ]; + requires = [ "network-online.target" "docker.service" "vector.service" "metadata.service" ]; serviceConfig = { Type = "exec"; User = "root"; - EnvironmentFile = "-/run/container/env"; - ExecStartPre = "+" + toString ( - pkgs.writeShellScript "container-start-pre" '' - set -eu -o pipefail - : "''${CONTAINER_IMAGE:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_image" -H "Metadata-Flavor: Google")}" - : "''${CONTAINER_HUB:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_hub" -H "Metadata-Flavor: Google")}" - : "''${CONTAINER_USER:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_user" -H "Metadata-Flavor: Google")}" - : "''${CONTAINER_TOKEN:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_token" -H "Metadata-Flavor: Google")}" - - : "''${CONTAINER_IMAGE:?Error: Missing CONTAINER_IMAGE}" - : "''${CONTAINER_HUB:?Error: Missing CONTAINER_HUB}" - - mkdir -p /run/container - cat >/run/container/env <&2 test -c /dev/tdx_guest && tdx-extend --digest "$DIGEST" --rtmr 3 - exec docker run --env "GOOGLE_METADATA=1" --network=host --init --privileged "sha256:$DIGEST" + + docker run -d --rm \ + --name tdx_container \ + --env "GOOGLE_METADATA=1" \ + --network=host \ + --init \ + --privileged \ + "sha256:$DIGEST" + exec docker wait tdx_container ''; postStop = lib.mkDefault '' diff --git a/packages/tdx_google/metadata.nix b/packages/tdx_google/metadata.nix new file mode 100644 index 0000000..98be396 --- /dev/null +++ b/packages/tdx_google/metadata.nix @@ -0,0 +1,53 @@ +{ lib +, modulesPath +, pkgs +, ... +}: { + systemd.services.metadata = { + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + }; + enable = true; + path = [ pkgs.curl pkgs.docker pkgs.teepot.teepot.tdx_extend pkgs.coreutils ]; + wantedBy = [ "default.target" ]; + after = [ "network-online.target" "docker.service" ]; + requires = [ "network-online.target" "docker.service" ]; + script = '' + set -eu -o pipefail + : "''${CONTAINER_HUB:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_hub" -H "Metadata-Flavor: Google")}" + : "''${CONTAINER_IMAGE:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_image" -H "Metadata-Flavor: Google")}" + : "''${CONTAINER_TOKEN:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_token" -H "Metadata-Flavor: Google")}" + : "''${CONTAINER_USER:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_user" -H "Metadata-Flavor: Google")}" + : "''${HOST_ID:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/id" -H "Metadata-Flavor: Google")}" + : "''${HOST_IMAGE:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/image" -H "Metadata-Flavor: Google")}" + : "''${HOST_NAME:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/hostname" -H "Metadata-Flavor: Google")}" + : "''${KAFKA_TOPIC:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/kafka_topic" -H "Metadata-Flavor: Google")}" + : "''${KAFKA_URLS:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/kafka_urls" -H "Metadata-Flavor: Google")}" + + : "''${CONTAINER_IMAGE:?Error: Missing CONTAINER_IMAGE}" + : "''${CONTAINER_HUB:?Error: Missing CONTAINER_HUB}" + + if [[ $CONTAINER_USER ]] && [[ $CONTAINER_TOKEN ]]; then + docker login -u "$CONTAINER_USER" -p "$CONTAINER_TOKEN" "$CONTAINER_HUB" + fi + + docker pull "''${CONTAINER_HUB}/''${CONTAINER_IMAGE}" + CONTAINER_DIGEST=$(docker inspect --format '{{.Id}}' "''${CONTAINER_HUB}/''${CONTAINER_IMAGE}") + + mkdir -p /run/env + cat >/run/env/env </run/vector/env < Date: Wed, 19 Feb 2025 11:09:13 +0100 Subject: [PATCH 3/3] chore(tdx_google): remove unused `teepot` package from system environment Signed-off-by: Harald Hoyer --- packages/tdx_google/configuration.nix | 4 ---- 1 file changed, 4 deletions(-) diff --git a/packages/tdx_google/configuration.nix b/packages/tdx_google/configuration.nix index 0fee234..1298769 100644 --- a/packages/tdx_google/configuration.nix +++ b/packages/tdx_google/configuration.nix @@ -43,10 +43,6 @@ ]; }; - environment.systemPackages = with pkgs; [ - teepot.teepot - ]; - # /var is on tmpfs anyway services.journald.storage = "volatile";