feat(tdx): add nix build for TDX google VMs

Signed-off-by: Harald Hoyer <harald@matterlabs.dev>
This commit is contained in:
Harald Hoyer 2025-01-08 08:59:23 +01:00
parent 8270c389e4
commit dc1e756ec6
Signed by: harald
GPG key ID: F519A1143B3FBE32
11 changed files with 638 additions and 16 deletions

View file

@ -1,13 +1,5 @@
# teepot # teepot
Key Value store in a TEE with Remote Attestation for Authentication
## Introduction
This project is a key-value store that runs in a Trusted Execution Environment (TEE) and uses Remote Attestation for
Authentication.
The key-value store is implemented using Hashicorp Vault running in an Intel SGX enclave via the Gramine runtime.
## Parts of this project ## Parts of this project
- `teepot`: The main rust crate that abstracts TEEs and key-value stores. - `teepot`: The main rust crate that abstracts TEEs and key-value stores.
@ -22,6 +14,18 @@ The key-value store is implemented using Hashicorp Vault running in an Intel SGX
- `verify-attestation`: A client utility that verifies the attestation of an enclave. - `verify-attestation`: A client utility that verifies the attestation of an enclave.
- `tee-key-preexec`: A pre-exec utility that generates a p256 secret key and passes it as an environment variable to the - `tee-key-preexec`: A pre-exec utility that generates a p256 secret key and passes it as an environment variable to the
enclave along with the attestation quote containing the hash of the public key. enclave along with the attestation quote containing the hash of the public key.
- `tdx_google`: A base VM running on Google Cloud TDX. It receives a container URL via the instance metadata,
measures the sha384 of the URL to RTMR3 and launches the container.
- `tdx-extend`: A utility to extend an RTMR register with a hash value.
- `rtmr-calc`: A utility to calculate RTMR1 and RTMR2 from a GPT disk, the linux kernel, the linux initrd
and a UKI (unified kernel image).
- `sha384-extend`: A utility to calculate RTMR registers after extending them with a digest.
## Vault
Part of this project is a key-value store that runs in a Trusted Execution Environment (TEE) and uses Remote Attestation
for Authentication. The key-value store is implemented using Hashicorp Vault running in an Intel SGX enclave via the
Gramine runtime.
## Development ## Development
@ -96,3 +100,9 @@ Attributes:
isv_svn: 0 isv_svn: 0
debug_enclave: False debug_enclave: False
``` ```
### TDX VM testing
```shell
nixos-rebuild -L --flake .#tdxtest build-vm && ./result/bin/run-tdxtest-vm
```

45
assets/gcloud-deploy.sh Executable file
View file

@ -0,0 +1,45 @@
#!/usr/bin/env bash
#
# SPDX-License-Identifier: Apache-2.0
# Copyright (c) 2025 Matter Labs
#
set -ex
NO=${NO:-1}
nix build -L .#tdx_google
gsutil cp result/tdx_base_1.vmdk gs://tdx_vms/
gcloud migration vms image-imports create \
--location=us-central1 \
--target-project=tdx-pilot \
--project=tdx-pilot \
--skip-os-adaptation \
--source-file=gs://tdx_vms/tdx_base_1.vmdk \
tdx-img-pre-"${NO}"
gcloud compute instances stop tdx-pilot --zone us-central1-c --project tdx-pilot || :
gcloud compute instances delete tdx-pilot --zone us-central1-c --project tdx-pilot || :
while gcloud migration vms image-imports list --location=us-central1 --project=tdx-pilot | grep -F RUNNING; do
sleep 1
done
gcloud compute images create \
--project tdx-pilot \
--guest-os-features=UEFI_COMPATIBLE,TDX_CAPABLE,GVNIC,VIRTIO_SCSI_MULTIQUEUE \
--storage-location=us-central1 \
--source-image=tdx-img-pre-"${NO}" \
tdx-img-f-"${NO}"
gcloud compute instances create tdx-pilot \
--machine-type c3-standard-4 --zone us-central1-c \
--confidential-compute-type=TDX \
--maintenance-policy=TERMINATE \
--image-project=tdx-pilot \
--project tdx-pilot \
--metadata=container_hub="docker.io",container_image="amd64/hello-world@sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57" \
--image tdx-img-f-"${NO}"

View file

@ -25,7 +25,9 @@
}; };
outputs = inputs: outputs = inputs:
let src = ./.; in let
src = ./.;
in
inputs.snowfall-lib.mkFlake { inputs.snowfall-lib.mkFlake {
inherit inputs; inherit inputs;
inherit src; inherit src;

3
lib/default.nix Normal file
View file

@ -0,0 +1,3 @@
{ ... }: {
nixosGenerate = import ./nixos-generate.nix;
}

33
lib/nixos-generate.nix Normal file
View file

@ -0,0 +1,33 @@
{ pkgs
, nixosSystem
, formatModule
, system
, specialArgs ? { }
, modules ? [ ]
}:
let
image = nixosSystem {
inherit pkgs specialArgs;
modules =
[
formatModule
(
{ lib, ... }: {
options = {
fileExtension = lib.mkOption {
type = lib.types.str;
description = "Declare the path of the wanted file in the output directory";
default = "";
};
formatAttr = lib.mkOption {
type = lib.types.str;
description = "Declare the default attribute to build";
};
};
}
)
]
++ modules;
};
in
image.config.system.build.${image.config.formatAttr}

View file

@ -0,0 +1,180 @@
{ lib
, modulesPath
, pkgs
, ...
}: {
imports = [
"${toString modulesPath}/profiles/minimal.nix"
"${toString modulesPath}/profiles/qemu-guest.nix"
];
/*
# SSH login for debugging
services.sshd.enable = true;
networking.firewall.allowedTCPPorts = [ 22 ];
services.openssh.settings.PermitRootLogin = lib.mkOverride 999 "yes";
users.users.root.openssh.authorizedKeys.keys = [
"sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIDsb/Tr69YN5MQLweWPuJaRGm+h2kOyxfD6sqKEDTIwoAAAABHNzaDo="
"sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBACLgT81iB1iWWVuXq6PdQ5GAAGhaZhSKnveQCvcNnAOZ5WKH80bZShKHyAYzrzbp8IGwLWJcZQ7TqRK+qZdfagAAAAEc3NoOg=="
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAYbUTKpy4QR3s944/hjJ1UK05asFEs/SmWeUbtS0cdA660sT4xHnRfals73FicOoz+uIucJCwn/SCM804j+wtM="
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMNsmP15vH8BVKo7bdvIiiEjiQboPGcRPqJK0+bH4jKD"
];
*/
# the container might want to listen on ports
networking.firewall.enable = true;
networking.firewall.allowedTCPPortRanges = [{ from = 1024; to = 65535; }];
networking.firewall.allowedUDPPortRanges = [{ from = 1024; to = 65535; }];
networking.useNetworkd = lib.mkDefault true;
# don't fill up the logs
networking.firewall.logRefusedConnections = false;
virtualisation.docker.enable = true;
systemd.services.docker_start_container = {
description = "The main application container";
wantedBy = [ "multi-user.target" ];
after = [ "network-online.target" "docker.service" ];
requires = [ "network-online.target" "docker.service" ];
serviceConfig = {
Type = "exec";
User = "root";
};
path = [ pkgs.curl pkgs.docker pkgs.teepot.teepot.tdx_extend pkgs.coreutils ];
script = ''
set -eu -o pipefail
: "''${CONTAINER_IMAGE:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_image" -H "Metadata-Flavor: Google")}"
: "''${CONTAINER_HUB:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_hub" -H "Metadata-Flavor: Google")}"
: "''${CONTAINER_USER:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_user" -H "Metadata-Flavor: Google")}"
: "''${CONTAINER_TOKEN:=$(curl --silent --fail "http://metadata.google.internal/computeMetadata/v1/instance/attributes/container_token" -H "Metadata-Flavor: Google")}"
: "''${CONTAINER_IMAGE:?Error: Missing CONTAINER_IMAGE}"
: "''${CONTAINER_HUB:?Error: Missing CONTAINER_HUB}"
if [[ $CONTAINER_USER ]] && [[ $CONTAINER_TOKEN ]]; then
docker login -u "$CONTAINER_USER" -p "$CONTAINER_TOKEN" "$CONTAINER_HUB"
fi
docker pull "''${CONTAINER_HUB}/''${CONTAINER_IMAGE}"
DIGEST=$(docker inspect --format '{{.Id}}' "''${CONTAINER_HUB}/''${CONTAINER_IMAGE}")
DIGEST=''${DIGEST#sha256:}
echo "Measuring $DIGEST" >&2
test -c /dev/tdx_guest && tdx-extend --digest "$DIGEST" --rtmr 3
exec docker run --init --privileged "sha256:$DIGEST"
'';
postStop = lib.mkDefault ''
shutdown --reboot +5
'';
};
services.prometheus.exporters.node = {
enable = true;
port = 9100;
enabledCollectors = [
"logind"
"systemd"
];
disabledCollectors = [
"textfile"
];
#openFirewall = true;
#firewallFilter = "-i br0 -p tcp -m tcp --dport 9100";
};
environment.systemPackages = with pkgs; [
teepot.teepot
];
# /var is on tmpfs anyway
services.journald.storage = "volatile";
# we can't rely on/trust the hypervisor
services.timesyncd.enable = false;
services.chrony = {
enable = true;
enableNTS = true;
servers = [
"time.cloudflare.com"
"ntppool1.time.nl"
"ntppool2.time.nl"
];
};
systemd.services."chronyd".after = [ "network-online.target" ];
boot.kernelPackages = lib.mkForce pkgs.linuxPackages_6_12;
boot.kernelPatches = [
{
name = "tdx-rtmr";
patch = pkgs.fetchurl {
url = "https://github.com/haraldh/linux/commit/12d08008a5c94175e7a7dfcee40dff33431d9033.patch";
hash = "sha256-sVDhvC3qnXpL5FRxWiQotH7Nl/oqRBQGjJGyhsKeBTA=";
};
}
];
boot.kernelParams = [
"console=ttyS0,115200n8"
"random.trust_cpu=on"
];
boot.consoleLogLevel = 7;
boot.initrd.includeDefaultModules = false;
boot.initrd.availableKernelModules = [
"tdx_guest"
"nvme"
"sd_mod"
"dm_mod"
"ata_piix"
];
boot.initrd.systemd.enable = lib.mkDefault true;
services.logind.extraConfig = ''
NAutoVTs=0
ReserveVT=0
'';
services.dbus.implementation = "broker";
boot.initrd.systemd.tpm2.enable = lib.mkForce false;
systemd.tpm2.enable = lib.mkForce false;
nix.enable = false; # it's a read-only nix store anyway
security.pam.services.su.forwardXAuth = lib.mkForce false;
users.mutableUsers = false;
users.allowNoPasswordLogin = true;
system.stateVersion = lib.version;
system.switch.enable = lib.mkForce false;
documentation.info.enable = lib.mkForce false;
documentation.nixos.enable = lib.mkForce false;
documentation.man.enable = lib.mkForce false;
documentation.enable = lib.mkForce false;
services.udisks2.enable = false; # udisks has become too bloated to have in a headless system
# Get rid of the perl ecosystem to minimize the TCB and disk size
# Remove perl from activation
system.etc.overlay.enable = lib.mkDefault true;
services.userborn.enable = lib.mkDefault true;
# Random perl remnants
system.disableInstallerTools = lib.mkForce true;
programs.less.lessopen = lib.mkDefault null;
programs.command-not-found.enable = lib.mkDefault false;
boot.enableContainers = lib.mkForce false;
boot.loader.grub.enable = lib.mkDefault false;
environment.defaultPackages = lib.mkDefault [ ];
# Check that the system does not contain a Nix store path that contains the
# string "perl".
system.forbiddenDependenciesRegexes = [ "perl" ];
}

View file

@ -0,0 +1,15 @@
# SPDX-License-Identifier: Apache-2.0
# Copyright (c) 2024 Matter Labs
{ lib
, pkgs
, system
, ...
}: lib.teepot.nixosGenerate {
inherit (lib) nixosSystem;
inherit system pkgs;
modules = [
./configuration.nix
./google.nix
];
formatModule = ./verity.nix;
}

View file

@ -0,0 +1,33 @@
{ lib
, pkgs
, modulesPath
, ...
}: {
imports = [
"${toString modulesPath}/profiles/headless.nix"
];
system.image.id = "tdx_base";
boot.initrd.kernelModules = [ "virtio_scsi" ];
boot.kernelModules = [ "virtio_pci" "virtio_net" ];
# Force getting the hostname from Google Compute.
networking.hostName = lib.mkForce "";
# Configure default metadata hostnames
networking.extraHosts = ''
169.254.169.254 metadata.google.internal metadata
'';
networking.timeServers = [ "metadata.google.internal" ];
environment.etc."sysctl.d/60-gce-network-security.conf".source = "${pkgs.google-guest-configs}/etc/sysctl.d/60-gce-network-security.conf";
networking.usePredictableInterfaceNames = false;
# GC has 1460 MTU
networking.interfaces.eth0.mtu = 1460;
boot.extraModprobeConfig = lib.readFile "${pkgs.google-guest-configs}/etc/modprobe.d/gce-blacklist.conf";
}

View file

@ -0,0 +1,127 @@
{ config
, pkgs
, lib
, modulesPath
, ...
}:
let
inherit (config.image.repart.verityStore) partitionIds;
in
{
imports = [
"${toString modulesPath}/image/repart.nix"
];
fileSystems = {
"/" = {
fsType = "tmpfs";
options = [ "mode=0755" "noexec" ];
};
"/dev/shm" = {
fsType = "tmpfs";
options = [ "defaults" "nosuid" "noexec" "nodev" "size=2G" ];
};
"/run" = {
fsType = "tmpfs";
options = [ "defaults" "mode=0755" "nosuid" "noexec" "nodev" "size=512M" ];
};
"/usr" = {
device = "/dev/mapper/usr";
# explicitly mount it read-only otherwise systemd-remount-fs will fail
options = [ "ro" ];
fsType = config.image.repart.partitions.${partitionIds.store}.repartConfig.Format;
};
# bind-mount the store
"/nix/store" = {
device = "/usr/nix/store";
options = [ "bind" ];
};
};
image.repart = {
verityStore = {
enable = true;
ukiPath = "/EFI/BOOT/BOOTx64.EFI";
};
partitions = {
${partitionIds.esp} = {
# the UKI is injected into this partition by the verityStore module
repartConfig = {
Type = "esp";
Format = "vfat";
SizeMinBytes = "64M";
};
};
${partitionIds.store-verity}.repartConfig = {
Minimize = "best";
};
${partitionIds.store}.repartConfig = {
Minimize = "best";
Format = "squashfs";
};
};
};
boot = {
loader.grub.enable = false;
initrd.systemd.enable = true;
};
system.image = {
id = lib.mkDefault "nixos-appliance";
version = "1";
};
# don't create /usr/bin/env
# this would require some extra work on read-only /usr
# and it is not a strict necessity
system.activationScripts.usrbinenv = lib.mkForce "";
boot.kernelParams = [
"systemd.verity_usr_options=panic-on-corruption"
"panic=30"
"boot.panic_on_fail" # reboot the machine upon fatal boot issues
"lockdown=1"
];
system.build.vmdk_verity =
config.system.build.finalImage.overrideAttrs
(
finalAttrs: previousAttrs:
let
kernel = config.boot.uki.settings.UKI.Linux;
ukifile = "${config.system.build.uki}/${config.system.boot.loader.ukiFile}";
in
{
nativeBuildInputs =
previousAttrs.nativeBuildInputs
++ [
pkgs.qemu
pkgs.teepot.teepot.rtmr_calc
];
postInstall = ''
qemu-img convert -f raw -O vmdk \
$out/${config.image.repart.imageFileBasename}.raw \
$out/${config.image.repart.imageFileBasename}.vmdk
qemu-img info \
$out/${config.image.repart.imageFileBasename}.vmdk
echo "kernel: ${kernel}"
echo "uki: ${ukifile}"
rtmr-calc \
--image $out/${config.image.repart.imageFileBasename}.raw \
--bootefi "${ukifile}" \
--kernel "${kernel}" | tee $out/${config.image.repart.imageFileBasename}_rtmr.json
rm -vf $out/${config.image.repart.imageFileBasename}.raw
'';
}
);
formatAttr = lib.mkForce "vmdk_verity";
fileExtension = lib.mkForce ".raw";
}

View file

@ -1,16 +1,12 @@
# SPDX-License-Identifier: Apache-2.0 # SPDX-License-Identifier: Apache-2.0
# Copyright (c) 2024 Matter Labs # Copyright (c) 2024 Matter Labs
{ lib { lib
, pkgs
, mkShell , mkShell
, teepot , teepot
, dive
, taplo
, vault
, cargo-release
, nixsgx , nixsgx
, stdenv , stdenv
, teepotCrate , teepotCrate
, pkg-config
}: }:
let let
toolchain_with_src = (teepotCrate.rustVersion.override { toolchain_with_src = (teepotCrate.rustVersion.override {
@ -20,20 +16,26 @@ in
mkShell { mkShell {
inputsFrom = [ teepot.teepot ]; inputsFrom = [ teepot.teepot ];
nativeBuildInputs = [ nativeBuildInputs = with pkgs; [
toolchain_with_src toolchain_with_src
pkg-config pkg-config
teepotCrate.rustPlatform.bindgenHook teepotCrate.rustPlatform.bindgenHook
]; ];
packages = [ packages = with pkgs; [
dive dive
taplo taplo
vault vault
cargo-release cargo-release
google-cloud-sdk-gce
azure-cli
kubectl
kubectx
k9s
]; ];
TEE_LD_LIBRARY_PATH = lib.makeLibraryPath [ TEE_LD_LIBRARY_PATH = lib.makeLibraryPath [
pkgs.curl
nixsgx.sgx-dcap nixsgx.sgx-dcap
nixsgx.sgx-dcap.quote_verify nixsgx.sgx-dcap.quote_verify
nixsgx.sgx-dcap.default_qpl nixsgx.sgx-dcap.default_qpl

View file

@ -0,0 +1,172 @@
{ config
, pkgs
, lib
, ...
}: {
imports = [
./../../../packages/tdx_google/configuration.nix
];
systemd.services.docker_start_container = {
environment = {
CONTAINER_IMAGE = "amd64/hello-world@sha256:e2fc4e5012d16e7fe466f5291c476431beaa1f9b90a5c2125b493ed28e2aba57";
CONTAINER_HUB = "docker.io";
CONTAINER_USER = "";
CONTAINER_TOKEN = "";
};
postStop = ''
:
'';
};
console.enable = true;
services.getty.autologinUser = lib.mkOverride 999 "root";
networking.firewall.allowedTCPPorts = [ 22 ];
services.sshd.enable = true;
services.openssh.settings.PermitRootLogin = lib.mkOverride 999 "yes";
users.users.root.openssh.authorizedKeys.keys = [
"sk-ssh-ed25519@openssh.com AAAAGnNrLXNzaC1lZDI1NTE5QG9wZW5zc2guY29tAAAAIDsb/Tr69YN5MQLweWPuJaRGm+h2kOyxfD6sqKEDTIwoAAAABHNzaDo="
"sk-ecdsa-sha2-nistp256@openssh.com AAAAInNrLWVjZHNhLXNoYTItbmlzdHAyNTZAb3BlbnNzaC5jb20AAAAIbmlzdHAyNTYAAABBBACLgT81iB1iWWVuXq6PdQ5GAAGhaZhSKnveQCvcNnAOZ5WKH80bZShKHyAYzrzbp8IGwLWJcZQ7TqRK+qZdfagAAAAEc3NoOg=="
"ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAYbUTKpy4QR3s944/hjJ1UK05asFEs/SmWeUbtS0cdA660sT4xHnRfals73FicOoz+uIucJCwn/SCM804j+wtM="
"ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMNsmP15vH8BVKo7bdvIiiEjiQboPGcRPqJK0+bH4jKD"
];
fileSystems = {
"/" = {
fsType = "ext4";
device = "/dev/disk/by-id/test";
options = [ "mode=0755" ];
};
};
boot = {
loader.grub.enable = false;
initrd.systemd.enable = true;
};
virtualisation.vmVariant = {
# following configuration is added only when building VM with build-vm
virtualisation = {
memorySize = 2048; # Use 2048MiB memory.
cores = 4;
};
};
/*
services.loki = {
enable = true;
configuration = {
server.http_listen_port = 3030;
auth_enabled = false;
analytics.reporting_enabled = false;
ingester = {
lifecycler = {
address = "127.0.0.1";
ring = {
kvstore = {
store = "inmemory";
};
replication_factor = 1;
};
};
chunk_idle_period = "1h";
max_chunk_age = "1h";
chunk_target_size = 999999;
chunk_retain_period = "30s";
};
schema_config = {
configs = [
{
from = "2024-04-25";
store = "tsdb";
object_store = "filesystem";
schema = "v13";
index = {
prefix = "index_";
period = "24h";
};
}
];
};
storage_config = {
tsdb_shipper = {
active_index_directory = "/var/lib/loki/tsdb-shipper-active";
cache_location = "/var/lib/loki/tsdb-shipper-cache";
cache_ttl = "24h";
};
filesystem = {
directory = "/var/lib/loki/chunks";
};
};
limits_config = {
reject_old_samples = true;
reject_old_samples_max_age = "168h";
volume_enabled = true;
};
table_manager = {
retention_deletes_enabled = false;
retention_period = "0s";
};
compactor = {
working_directory = "/var/lib/loki";
compactor_ring = {
kvstore = {
store = "inmemory";
};
};
};
};
};
services.promtail = {
enable = true;
configuration = {
server = {
http_listen_port = 3031;
grpc_listen_port = 0;
};
clients = [
{
url = "http://127.0.0.1:${toString config.services.loki.configuration.server.http_listen_port}/loki/api/v1/push";
}
];
scrape_configs = [{
job_name = "journal";
journal = {
max_age = "12h";
labels = {
job = "systemd-journal";
};
};
relabel_configs = [
{
source_labels = [ "__journal__systemd_unit" ];
target_label = "systemd_unit";
}
{
source_labels = [ "__journal__hostname" ];
target_label = "nodename";
}
{
source_labels = [ "__journal_container_id" ];
target_label = "container_id";
}
];
}];
};
# extraFlags
};
*/
}