From de06acbef97259e22d9002300306ff246196438f Mon Sep 17 00:00:00 2001 From: Harald Hoyer Date: Wed, 12 Jun 2024 13:15:50 +0200 Subject: [PATCH 1/5] fix: don't tag the nix produced container with `latest` leave it to the github workflow on push to main Signed-off-by: Harald Hoyer --- .../default.nix | 2 +- .../default.nix | 4 +-- packages/container-vault-admin/default.nix | 28 +++++++++++++++++++ .../container-vault-sgx-azure/default.nix | 2 +- .../default.nix | 2 +- packages/container-vault-unseal/default.nix | 1 - .../default.nix | 1 - 7 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 packages/container-vault-admin/default.nix diff --git a/packages/container-self-attestation-test-sgx-azure/default.nix b/packages/container-self-attestation-test-sgx-azure/default.nix index 56a061b..cdc240e 100644 --- a/packages/container-self-attestation-test-sgx-azure/default.nix +++ b/packages/container-self-attestation-test-sgx-azure/default.nix @@ -6,7 +6,7 @@ , teepot , nixsgx , container-name ? "teepot-self-attestation-test-sgx-azure" -, tag ? "latest" +, tag ? null , isAzure ? true }: pkgs.callPackage inputs.nixsgx-flake.lib.mkSGXContainer { diff --git a/packages/container-vault-admin-sgx-azure/default.nix b/packages/container-vault-admin-sgx-azure/default.nix index 975d9b9..c2e7820 100644 --- a/packages/container-vault-admin-sgx-azure/default.nix +++ b/packages/container-vault-admin-sgx-azure/default.nix @@ -6,8 +6,8 @@ , teepot , nixsgx , container-name ? "teepot-vault-admin-sgx-azure" -, tag ? "latest" -, isAzure ? true +, tag ? null +, isAzure ? null }: pkgs.callPackage inputs.nixsgx-flake.lib.mkSGXContainer { name = container-name; diff --git a/packages/container-vault-admin/default.nix b/packages/container-vault-admin/default.nix new file mode 100644 index 0000000..235ddd7 --- /dev/null +++ b/packages/container-vault-admin/default.nix @@ -0,0 +1,28 @@ +# SPDX-License-Identifier: Apache-2.0 +# Copyright (c) 2024 Matter Labs +{ dockerTools +, nixsgx +, teepot +, buildEnv +, curl +}: +dockerTools.buildLayeredImage { + name = "vault-unseal"; + + config.Entrypoint = [ "${teepot.teepot.vault_unseal}/bin/vault-unseal" ]; + + contents = buildEnv { + name = "image-root"; + paths = with dockerTools; with nixsgx;[ + azure-dcap-client + curl.out + sgx-dcap.quote_verify + usrBinEnv + binSh + caCertificates + fakeNss + teepot.teepot.vault_unseal + ]; + pathsToLink = [ "/bin" "/lib" "/etc" ]; + }; +} diff --git a/packages/container-vault-sgx-azure/default.nix b/packages/container-vault-sgx-azure/default.nix index 35cab91..6935c5e 100644 --- a/packages/container-vault-sgx-azure/default.nix +++ b/packages/container-vault-sgx-azure/default.nix @@ -8,7 +8,7 @@ , vat , vault , container-name ? "teepot-vault-sgx-azure" -, tag ? "latest" +, tag ? null , isAzure ? true }: let diff --git a/packages/container-vault-unseal-sgx-azure/default.nix b/packages/container-vault-unseal-sgx-azure/default.nix index 56d2989..90ada64 100644 --- a/packages/container-vault-unseal-sgx-azure/default.nix +++ b/packages/container-vault-unseal-sgx-azure/default.nix @@ -7,7 +7,7 @@ , nixsgx , vat , container-name ? "teepot-vault-unseal-sgx-azure" -, tag ? "latest" +, tag ? null , isAzure ? true }: pkgs.callPackage inputs.nixsgx-flake.lib.mkSGXContainer { diff --git a/packages/container-vault-unseal/default.nix b/packages/container-vault-unseal/default.nix index 22a4387..235ddd7 100644 --- a/packages/container-vault-unseal/default.nix +++ b/packages/container-vault-unseal/default.nix @@ -8,7 +8,6 @@ }: dockerTools.buildLayeredImage { name = "vault-unseal"; - tag = "latest"; config.Entrypoint = [ "${teepot.teepot.vault_unseal}/bin/vault-unseal" ]; diff --git a/packages/container-verify-attestation-sgx/default.nix b/packages/container-verify-attestation-sgx/default.nix index 2918514..3d88a1a 100644 --- a/packages/container-verify-attestation-sgx/default.nix +++ b/packages/container-verify-attestation-sgx/default.nix @@ -9,7 +9,6 @@ }: dockerTools.buildLayeredImage { name = "verify-attestation-sgx"; - tag = "latest"; config.Cmd = [ "${teepot.teepot.verify_attestation}/bin/verify-attestation" ]; config.Env = [ "LD_LIBRARY_PATH=/lib" ]; From 9c01b0a2810bc84e9f9aa529d13383b1e16242f7 Mon Sep 17 00:00:00 2001 From: Harald Hoyer Date: Wed, 12 Jun 2024 13:22:03 +0200 Subject: [PATCH 2/5] feat: add `container-vault-admin` Signed-off-by: Harald Hoyer --- packages/container-vault-admin/default.nix | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/packages/container-vault-admin/default.nix b/packages/container-vault-admin/default.nix index 235ddd7..ec70476 100644 --- a/packages/container-vault-admin/default.nix +++ b/packages/container-vault-admin/default.nix @@ -1,27 +1,29 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2024 Matter Labs { dockerTools -, nixsgx -, teepot , buildEnv +, teepot +, openssl , curl +, nixsgx }: dockerTools.buildLayeredImage { - name = "vault-unseal"; + name = "vault-admin"; - config.Entrypoint = [ "${teepot.teepot.vault_unseal}/bin/vault-unseal" ]; + config.Entrypoint = [ "${teepot.teepot.vault_admin}/bin/vault-admin" ]; contents = buildEnv { name = "image-root"; paths = with dockerTools; with nixsgx;[ - azure-dcap-client + openssl.out curl.out sgx-dcap.quote_verify + sgx-dcap.default_qpl usrBinEnv binSh caCertificates fakeNss - teepot.teepot.vault_unseal + teepot.teepot.vault_admin ]; pathsToLink = [ "/bin" "/lib" "/etc" ]; }; From 43a7931a4094c21e34aa6ed7466346b1217ad351 Mon Sep 17 00:00:00 2001 From: Harald Hoyer Date: Wed, 12 Jun 2024 13:32:51 +0200 Subject: [PATCH 3/5] fix(container-vault-unseal): remove azure config Not needed anymore. Stuff can be gathered via the default qpl Signed-off-by: Harald Hoyer --- packages/container-vault-unseal/default.nix | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/packages/container-vault-unseal/default.nix b/packages/container-vault-unseal/default.nix index 235ddd7..1345d09 100644 --- a/packages/container-vault-unseal/default.nix +++ b/packages/container-vault-unseal/default.nix @@ -1,10 +1,11 @@ # SPDX-License-Identifier: Apache-2.0 # Copyright (c) 2024 Matter Labs { dockerTools -, nixsgx -, teepot , buildEnv +, teepot +, openssl , curl +, nixsgx }: dockerTools.buildLayeredImage { name = "vault-unseal"; @@ -14,9 +15,10 @@ dockerTools.buildLayeredImage { contents = buildEnv { name = "image-root"; paths = with dockerTools; with nixsgx;[ - azure-dcap-client + openssl.out curl.out sgx-dcap.quote_verify + sgx-dcap.default_qpl usrBinEnv binSh caCertificates From 4aa1f40c507fc3046822867c4160153af9258334 Mon Sep 17 00:00:00 2001 From: Harald Hoyer Date: Wed, 12 Jun 2024 13:34:06 +0200 Subject: [PATCH 4/5] docs(examples): fix the container names Signed-off-by: Harald Hoyer --- examples/README.md | 46 ++++++++++++++++++----------------- examples/k8s/vault-1-pod.yaml | 4 +-- examples/k8s/vault-2-pod.yaml | 4 +-- examples/k8s/vault-3-pod.yaml | 4 +-- 4 files changed, 30 insertions(+), 28 deletions(-) diff --git a/examples/README.md b/examples/README.md index 6b2e979..2533743 100644 --- a/examples/README.md +++ b/examples/README.md @@ -47,12 +47,13 @@ Vault is unsealed! ``` -With `teepot-admin` being the name of the image running the tee-vault-admin service, the following commands can be used +With `teepot-vault-admin-sgx-azure` being the name of the image running the teepot-vault-admin-sgx-azure service, the +following commands can be used to sign the admin tee: ```bash -❯ (id=$(docker create teepot-admin); docker cp $id:/app/tee-vault-admin.sig ~/tee-vault-admin.sig; docker rm -v $id) -❯ cargo run -p vault-admin -- create-sign-request --tee-name admin ~/tee-vault-admin.sig > ~/sign_admin_tee.json +❯ (id=$(docker create teepot-vault-admin-sgx-azure); docker cp $id:/app/teepot-vault-admin-sgx-azure.sig ~/teepot-vault-admin-sgx-azure.sig; docker rm -v $id) +❯ cargo run -p vault-admin -- create-sign-request --tee-name admin ~/teepot-vault-admin-sgx-azure.sig > ~/sign_admin_tee.json ❯ vim sign_admin_tee.json ❯ gpg --local-user test@example.com --detach-sign --armor ~/sign_admin_tee.json ❯ RUST_LOG=info cargo run -p vault-admin -- \ @@ -91,8 +92,8 @@ Attributes: ```bash ❯ docker compose build && (docker compose rm; docker volume rm teepot_vault-storage teepot_ha-raft-1 teepot_shared-1 teepot_ha-raft-2 teepot_shared-2 teepot_ha-raft-3 teepot_shared-3; docke r compose up --remove-orphans vault-1 tvu-1) -❯ (id=$(docker create teepot-admin); docker cp $id:/app/tee-vault-admin.sig ~/tee-vault-admin.sig; docker rm -v $id) -❯ gramine-sgx-sigstruct-view ~/tee-vault-admin.sig +❯ (id=$(docker create teepot-vault-admin-sgx-azure); docker cp $id:/app/teepot-vault-admin-sgx-azure.sig ~/teepot-vault-admin-sgx-azure.sig; docker rm -v $id) +❯ gramine-sgx-sigstruct-view ~/teepot-vault-admin-sgx-azure.sig Attributes: mr_signer: c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d mr_enclave: 265ca491bf13e2486fd67d12038fcce02f133c5d91277e42f58c0ab464d5b46b @@ -117,10 +118,10 @@ Passphrase: ## Kubernetes -Find out the `mr_enclave` value of the tee-vault-admin enclave and extract the sigstruct file: +Find out the `mr_enclave` value of the teepot-vault-admin-sgx-azure enclave and extract the sigstruct file: ```bash -❯ docker run -v .:/mnt --pull always -it matterlabsrobot/teepot-tva:latest 'gramine-sgx-sigstruct-view tee-vault-admin.sig; cp tee-vault-admin.sig /mnt' +❯ docker run -v .:/mnt --pull always -it matterlabsrobot/teepot-vault-admin-sgx-azure:latest 'gramine-sgx-sigstruct-view teepot-vault-admin-sgx-azure.sig; cp teepot-vault-admin-sgx-azure.sig /mnt' [...] Attributes: mr_signer: c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d @@ -128,8 +129,8 @@ Attributes: isv_prod_id: 0 isv_svn: 0 debug_enclave: False -❯ ls -l ~/tee-vault-admin.sig --rw-r--r--. 1 harald harald 1808 2. Nov 10:46 tee-vault-admin.sig +❯ ls -l ~/teepot-vault-admin-sgx-azure.sig +-rw-r--r--. 1 harald harald 1808 2. Nov 10:46 teepot-vault-admin-sgx-azure.sig ``` Start the vault service and pod and forward the port @@ -145,7 +146,8 @@ Start the vault service and pod and forward the port Initialize the instance. This can take up to 6 minutes, depending on the `performance_multiplier` setting in vault. -Adjust the `--admin-tee-mrenclave` parameter to match the `mr_enclave` value of the tee-vault-admin container. +Adjust the `--admin-tee-mrenclave` parameter to match the `mr_enclave` value of the teepot-vault-admin-sgx-azure +container. ```bash ❯ RUST_LOG=info cargo run -p vault-unseal -- \ @@ -209,40 +211,40 @@ The vault cluster should now settle to be completely unsealed and synced. Start the vault-admin pod and forward the port: ```bash -❯ kubectl port-forward pods/tee-vault-admin 8444 +❯ kubectl port-forward pods/teepot-vault-admin-sgx-azure 8444 ``` Next is to sign the admin tee with the vault-admin tool: ```bash -❯ cargo run -p vault-admin -- create-sign-request --tee-name admin ~/tee-vault-admin.sig > ~/tee-vault-admin.json -❯ gpg --local-user test@example.com --detach-sign --armor ~/tee-vault-admin.json +❯ cargo run -p vault-admin -- create-sign-request --tee-name admin ~/teepot-vault-admin-sgx-azure.sig > ~/teepot-vault-admin-sgx-azure.json +❯ gpg --local-user test@example.com --detach-sign --armor ~/teepot-vault-admin-sgx-azure.json ❯ cargo run -p vault-admin -- command \ --server https://127.0.0.1:8444 \ --sgx-allowed-tcb-levels SwHardeningNeeded \ - --out ~/tee-vault-admin-new.sig \ - ~/tee-vault-admin.json ~/tee-vault-admin.json.asc + --out ~/teepot-vault-admin-sgx-azure-new.sig \ + ~/teepot-vault-admin-sgx-azure.json ~/teepot-vault-admin-sgx-azure.json.asc ``` -Then replace `tee-vault-admin.sig` with `tee-vault-admin-new.sig` in the container -image `matterlabsrobot/teepot-tva:latest` with this Dockerfile: +Then replace `teepot-vault-admin-sgx-azure.sig` with `teepot-vault-admin-sgx-azure-new.sig` in the container +image `matterlabsrobot/teepot-vault-admin-sgx-azure:latest` with this Dockerfile: ```Dockerfile -FROM matterlabsrobot/teepot-tva:latest -COPY tee-vault-admin-new.sig /app/tee-vault-admin.sig +FROM matterlabsrobot/teepot-vault-admin-sgx-azure:latest +COPY teepot-vault-admin-sgx-azure-new.sig /app/teepot-vault-admin-sgx-azure.sig ``` Build and push the new image: ```bash -❯ docker build -t matterlabsrobot/teepot-tva-signed:latest . -❯ docker push matterlabsrobot/teepot-tva-signed:latest +❯ docker build -t matterlabsrobot/teepot-vault-admin-sgx-azure-signed:latest . +❯ docker push matterlabsrobot/teepot-vault-admin-sgx-azure-signed:latest ``` Delete the old vault-admin pod and start the new one: ```bash -❯ kubectl delete pod/tee-vault-admin +❯ kubectl delete pod/teepot-vault-admin-sgx-azure ❯ kubectl apply -f examples/k8s/vault-admin-signed-pod.yaml ``` diff --git a/examples/k8s/vault-1-pod.yaml b/examples/k8s/vault-1-pod.yaml index dbc13e5..fc018a1 100644 --- a/examples/k8s/vault-1-pod.yaml +++ b/examples/k8s/vault-1-pod.yaml @@ -27,7 +27,7 @@ spec: imagePullSecrets: - name: docker-regcred containers: - - image: matterlabsrobot/teepot-vault:latest + - image: matterlabsrobot/teepot-vault-sgx-azure:latest name: vault imagePullPolicy: Always env: @@ -64,7 +64,7 @@ spec: name: shared-1 - mountPath: /opt/vault/data name: data-1 - - image: matterlabsrobot/teepot-tvu:latest + - image: matterlabsrobot/teepot-vault-unseal-sgx-azure:latest name: vault-unseal imagePullPolicy: Always env: diff --git a/examples/k8s/vault-2-pod.yaml b/examples/k8s/vault-2-pod.yaml index c8e739c..3c8c0ff 100644 --- a/examples/k8s/vault-2-pod.yaml +++ b/examples/k8s/vault-2-pod.yaml @@ -27,7 +27,7 @@ spec: imagePullSecrets: - name: docker-regcred containers: - - image: matterlabsrobot/teepot-vault:latest + - image: matterlabsrobot/teepot-vault-sgx-azure:latest name: vault imagePullPolicy: Always env: @@ -64,7 +64,7 @@ spec: name: shared-2 - mountPath: /opt/vault/data name: data-2 - - image: matterlabsrobot/teepot-tvu:latest + - image: matterlabsrobot/teepot-vault-unseal-sgx-azure:latest name: vault-unseal imagePullPolicy: Always env: diff --git a/examples/k8s/vault-3-pod.yaml b/examples/k8s/vault-3-pod.yaml index 2d5b975..48b28d2 100644 --- a/examples/k8s/vault-3-pod.yaml +++ b/examples/k8s/vault-3-pod.yaml @@ -27,7 +27,7 @@ spec: imagePullSecrets: - name: docker-regcred containers: - - image: matterlabsrobot/teepot-vault:latest + - image: matterlabsrobot/teepot-vault-sgx-azure:latest name: vault imagePullPolicy: Always env: @@ -64,7 +64,7 @@ spec: name: shared-3 - mountPath: /opt/vault/data name: data-3 - - image: matterlabsrobot/teepot-tvu:latest + - image: matterlabsrobot/teepot-vault-unseal-sgx-azure:latest name: vault-unseal imagePullPolicy: Always env: From cfb133bca9f037cefa5fbd6cbd7c8c83d2e53cb5 Mon Sep 17 00:00:00 2001 From: Harald Hoyer Date: Wed, 12 Jun 2024 13:50:06 +0200 Subject: [PATCH 5/5] ci: fix and revise docker push strategy - containers are not `latest` by default anymore - `latest` tag is only set on push to main branch - buildid tag is only set on push to main branch, and changed to the infra repo soonish - added the missing `vault-unseal` and `vault-admin` container Signed-off-by: Harald Hoyer --- .github/workflows/nix.yml | 53 ++++++++++++++++++++++----------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/.github/workflows/nix.yml b/.github/workflows/nix.yml index af13702..3240cb1 100644 --- a/.github/workflows/nix.yml +++ b/.github/workflows/nix.yml @@ -51,7 +51,6 @@ jobs: push_to_docker: needs: build - if: ${{ github.event_name == 'push' }} runs-on: ubuntu-latest concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.config.nixpackage }} @@ -60,11 +59,13 @@ jobs: fail-fast: false matrix: config: - - { nixpackage: 'container-vault-sgx-azure', dockerfile: 'packages/container-vault-sgx-azure/Dockerfile', repository: 'teepot-vault' } - - { nixpackage: 'container-vault-unseal-sgx-azure', dockerfile: 'packages/container-vault-unseal-sgx-azure/Dockerfile', repository: 'teepot-tvu' } - - { nixpackage: 'container-vault-admin-sgx-azure', dockerfile: 'packages/container-vault-admin-sgx-azure/Dockerfile', repository: 'teepot-tva' } - - { nixpackage: 'container-self-attestation-test-sgx-dcap', dockerfile: 'packages/container-self-attestation-test-sgx-dcap/Dockerfile', repository: 'teepot-self-attestation-test-sgx-dcap' } - - { nixpackage: 'container-self-attestation-test-sgx-azure', dockerfile: 'packages/container-self-attestation-test-sgx-azure/Dockerfile', repository: 'teepot-self-attestation-test-sgx-azure' } + - { nixpackage: 'container-vault-sgx-azure' } + - { nixpackage: 'container-vault-unseal-sgx-azure' } + - { nixpackage: 'container-vault-admin-sgx-azure' } + - { nixpackage: 'container-vault-unseal' } + - { nixpackage: 'container-vault-admin' } + - { nixpackage: 'container-self-attestation-test-sgx-dcap' } + - { nixpackage: 'container-self-attestation-test-sgx-azure' } - { nixpackage: 'container-verify-attestation-sgx' } steps: - uses: actions/checkout@v4 @@ -86,30 +87,36 @@ jobs: username: ${{ secrets.DOCKERHUB_USER }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - name: Load and Push nix container + - name: Load container + id: build run: | - nix build -L .#${{ matrix.config.nixpackage }} + nix build --accept-flake-config -L .#${{ matrix.config.nixpackage }} export IMAGE_TAG=$(docker load < result | grep -Po 'Loaded image.*: \K.*') - echo "Pushing image ${IMAGE_TAG} to Docker Hub" - docker tag "${IMAGE_TAG}" matterlabsrobot/"${IMAGE_TAG}" - docker push matterlabsrobot/"${IMAGE_TAG}" - docker tag matterlabsrobot/"${IMAGE_TAG}" matterlabsrobot/"${IMAGE_TAG%:*}:latest" - docker push matterlabsrobot/"${IMAGE_TAG%:*}:latest" + echo "IMAGE_TAG=${IMAGE_TAG}" >> "$GITHUB_OUTPUT" + echo "IMAGE_NAME=${IMAGE_TAG%:*}" >> "$GITHUB_OUTPUT" + + - name: Push container + run: | + echo "Pushing image ${{ steps.build.outputs.IMAGE_TAG }} to Docker Hub" + docker tag "${{ steps.build.outputs.IMAGE_TAG }}" matterlabsrobot/"${{ steps.build.outputs.IMAGE_TAG }}" + docker push matterlabsrobot/"${{ steps.build.outputs.IMAGE_TAG }}" + + - name: Tag container as latest + if: ${{ github.event_name == 'push' || github.event_name == 'schedule' }} + run: | + docker tag "${{ steps.build.outputs.IMAGE_TAG }}" matterlabsrobot/"${{ steps.build.outputs.IMAGE_NAME }}:latest" + docker push matterlabsrobot/"${{ steps.build.outputs.IMAGE_NAME }}:latest" - name: Generate build ID for Flux Image Automation - if: ${{ matrix.config.dockerfile }} + if: ${{ github.event_name == 'push' || github.event_name == 'schedule' }} id: buildid run: | sha=$(git rev-parse --short HEAD) ts=$(date +%s%N | cut -b1-13) echo "BUILD_ID=${sha}-${ts}" >> "$GITHUB_OUTPUT" - - name: Build and Push Container - if: ${{ matrix.config.dockerfile }} - uses: docker/build-push-action@v5 - with: - file: ${{ matrix.config.dockerfile }} - tags: | - "matterlabsrobot/${{ matrix.config.repository }}:latest" - "matterlabsrobot/${{ matrix.config.repository }}:${{ steps.buildid.outputs.BUILD_ID }}" - push: ${{ github.event_name == 'push' || github.event_name == 'schedule' }} + - name: Push Docker image to matterlabs-infra + if: ${{ github.event_name == 'push' || github.event_name == 'schedule' }} + run: | + docker tag "${{ steps.build.outputs.IMAGE_TAG }}" matterlabsrobot/"${{ steps.build.outputs.IMAGE_NAME }}::${{ steps.buildid.outputs.BUILD_ID }}" + docker push matterlabsrobot/"${{ steps.build.outputs.IMAGE_NAME }}::${{ steps.buildid.outputs.BUILD_ID }}"