feat: initial commit

Signed-off-by: Harald Hoyer <harald@matterlabs.dev>

bin/tee-key-preexec/Cargo.toml

@@ -0,0 +1,20 @@
+[package]
+name = "tee-key-preexec"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+anyhow.workspace = true
+hex.workspace = true
+k256.workspace = true
+rand.workspace = true
+sha2.workspace = true
+teepot.workspace = true
+tracing-log.workspace = true
+tracing-subscriber.workspace = true
+tracing.workspace = true

bin/tee-key-preexec/src/main.rs

@@ -0,0 +1,75 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2024 Matter Labs
+
+//! Pre-exec for binary running in a TEE needing attestation of a secret signing key
+
+#![deny(missing_docs)]
+#![deny(clippy::all)]
+
+use anyhow::{Context, Result};
+use hex::ToHex;
+use k256::ecdsa::SigningKey;
+use sha2::Digest;
+use sha2::Sha256;
+use std::env;
+use std::os::unix::process::CommandExt;
+use std::process::Command;
+use teepot::quote::get_quote;
+use tracing::error;
+use tracing_log::LogTracer;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter, Registry};
+
+const TEE_QUOTE_FILE: &str = "/tmp/tee_quote";
+
+fn main_with_error() -> Result<()> {
+    LogTracer::init().context("Failed to set logger")?;
+
+    let subscriber = Registry::default()
+        .with(EnvFilter::from_default_env())
+        .with(fmt::layer().with_writer(std::io::stderr));
+    tracing::subscriber::set_global_default(subscriber).context("Failed to set logger")?;
+
+    let args = env::args_os().collect::<Box<_>>();
+
+    if args.len() < 2 {
+        return Err(anyhow::anyhow!(
+            "Usage: {} <command> [args...]",
+            args[0].to_string_lossy()
+        ));
+    }
+
+    let mut rng = rand::thread_rng();
+    let signing_key = SigningKey::random(&mut rng);
+    let verifying_key_bytes = signing_key.verifying_key().to_sec1_bytes();
+    let hash_verifying_key = Sha256::digest(verifying_key_bytes);
+    let signing_key_string = signing_key.to_bytes().encode_hex::<String>();
+    let tee_type = match get_quote(&hash_verifying_key) {
+        Ok(quote) => {
+            // save quote to file
+            std::fs::write(TEE_QUOTE_FILE, quote)?;
+            "sgx"
+        }
+        Err(e) => {
+            error!("Failed to get quote: {}", e);
+            std::fs::write(TEE_QUOTE_FILE, [])?;
+            "none"
+        }
+    };
+
+    let err = Command::new(&args[1])
+        .args(&args[2..])
+        .env("TEE_SIGNING_KEY", signing_key_string)
+        .env("TEE_QUOTE_FILE", TEE_QUOTE_FILE)
+        .env("TEE_TYPE", tee_type)
+        .exec();
+
+    Err(err).with_context(|| format!("exec of `{cmd}` failed", cmd = args[1].to_string_lossy()))
+}
+
+fn main() -> Result<()> {
+    let ret = main_with_error();
+    if let Err(e) = &ret {
+        error!("Error: {}", e);
+    }
+    ret
+}

bin/tee-self-attestation-test/Cargo.toml

@@ -0,0 +1,15 @@
+[package]
+name = "tee-self-attestation-test"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+actix-web.workspace = true
+anyhow.workspace = true
+teepot.workspace = true
+tracing-log.workspace = true
+tracing-subscriber.workspace = true
+tracing.workspace = true

bin/tee-self-attestation-test/Dockerfile-azure

@@ -0,0 +1,85 @@
+FROM docker.io/ubuntu:20.04 AS azuredcap
+WORKDIR /build
+ADD https://github.com/microsoft/Azure-DCAP-Client/archive/refs/tags/1.12.0.tar.gz ./Azure-DCAP-Client.tar.gz
+RUN tar -xvf Azure-DCAP-Client.tar.gz
+COPY assets/Azure-DCAP-Client.patch ./Azure-DCAP-Client.patch
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y software-properties-common; \
+    add-apt-repository ppa:team-xbmc/ppa -y; \
+    apt-get update; \
+    apt-get install -y \
+        build-essential \
+        cmake \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        pkg-config \
+        nlohmann-json3-dev \
+        wget \
+        dos2unix \
+        ;
+
+WORKDIR /build/Azure-DCAP-Client-1.12.0
+RUN dos2unix src/dcap_provider.cpp && patch -p1 < ../Azure-DCAP-Client.patch
+WORKDIR /build/Azure-DCAP-Client-1.12.0/src/Linux
+RUN ./configure && make && make install
+
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-self-attestation-test.manifest.template.toml' -av /data/ ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-self-attestation-test --bin tee-self-attestation-test \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-self-attestation-test ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/microsoft.asc https://packages.microsoft.com/keys/microsoft.asc \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.asc] https://packages.microsoft.com/ubuntu/20.04/prod focal main" > /etc/apt/sources.list.d/msprod.list \
+    && apt-get update \
+    && apt purge -y libsgx-dcap-default-qpl \
+    && apt-get install -y az-dcap-client
+
+RUN apt purge -y libsgx-ae-qve
+#    libsgx-urts
+
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /etc/sgx_default_qcnl.conf
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-self-attestation-test .
+COPY ./bin/tee-self-attestation-test/tee-self-attestation-test.manifest.template.toml .
+COPY vault/enclave-key.pem .
+
+# The original Azure library is still delivering expired collateral, so we have to use a patched version
+COPY --from=azuredcap /usr/local/lib/libdcap_quoteprov.so /usr/lib/
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-self-attestation-test.manifest.template.toml tee-self-attestation-test.manifest \
+    && gramine-sgx-sign --manifest tee-self-attestation-test.manifest --output tee-self-attestation-test.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-self-attestation-test" ]

bin/tee-self-attestation-test/Dockerfile-intel

@@ -0,0 +1,58 @@
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-vault-admin.manifest.template' -av /data/ ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-self-attestation-test --bin tee-self-attestation-test \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-self-attestation-test ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        libsgx-dcap-default-qpl \
+        libsgx-urts \
+        libsgx-enclave-common \
+        libsgx-dcap-quote-verify
+RUN apt purge -y libsgx-ae-qve
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /lib/libdcap_quoteprov.so
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-self-attestation-test .
+COPY ./bin/tee-self-attestation-test/tee-self-attestation-test.template.toml .
+COPY vault/enclave-key.pem .
+
+COPY assets/sgx_default_qcnl.conf.json /etc/sgx_default_qcnl.conf
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-self-attestation-test.template.toml tee-self-attestation-test.manifest \
+    && gramine-sgx-sign --manifest tee-self-attestation-test.manifest --output tee-self-attestation-test.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "exec gramine-sgx tee-self-attestation-test" ]

bin/tee-self-attestation-test/src/main.rs

@@ -0,0 +1,30 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+//! Simple TEE self-attestation test
+
+#![deny(missing_docs)]
+#![deny(clippy::all)]
+
+use anyhow::{Context, Result};
+use teepot::server::attestation::get_quote_and_collateral;
+use tracing::error;
+use tracing_log::LogTracer;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter, Registry};
+
+#[actix_web::main]
+async fn main() -> Result<()> {
+    LogTracer::init().context("Failed to set logger")?;
+
+    let subscriber = Registry::default()
+        .with(EnvFilter::from_default_env())
+        .with(fmt::layer().with_writer(std::io::stderr));
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    let report_data = [0u8; 64];
+    if let Err(e) = get_quote_and_collateral(None, &report_data) {
+        error!("failed to get quote and collateral: {e:?}");
+        return Err(e);
+    }
+    Ok(())
+}

bin/tee-self-attestation-test/tee-self-attestation-test.template.toml

@@ -0,0 +1,57 @@
+libos.entrypoint = "/app/tee-self-attestation-test"
+
+[loader]
+argv = ["/app/tee-self-attestation-test"]
+entrypoint = "file:{{ gramine.libos }}"
+env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}:/usr{{ arch_libdir }}:/lib"
+env.HOME = "/app"
+env.MALLOC_ARENA_MAX = "1"
+env.AZDCAP_DEBUG_LOG_LEVEL = "ignore"
+env.AZDCAP_COLLATERAL_VERSION = "v4"
+
+### DEBUG ###
+env.RUST_BACKTRACE = "1"
+env.RUST_LOG = "info"
+
+[fs]
+root.uri = "file:/"
+start_dir = "/app"
+mounts = [
+  { path = "{{ execdir }}", uri = "file:{{ execdir }}" },
+  { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
+  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
+  { path = "/etc", uri = "file:/etc" },
+  { type = "tmpfs", path = "/var/tmp" },
+  { type = "tmpfs", path = "/tmp" },
+  { type = "tmpfs", path = "/app/.dcap-qcnl" },
+  { type = "tmpfs", path = "/app/.az-dcap-client" },
+  { path = "/lib/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" },
+]
+
+[sgx]
+trusted_files = [
+  "file:/etc/ld.so.cache",
+  "file:/app/",
+  "file:{{ execdir }}/",
+  "file:{{ arch_libdir }}/",
+  "file:/usr/{{ arch_libdir }}/",
+  "file:{{ gramine.libos }}",
+  "file:{{ gramine.runtimedir() }}/",
+  "file:/usr/lib/ssl/openssl.cnf",
+  "file:/etc/ssl/",
+  "file:/etc/sgx_default_qcnl.conf",
+  "file:/lib/libdcap_quoteprov.so",
+]
+remote_attestation = "dcap"
+max_threads = 64
+edmm_enable = false
+## max enclave size
+enclave_size = "1G"
+
+[sys]
+enable_extra_runtime_domain_names_conf = true
+enable_sigterm_injection = true
+
+# possible tweak option, if problems with mio
+# currently mio is compiled with `mio_unsupported_force_waker_pipe`
+# insecure__allow_eventfd = true

bin/tee-stress-client/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "tee-stress-client"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+actix-web.workspace = true
+anyhow.workspace = true
+awc.workspace = true
+base64.workspace = true
+bytemuck.workspace = true
+clap.workspace = true
+hex.workspace = true
+mio.workspace = true
+rustls.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+teepot.workspace = true
+tracing-actix-web.workspace = true
+tracing-log.workspace = true
+tracing-subscriber.workspace = true
+tracing.workspace = true

bin/tee-stress-client/Dockerfile-azure

@@ -0,0 +1,85 @@
+FROM docker.io/ubuntu:20.04 AS azuredcap
+WORKDIR /build
+ADD https://github.com/microsoft/Azure-DCAP-Client/archive/refs/tags/1.12.0.tar.gz ./Azure-DCAP-Client.tar.gz
+RUN tar -xvf Azure-DCAP-Client.tar.gz
+COPY assets/Azure-DCAP-Client.patch ./Azure-DCAP-Client.patch
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y software-properties-common; \
+    add-apt-repository ppa:team-xbmc/ppa -y; \
+    apt-get update; \
+    apt-get install -y \
+        build-essential \
+        cmake \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        pkg-config \
+        nlohmann-json3-dev \
+        wget \
+        dos2unix \
+        ;
+
+WORKDIR /build/Azure-DCAP-Client-1.12.0
+RUN dos2unix src/dcap_provider.cpp && patch -p1 < ../Azure-DCAP-Client.patch
+WORKDIR /build/Azure-DCAP-Client-1.12.0/src/Linux
+RUN ./configure && make && make install
+
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-stress-client.manifest.template' -av /data/ ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-stress-client --bin tee-stress-client \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-stress-client ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/microsoft.asc https://packages.microsoft.com/keys/microsoft.asc \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.asc] https://packages.microsoft.com/ubuntu/20.04/prod focal main" > /etc/apt/sources.list.d/msprod.list \
+    && apt-get update \
+    && apt purge -y libsgx-dcap-default-qpl \
+    && apt-get install -y az-dcap-client
+
+RUN apt purge -y libsgx-ae-qve
+#    libsgx-urts
+
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /etc/sgx_default_qcnl.conf
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-stress-client .
+COPY ./bin/tee-stress-client/tee-stress-client.manifest.template .
+COPY vault/enclave-key.pem .
+
+# The original Azure library is still delivering expired collateral, so we have to use a patched version
+COPY --from=azuredcap /usr/local/lib/libdcap_quoteprov.so /usr/lib/
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-stress-client.manifest.template tee-stress-client.manifest \
+    && gramine-sgx-sign --manifest tee-stress-client.manifest --output tee-stress-client.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-stress-client" ]

bin/tee-stress-client/Dockerfile-intel

@@ -0,0 +1,58 @@
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-stress-client.manifest.template' -av /data/ ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-stress-client --bin tee-stress-client \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-stress-client ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        libsgx-dcap-default-qpl \
+        libsgx-urts \
+        libsgx-enclave-common \
+        libsgx-dcap-quote-verify
+RUN apt purge -y libsgx-ae-qve
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /lib/libdcap_quoteprov.so
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-stress-client .
+COPY ./bin/tee-stress-client/tee-stress-client.manifest.template .
+COPY vault/enclave-key.pem .
+
+COPY assets/sgx_default_qcnl.conf.json /etc/sgx_default_qcnl.conf
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-stress-client.manifest.template tee-stress-client.manifest \
+    && gramine-sgx-sign --manifest tee-stress-client.manifest --output tee-stress-client.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-stress-client" ]

bin/tee-stress-client/src/main.rs

@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: Apache-2.0
+
+//! Server to handle requests to the Vault TEE
+
+#![deny(missing_docs)]
+#![deny(clippy::all)]
+
+use actix_web::rt::time::sleep;
+use anyhow::{Context, Result};
+use clap::Parser;
+use serde::{Deserialize, Serialize};
+use std::time::Duration;
+use teepot::client::vault::VaultConnection;
+use teepot::server::attestation::{get_quote_and_collateral, VaultAttestationArgs};
+use teepot::server::pki::make_self_signed_cert;
+use teepot::sgx::{parse_tcb_levels, EnumSet, TcbLevel};
+use tracing::{error, trace};
+use tracing_log::LogTracer;
+use tracing_subscriber::Registry;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter};
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Arguments {
+    /// allowed TCB levels, comma separated
+    #[arg(long, value_parser = parse_tcb_levels, env = "ALLOWED_TCB_LEVELS", default_value = "Ok")]
+    my_sgx_allowed_tcb_levels: EnumSet<TcbLevel>,
+    #[clap(flatten)]
+    pub attestation: VaultAttestationArgs,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct MySecret {
+    val: usize,
+}
+
+#[actix_web::main]
+async fn main() -> Result<()> {
+    LogTracer::init().context("Failed to set logger")?;
+
+    let subscriber = Registry::default()
+        .with(EnvFilter::from_default_env())
+        .with(fmt::layer().with_writer(std::io::stderr));
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    let args = Arguments::parse();
+
+    let (report_data, _cert_chain, _priv_key) = make_self_signed_cert()?;
+    if let Err(e) = get_quote_and_collateral(Some(args.my_sgx_allowed_tcb_levels), &report_data) {
+        error!("failed to get quote and collateral: {e:?}");
+        // don't return for now, we can still serve requests but we won't be able to attest
+    }
+
+    let mut vault_1 = args.attestation.clone();
+    let mut vault_2 = args.attestation.clone();
+    let mut vault_3 = args.attestation.clone();
+
+    vault_1.vault_addr = "https://vault-1:8210".to_string();
+    vault_2.vault_addr = "https://vault-2:8210".to_string();
+    vault_3.vault_addr = "https://vault-3:8210".to_string();
+
+    let servers = vec![vault_1.clone(), vault_2.clone(), vault_3.clone()];
+
+    let mut val: usize = 1;
+
+    loop {
+        let mut conns = Vec::new();
+        for server in &servers {
+            match VaultConnection::new(&server.into(), "stress".to_string()).await {
+                Ok(conn) => conns.push(conn),
+                Err(e) => {
+                    error!("connecting to {}: {}", server.vault_addr, e);
+                    continue;
+                }
+            }
+        }
+
+        if conns.is_empty() {
+            error!("no connections");
+            sleep(Duration::from_secs(1)).await;
+            continue;
+        }
+
+        let i = val % conns.len();
+        trace!("storing secret");
+        conns[i]
+            .store_secret(MySecret { val }, "val")
+            .await
+            .context("storing secret")?;
+        for conn in conns {
+            let got: MySecret = conn
+                .load_secret("val")
+                .await
+                .context("loading secret")?
+                .context("loading secret")?;
+            assert_eq!(got.val, val,);
+        }
+        val += 1;
+        sleep(Duration::from_secs(1)).await;
+    }
+}

bin/tee-stress-client/tee-stress-client.manifest.template

@@ -0,0 +1,66 @@
+libos.entrypoint = "/app/tee-stress-client"
+
+[loader]
+argv = [ "/app/tee-stress-client" ]
+entrypoint = "file:{{ gramine.libos }}"
+env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}:/usr{{ arch_libdir }}:/lib"
+env.HOME = "/app"
+env.MALLOC_ARENA_MAX = "1"
+env.AZDCAP_DEBUG_LOG_LEVEL = "ignore"
+env.AZDCAP_COLLATERAL_VERSION = "v4"
+
+### Admin Config ###
+env.PORT  = { passthrough = true }
+
+### VAULT attestation ###
+env.VAULT_ADDR = { passthrough = true }
+env.VAULT_SGX_MRENCLAVE = { passthrough = true }
+env.VAULT_SGX_MRSIGNER = { passthrough = true }
+env.VAULT_SGX_ALLOWED_TCB_LEVELS = { passthrough = true }
+
+### DEBUG ###
+env.RUST_BACKTRACE = "1"
+env.RUST_LOG="info"
+
+[fs]
+root.uri = "file:/"
+start_dir = "/app"
+mounts = [
+  { path = "{{ execdir }}", uri = "file:{{ execdir }}" },
+  { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
+  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
+  { path = "/etc", uri = "file:/etc" },
+  { type = "tmpfs", path = "/var/tmp" },
+  { type = "tmpfs", path = "/tmp" },
+  { type = "tmpfs", path = "/app/.dcap-qcnl" },
+  { type = "tmpfs", path = "/app/.az-dcap-client" },
+  { path = "/lib/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" },
+]
+
+[sgx]
+trusted_files = [
+  "file:/etc/ld.so.cache",
+  "file:/app/",
+  "file:{{ execdir }}/",
+  "file:{{ arch_libdir }}/",
+  "file:/usr/{{ arch_libdir }}/",
+  "file:{{ gramine.libos }}",
+  "file:{{ gramine.runtimedir() }}/",
+  "file:/usr/lib/ssl/openssl.cnf",
+  "file:/etc/ssl/",
+  "file:/etc/sgx_default_qcnl.conf",
+  "file:/lib/libdcap_quoteprov.so",
+]
+remote_attestation = "dcap"
+max_threads = 64
+edmm_enable = false
+## max enclave size
+enclave_size = "8G"
+
+[sys]
+enable_extra_runtime_domain_names_conf = true
+enable_sigterm_injection = true
+
+# possible tweak option, if problems with mio
+# currently mio is compiled with `mio_unsupported_force_waker_pipe`
+# insecure__allow_eventfd = true

bin/tee-vault-admin/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "tee-vault-admin"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+actix-web.workspace = true
+anyhow.workspace = true
+awc.workspace = true
+base64.workspace = true
+bytemuck.workspace = true
+clap.workspace = true
+hex.workspace = true
+mio.workspace = true
+rustls.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+teepot.workspace = true
+tracing-actix-web.workspace = true
+tracing-log.workspace = true
+tracing-subscriber.workspace = true
+tracing.workspace = true

bin/tee-vault-admin/Dockerfile-azure

@@ -0,0 +1,85 @@
+FROM docker.io/ubuntu:20.04 AS azuredcap
+WORKDIR /build
+ADD https://github.com/microsoft/Azure-DCAP-Client/archive/refs/tags/1.12.0.tar.gz ./Azure-DCAP-Client.tar.gz
+RUN tar -xvf Azure-DCAP-Client.tar.gz
+COPY assets/Azure-DCAP-Client.patch ./Azure-DCAP-Client.patch
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y software-properties-common; \
+    add-apt-repository ppa:team-xbmc/ppa -y; \
+    apt-get update; \
+    apt-get install -y \
+        build-essential \
+        cmake \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        pkg-config \
+        nlohmann-json3-dev \
+        wget \
+        dos2unix \
+        ;
+
+WORKDIR /build/Azure-DCAP-Client-1.12.0
+RUN dos2unix src/dcap_provider.cpp && patch -p1 < ../Azure-DCAP-Client.patch
+WORKDIR /build/Azure-DCAP-Client-1.12.0/src/Linux
+RUN ./configure && make && make install
+
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-vault-admin.manifest.template' -av /data/ ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-vault-admin --bin tee-vault-admin \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-vault-admin ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/microsoft.asc https://packages.microsoft.com/keys/microsoft.asc \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.asc] https://packages.microsoft.com/ubuntu/20.04/prod focal main" > /etc/apt/sources.list.d/msprod.list \
+    && apt-get update \
+    && apt purge -y libsgx-dcap-default-qpl \
+    && apt-get install -y az-dcap-client
+
+RUN apt purge -y libsgx-ae-qve
+#    libsgx-urts
+
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /etc/sgx_default_qcnl.conf
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-vault-admin .
+COPY ./bin/tee-vault-admin/tee-vault-admin.manifest.template .
+COPY vault/enclave-key.pem .
+
+# The original Azure library is still delivering expired collateral, so we have to use a patched version
+COPY --from=azuredcap /usr/local/lib/libdcap_quoteprov.so /usr/lib/
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-vault-admin.manifest.template tee-vault-admin.manifest \
+    && gramine-sgx-sign --manifest tee-vault-admin.manifest --output tee-vault-admin.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-vault-admin" ]

bin/tee-vault-admin/Dockerfile-intel

@@ -0,0 +1,58 @@
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-vault-admin.manifest.template' -av /data/ ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-vault-admin --bin tee-vault-admin \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-vault-admin ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        libsgx-dcap-default-qpl \
+        libsgx-urts \
+        libsgx-enclave-common \
+        libsgx-dcap-quote-verify
+RUN apt purge -y libsgx-ae-qve
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /lib/libdcap_quoteprov.so
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-vault-admin .
+COPY ./bin/tee-vault-admin/tee-vault-admin.manifest.template .
+COPY vault/enclave-key.pem .
+
+COPY assets/sgx_default_qcnl.conf.json /etc/sgx_default_qcnl.conf
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-vault-admin.manifest.template tee-vault-admin.manifest \
+    && gramine-sgx-sign --manifest tee-vault-admin.manifest --output tee-vault-admin.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-vault-admin" ]

bin/tee-vault-admin/src/attestation.rs

@@ -0,0 +1,25 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+//! attestation
+
+use crate::ServerState;
+use actix_web::http::StatusCode;
+use actix_web::web::{Data, Json};
+use anyhow::{Context, Result};
+use std::sync::Arc;
+use teepot::json::http::AttestationResponse;
+use teepot::server::attestation::get_quote_and_collateral;
+use teepot::server::{HttpResponseError, Status};
+use tracing::instrument;
+
+/// Get attestation
+#[instrument(level = "info", name = "/v1/sys/attestation", skip_all)]
+pub async fn get_attestation(
+    worker: Data<Arc<ServerState>>,
+) -> Result<Json<AttestationResponse>, HttpResponseError> {
+    get_quote_and_collateral(None, &worker.report_data)
+        .context("Error getting attestation")
+        .map(Json)
+        .status(StatusCode::INTERNAL_SERVER_ERROR)
+}

bin/tee-vault-admin/src/command.rs

@@ -0,0 +1,89 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+//! post commands
+
+use crate::ServerState;
+use actix_web::web;
+use anyhow::{anyhow, Context, Result};
+use awc::http::StatusCode;
+use sha2::{Digest, Sha256};
+use std::sync::Arc;
+use teepot::client::vault::VaultConnection;
+use teepot::json::http::{
+    VaultCommandRequest, VaultCommandResponse, VaultCommands, VaultCommandsResponse,
+};
+use teepot::json::secrets::{AdminConfig, AdminState};
+use teepot::server::{signatures::VerifySig, HttpResponseError, Status};
+use tracing::instrument;
+
+/// Post command
+#[instrument(level = "info", name = "/v1/command", skip_all)]
+pub async fn post_command(
+    state: web::Data<Arc<ServerState>>,
+    item: web::Json<VaultCommandRequest>,
+) -> Result<web::Json<VaultCommandsResponse>, HttpResponseError> {
+    let conn = VaultConnection::new(&state.vault_attestation.clone().into(), "admin".to_string())
+        .await
+        .context("connecting to vault")
+        .status(StatusCode::BAD_GATEWAY)?;
+
+    let mut admin_state: AdminState = conn
+        .load_secret("state")
+        .await?
+        .context("empty admin state")
+        .status(StatusCode::BAD_GATEWAY)?;
+
+    let commands: VaultCommands = serde_json::from_str(&item.commands)
+        .context("parsing commands")
+        .status(StatusCode::BAD_REQUEST)?;
+
+    if admin_state.last_digest.to_ascii_lowercase() != commands.last_digest {
+        return Err(anyhow!(
+            "last digest does not match {} != {}",
+            admin_state.last_digest.to_ascii_lowercase(),
+            commands.last_digest
+        ))
+        .status(StatusCode::BAD_REQUEST);
+    }
+
+    let admin_config: AdminConfig = conn
+        .load_secret("config")
+        .await?
+        .context("empty admin config")
+        .status(StatusCode::BAD_GATEWAY)?;
+    admin_config.check_sigs(&item.signatures, item.commands.as_bytes())?;
+
+    let mut hasher = Sha256::new();
+    hasher.update(item.commands.as_bytes());
+    let hash = hasher.finalize();
+    let digest = hex::encode(hash);
+    admin_state.last_digest = digest.clone();
+    conn.store_secret(admin_state, "state").await?;
+
+    let mut responds = VaultCommandsResponse {
+        digest,
+        results: vec![],
+    };
+
+    for (pos, command) in commands.commands.iter().enumerate() {
+        let resp = conn
+            .vault_put(
+                &format!("Executing command {pos}"),
+                &command.url,
+                &command.data,
+            )
+            .await?;
+
+        let vcr = VaultCommandResponse {
+            status_code: resp.0.as_u16(),
+            value: resp.1,
+        };
+
+        responds.results.push(vcr);
+    }
+
+    let _ = conn.revoke_token().await;
+
+    Ok(web::Json(responds))
+}

bin/tee-vault-admin/src/digest.rs

@@ -0,0 +1,34 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+//! digest
+
+use crate::ServerState;
+use actix_web::{web, HttpResponse};
+use anyhow::{Context, Result};
+use awc::http::StatusCode;
+use serde_json::json;
+use std::sync::Arc;
+use teepot::client::vault::VaultConnection;
+use teepot::json::secrets::AdminState;
+use teepot::server::{HttpResponseError, Status};
+use tracing::instrument;
+
+/// Get last digest
+#[instrument(level = "info", name = "/v1/digest", skip_all)]
+pub async fn get_digest(
+    state: web::Data<Arc<ServerState>>,
+) -> Result<HttpResponse, HttpResponseError> {
+    let conn = VaultConnection::new(&state.vault_attestation.clone().into(), "admin".to_string())
+        .await
+        .context("connecting to vault")
+        .status(StatusCode::BAD_GATEWAY)?;
+
+    let admin_state: AdminState = conn
+        .load_secret("state")
+        .await?
+        .context("empty admin state")
+        .status(StatusCode::BAD_GATEWAY)?;
+
+    Ok(HttpResponse::Ok().json(json!({"last_digest": admin_state.last_digest })))
+}

bin/tee-vault-admin/src/main.rs

@@ -0,0 +1,148 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+//! Server to handle requests to the Vault TEE
+
+#![deny(missing_docs)]
+#![deny(clippy::all)]
+
+mod attestation;
+mod command;
+mod digest;
+mod sign;
+
+use actix_web::web::Data;
+use actix_web::{web, App, HttpServer};
+use anyhow::{Context, Result};
+use attestation::get_attestation;
+use clap::Parser;
+use command::post_command;
+use digest::get_digest;
+use rustls::ServerConfig;
+use sign::post_sign;
+use std::net::Ipv6Addr;
+use std::sync::Arc;
+use teepot::json::http::{SignRequest, VaultCommandRequest, ATTESTATION_URL, DIGEST_URL};
+use teepot::server::attestation::{get_quote_and_collateral, VaultAttestationArgs};
+use teepot::server::new_json_cfg;
+use teepot::server::pki::make_self_signed_cert;
+use teepot::sgx::{parse_tcb_levels, EnumSet, TcbLevel};
+use tracing::{error, info};
+use tracing_actix_web::TracingLogger;
+use tracing_log::LogTracer;
+use tracing_subscriber::Registry;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter};
+
+/// Server state
+pub struct ServerState {
+    /// Server TLS public key hash
+    pub report_data: [u8; 64],
+    /// Vault attestation args
+    pub vault_attestation: VaultAttestationArgs,
+}
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Arguments {
+    /// allowed TCB levels, comma separated
+    #[arg(long, value_parser = parse_tcb_levels, env = "ALLOWED_TCB_LEVELS", default_value = "Ok")]
+    server_sgx_allowed_tcb_levels: EnumSet<TcbLevel>,
+    /// port to listen on
+    #[arg(long, env = "PORT", default_value = "8444")]
+    port: u16,
+    #[clap(flatten)]
+    pub attestation: VaultAttestationArgs,
+}
+
+#[actix_web::main]
+async fn main() -> Result<()> {
+    LogTracer::init().context("Failed to set logger")?;
+
+    let subscriber = Registry::default()
+        .with(EnvFilter::from_default_env())
+        .with(fmt::layer().with_writer(std::io::stderr));
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    let args = Arguments::parse();
+
+    let (report_data, cert_chain, priv_key) = make_self_signed_cert()?;
+
+    if let Err(e) = get_quote_and_collateral(Some(args.server_sgx_allowed_tcb_levels), &report_data)
+    {
+        error!("failed to get quote and collateral: {e:?}");
+        // don't return for now, we can still serve requests but we won't be able to attest
+    }
+
+    // init server config builder with safe defaults
+    let config = ServerConfig::builder()
+        .with_no_client_auth()
+        .with_single_cert([cert_chain].into(), priv_key)
+        .context("Failed to load TLS key/cert files")?;
+
+    info!("Starting HTTPS server at port {}", args.port);
+
+    info!("Quote verified! Connection secure!");
+
+    let server_state = Arc::new(ServerState {
+        report_data,
+        vault_attestation: args.attestation,
+    });
+
+    let server = match HttpServer::new(move || {
+        App::new()
+            // enable logger
+            .wrap(TracingLogger::default())
+            .app_data(new_json_cfg())
+            .app_data(Data::new(server_state.clone()))
+            .service(web::resource(ATTESTATION_URL).route(web::get().to(get_attestation)))
+            .service(web::resource(VaultCommandRequest::URL).route(web::post().to(post_command)))
+            .service(web::resource(SignRequest::URL).route(web::post().to(post_sign)))
+            .service(web::resource(DIGEST_URL).route(web::get().to(get_digest)))
+    })
+    .bind_rustls_0_22((Ipv6Addr::UNSPECIFIED, args.port), config)
+    {
+        Ok(c) => c,
+        Err(e) => {
+            error!("Failed to bind to port {}: {e:?}", args.port);
+            return Err(e).context(format!("Failed to bind to port {}", args.port));
+        }
+    };
+
+    if let Err(e) = server.worker_max_blocking_threads(2).workers(8).run().await {
+        error!("failed to start HTTPS server: {e:?}");
+        return Err(e).context("Failed to start HTTPS server");
+    }
+
+    Ok(())
+}
+
+#[cfg(test)]
+mod tests {
+    use serde_json::json;
+    use teepot::json::http::{VaultCommand, VaultCommands};
+
+    const TEST_DATA: &str = include_str!("../../../tests/data/test.json");
+
+    #[test]
+    fn test_vault_commands() {
+        let cmd = VaultCommand {
+            url: "/v1/auth/tee/tees/test".to_string(),
+            data: json!({
+                "lease": "1000",
+                "name": "test",
+                "types": "sgx",
+                "sgx_allowed_tcb_levels": "Ok,SwHardeningNeeded",
+                "sgx_mrsigner": "c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d",
+                "token_policies": "test"
+            }),
+        };
+        let cmds = VaultCommands {
+            commands: vec![cmd],
+            last_digest: "".into(),
+        };
+
+        let test_data_cmds: VaultCommands = serde_json::from_str(TEST_DATA).unwrap();
+
+        assert_eq!(cmds, test_data_cmds);
+    }
+}

bin/tee-vault-admin/src/sign.rs

@@ -0,0 +1,149 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+//! post signing request
+
+use crate::ServerState;
+use actix_web::http::StatusCode;
+use actix_web::web;
+use anyhow::{anyhow, Context, Result};
+use sha2::{Digest, Sha256};
+use std::sync::Arc;
+use teepot::client::vault::VaultConnection;
+use teepot::json::http::{SignRequest, SignRequestData, SignResponse};
+use teepot::json::secrets::{AdminConfig, AdminState, SGXSigningKey};
+use teepot::server::signatures::VerifySig as _;
+use teepot::server::{HttpResponseError, Status};
+use teepot::sgx::sign::PrivateKey as _;
+use teepot::sgx::sign::{Author, Signature};
+use teepot::sgx::sign::{Body, RS256PrivateKey};
+use tracing::instrument;
+
+/// Sign command
+#[instrument(level = "info", name = "/v1/sign", skip_all)]
+pub async fn post_sign(
+    state: web::Data<Arc<ServerState>>,
+    item: web::Json<SignRequest>,
+) -> Result<web::Json<SignResponse>, HttpResponseError> {
+    let conn = VaultConnection::new(&state.vault_attestation.clone().into(), "admin".to_string())
+        .await
+        .context("connecting to vault")
+        .status(StatusCode::BAD_GATEWAY)?;
+
+    let mut admin_state: AdminState = conn
+        .load_secret("state")
+        .await?
+        .context("empty admin state")
+        .status(StatusCode::BAD_GATEWAY)?;
+
+    let sign_request: SignRequestData = serde_json::from_str(&item.sign_request_data)
+        .context("parsing sign request data")
+        .status(StatusCode::BAD_REQUEST)?;
+
+    // Sanity checks
+    if sign_request.tee_type != "sgx" {
+        return Err(anyhow!("tee_type not supported")).status(StatusCode::BAD_REQUEST);
+    }
+
+    let tee_name = sign_request.tee_name;
+
+    if !tee_name.is_ascii() {
+        return Err(anyhow!("tee_name is not ascii")).status(StatusCode::BAD_REQUEST);
+    }
+
+    // check if tee_name is alphanumeric
+    if !tee_name.chars().all(|c| c.is_alphanumeric()) {
+        return Err(anyhow!("tee_name is not alphanumeric")).status(StatusCode::BAD_REQUEST);
+    }
+
+    // check if tee_name starts with an alphabetic char
+    if !tee_name.chars().next().unwrap().is_alphabetic() {
+        return Err(anyhow!("tee_name does not start with an alphabetic char"))
+            .status(StatusCode::BAD_REQUEST);
+    }
+
+    if admin_state.last_digest != sign_request.last_digest {
+        return Err(anyhow!(
+            "last digest does not match {} != {}",
+            admin_state.last_digest.to_ascii_lowercase(),
+            sign_request.last_digest
+        ))
+        .status(StatusCode::BAD_REQUEST);
+    }
+
+    let admin_config: AdminConfig = conn
+        .load_secret("config")
+        .await?
+        .context("empty admin config")
+        .status(StatusCode::BAD_GATEWAY)?;
+    admin_config.check_sigs(&item.signatures, item.sign_request_data.as_bytes())?;
+
+    let mut hasher = Sha256::new();
+    hasher.update(item.sign_request_data.as_bytes());
+    let hash = hasher.finalize();
+    let digest = hex::encode(hash);
+    admin_state.last_digest = digest.clone();
+    conn.store_secret(admin_state, "state").await?;
+
+    // Sign SGX enclave
+    let key_path = format!("signing_keys/{}", tee_name);
+
+    let sgx_key = match conn
+        .load_secret::<SGXSigningKey>(&key_path)
+        .await
+        .context("Error loading signing key")
+        .status(StatusCode::INTERNAL_SERVER_ERROR)?
+    {
+        Some(key) => RS256PrivateKey::from_pem(&key.pem_pk)
+            .context("Failed to parse private key")
+            .status(StatusCode::INTERNAL_SERVER_ERROR)?,
+        None => {
+            let private_key = RS256PrivateKey::generate(3)
+                .context("Failed to generate private key")
+                .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+            let pem_pk = private_key
+                .to_pem()
+                .context("Failed to convert private key to pem")
+                .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+            let key = SGXSigningKey { pem_pk };
+
+            conn.store_secret(key.clone(), &key_path)
+                .await
+                .context("Error storing generated private key")
+                .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+            private_key
+        }
+    };
+
+    let signed_data = sign_sgx(&sign_request.data, &sgx_key)?;
+    let respond = SignResponse {
+        digest,
+        signed_data,
+    };
+
+    let _ = conn.revoke_token().await;
+
+    Ok(web::Json(respond))
+}
+
+fn sign_sgx(body_bytes: &[u8], sgx_key: &RS256PrivateKey) -> Result<Vec<u8>, HttpResponseError> {
+    let body: Body = bytemuck::try_pod_read_unaligned(body_bytes)
+        .context("Invalid SGX input data")
+        .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+    if body.can_set_debug() {
+        return Err(anyhow!("Not signing SGX enclave with debug flag"))
+            .status(StatusCode::BAD_REQUEST);
+    }
+
+    // FIXME: do we need the date and sw defined value?
+    let author = Author::new(0, 0);
+    let sig = Signature::new(sgx_key, author, body)
+        .context("Failed to create RSA signature")
+        .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+    Ok(bytemuck::bytes_of(&sig).to_vec())
+}

bin/tee-vault-admin/tee-vault-admin.manifest.template

@@ -0,0 +1,66 @@
+libos.entrypoint = "/app/tee-vault-admin"
+
+[loader]
+argv = [ "/app/tee-vault-admin" ]
+entrypoint = "file:{{ gramine.libos }}"
+env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}:/usr{{ arch_libdir }}:/lib"
+env.HOME = "/app"
+env.MALLOC_ARENA_MAX = "1"
+env.AZDCAP_DEBUG_LOG_LEVEL = "ignore"
+env.AZDCAP_COLLATERAL_VERSION = "v4"
+
+### Admin Config ###
+env.PORT  = { passthrough = true }
+
+### VAULT attestation ###
+env.VAULT_ADDR = { passthrough = true }
+env.VAULT_SGX_MRENCLAVE = { passthrough = true }
+env.VAULT_SGX_MRSIGNER = { passthrough = true }
+env.VAULT_SGX_ALLOWED_TCB_LEVELS = { passthrough = true }
+
+### DEBUG ###
+env.RUST_BACKTRACE = "1"
+env.RUST_LOG="info,tee_vault_admin=trace,teepot=trace,vault_tee_client=trace,tee_client=trace,awc=debug"
+
+[fs]
+root.uri = "file:/"
+start_dir = "/app"
+mounts = [
+  { path = "{{ execdir }}", uri = "file:{{ execdir }}" },
+  { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
+  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
+  { path = "/etc", uri = "file:/etc" },
+  { type = "tmpfs", path = "/var/tmp" },
+  { type = "tmpfs", path = "/tmp" },
+  { type = "tmpfs", path = "/app/.dcap-qcnl" },
+  { type = "tmpfs", path = "/app/.az-dcap-client" },
+  { path = "/lib/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" },
+]
+
+[sgx]
+trusted_files = [
+  "file:/etc/ld.so.cache",
+  "file:/app/",
+  "file:{{ execdir }}/",
+  "file:{{ arch_libdir }}/",
+  "file:/usr/{{ arch_libdir }}/",
+  "file:{{ gramine.libos }}",
+  "file:{{ gramine.runtimedir() }}/",
+  "file:/usr/lib/ssl/openssl.cnf",
+  "file:/etc/ssl/",
+  "file:/etc/sgx_default_qcnl.conf",
+  "file:/lib/libdcap_quoteprov.so",
+]
+remote_attestation = "dcap"
+max_threads = 64
+edmm_enable = false
+## max enclave size
+enclave_size = "8G"
+
+[sys]
+enable_extra_runtime_domain_names_conf = true
+enable_sigterm_injection = true
+
+# possible tweak option, if problems with mio
+# currently mio is compiled with `mio_unsupported_force_waker_pipe`
+# insecure__allow_eventfd = true

bin/tee-vault-unseal/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "tee-vault-unseal"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+actix-tls.workspace = true
+actix-web.workspace = true
+anyhow.workspace = true
+awc.workspace = true
+clap.workspace = true
+hex.workspace = true
+mio.workspace = true
+rustls-pemfile.workspace = true
+rustls.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+teepot.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+tracing-log.workspace = true
+tracing-subscriber.workspace = true
+tracing.workspace = true
+x509-cert.workspace = true

bin/tee-vault-unseal/Dockerfile-azure

@@ -0,0 +1,92 @@
+FROM ghcr.io/matter-labs/vault-auth-tee:latest AS vault-auth-tee
+
+FROM docker.io/ubuntu:20.04 AS azuredcap
+WORKDIR /build
+ADD https://github.com/microsoft/Azure-DCAP-Client/archive/refs/tags/1.12.0.tar.gz ./Azure-DCAP-Client.tar.gz
+RUN tar -xvf Azure-DCAP-Client.tar.gz
+COPY assets/Azure-DCAP-Client.patch ./Azure-DCAP-Client.patch
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y software-properties-common; \
+    add-apt-repository ppa:team-xbmc/ppa -y; \
+    apt-get update; \
+    apt-get install -y \
+        build-essential \
+        cmake \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        pkg-config \
+        nlohmann-json3-dev \
+        wget \
+        dos2unix \
+        ;
+
+WORKDIR /build/Azure-DCAP-Client-1.12.0
+RUN dos2unix src/dcap_provider.cpp && patch -p1 < ../Azure-DCAP-Client.patch
+WORKDIR /build/Azure-DCAP-Client-1.12.0/src/Linux
+RUN ./configure && make && make install
+
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+COPY --from=vault-auth-tee /opt/vault/plugins/vault-auth-tee ./
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-vault-unseal.manifest.template' -av /data/ ./
+RUN sha256sum /opt/vault/plugins/vault-auth-tee | ( read a _ ; echo -n $a ) | tee assets/vault-auth-tee.sha256
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-vault-unseal --bin tee-vault-unseal \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-vault-unseal ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/microsoft.asc https://packages.microsoft.com/keys/microsoft.asc \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.asc] https://packages.microsoft.com/ubuntu/20.04/prod focal main" > /etc/apt/sources.list.d/msprod.list \
+    && apt-get update \
+    && apt purge -y libsgx-dcap-default-qpl \
+    && apt-get install -y az-dcap-client
+
+RUN apt purge -y libsgx-ae-qve
+#    libsgx-urts
+
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /etc/sgx_default_qcnl.conf
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-vault-unseal .
+COPY ./bin/tee-vault-unseal/tee-vault-unseal.manifest.template .
+COPY vault/enclave-key.pem .
+RUN mkdir -p /opt/vault/tls && rm -rf /opt/vault/tls/*
+
+# The original Azure library is still delivering expired collateral, so we have to use a patched version
+COPY --from=azuredcap /usr/local/lib/libdcap_quoteprov.so /usr/lib/
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-vault-unseal.manifest.template tee-vault-unseal.manifest \
+    && gramine-sgx-sign --manifest tee-vault-unseal.manifest --output tee-vault-unseal.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+VOLUME /opt/vault/tls
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-vault-unseal" ]

bin/tee-vault-unseal/Dockerfile-intel

@@ -0,0 +1,65 @@
+FROM ghcr.io/matter-labs/vault-auth-tee:latest AS vault-auth-tee
+
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+COPY --from=vault-auth-tee /opt/vault/plugins/vault-auth-tee ./
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-vault-unseal.manifest.template' -av /data/ ./
+RUN sha256sum /opt/vault/plugins/vault-auth-tee | ( read a _ ; echo -n $a ) | tee assets/vault-auth-tee.sha256
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-vault-unseal --bin tee-vault-unseal \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-vault-unseal ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        libsgx-dcap-default-qpl \
+        libsgx-urts \
+        libsgx-enclave-common \
+        libsgx-dcap-quote-verify
+RUN apt purge -y libsgx-ae-qve
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /lib/libdcap_quoteprov.so
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-vault-unseal .
+COPY ./bin/tee-vault-unseal/tee-vault-unseal.manifest.template .
+COPY vault/enclave-key.pem .
+RUN mkdir -p /opt/vault/tls && rm -rf /opt/vault/tls/*
+
+COPY assets/sgx_default_qcnl.conf.json /etc/sgx_default_qcnl.conf
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-vault-unseal.manifest.template tee-vault-unseal.manifest \
+    && gramine-sgx-sign --manifest tee-vault-unseal.manifest --output tee-vault-unseal.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+VOLUME /opt/vault/tls
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-vault-unseal" ]

bin/tee-vault-unseal/src/admin-policy.hcl

@@ -0,0 +1,80 @@
+# Read system health check
+path "sys/health"
+{
+  capabilities = ["read", "sudo"]
+}
+
+# Create and manage ACL policies broadly across Vault
+
+# List existing policies
+path "sys/policies/acl"
+{
+  capabilities = ["list"]
+}
+
+# Create and manage ACL policies
+path "sys/policies/acl/*"
+{
+  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+}
+
+# Enable and manage authentication methods broadly across Vault
+
+# Manage auth methods broadly across Vault
+path "auth/*"
+{
+  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+}
+
+# Create, update, and delete auth methods
+path "sys/auth/*"
+{
+  capabilities = ["create", "update", "delete", "sudo"]
+}
+
+# List auth methods
+path "sys/auth"
+{
+  capabilities = ["read"]
+}
+
+# Enable and manage the key/value secrets engine at `secret/` path
+
+# List, create, update, and delete key/value secrets
+path "secret/*"
+{
+  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+}
+
+# Manage secrets engines
+path "sys/mounts/*"
+{
+  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+}
+
+# List existing secrets engines.
+path "sys/mounts"
+{
+  capabilities = ["read"]
+}
+
+# Manage plugins
+# https://developer.hashicorp.com/vault/api-docs/system/plugins-catalog
+path "sys/plugins/catalog/*"
+{
+  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+}
+
+# List existing plugins
+# https://developer.hashicorp.com/vault/api-docs/system/plugins-catalog
+path "sys/plugins/catalog"
+{
+  capabilities = ["list"]
+}
+
+# Reload plugins
+# https://developer.hashicorp.com/vault/api-docs/system/plugins-reload-backend
+path "sys/plugins/reload/backend"
+{
+  capabilities = ["create", "update", "sudo"]
+}

bin/tee-vault-unseal/src/attestation.rs

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+use crate::Worker;
+use actix_web::http::StatusCode;
+use actix_web::web::{Data, Json};
+use anyhow::{Context, Result};
+use teepot::json::http::AttestationResponse;
+use teepot::server::attestation::get_quote_and_collateral;
+use teepot::server::{HttpResponseError, Status};
+use tracing::instrument;
+
+#[instrument(level = "info", name = "/v1/sys/attestation", skip_all)]
+pub async fn get_attestation(
+    worker: Data<Worker>,
+) -> Result<Json<AttestationResponse>, HttpResponseError> {
+    let report_data: [u8; 64] = worker
+        .config
+        .report_data
+        .clone()
+        .try_into()
+        .map_err(|_| "Error getting attestation")?;
+    get_quote_and_collateral(None, &report_data)
+        .context("Error getting attestation")
+        .map(Json)
+        .status(StatusCode::INTERNAL_SERVER_ERROR)
+}

bin/tee-vault-unseal/src/init.rs

@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+use crate::{create_https_client, get_vault_status, UnsealServerState, Worker};
+use actix_web::error::ErrorBadRequest;
+use actix_web::{web, HttpResponse};
+use anyhow::{anyhow, Context, Result};
+use awc::http::StatusCode;
+use serde_json::json;
+use teepot::json::http::{Init, InitResponse, VaultInitRequest};
+use teepot::json::secrets::AdminConfig;
+use teepot::server::{HttpResponseError, Status};
+use tracing::{debug, error, info, instrument, trace};
+
+#[instrument(level = "info", name = "/v1/sys/init", skip_all)]
+pub async fn post_init(
+    worker: web::Data<Worker>,
+    init: web::Json<Init>,
+) -> Result<HttpResponse, HttpResponseError> {
+    let Init {
+        pgp_keys,
+        secret_shares,
+        secret_threshold,
+        admin_pgp_keys,
+        admin_threshold,
+        admin_tee_mrenclave,
+    } = init.into_inner();
+    let client = create_https_client(worker.client_tls_config.clone());
+    let vault_url = &worker.config.vault_url;
+
+    let vault_init = VaultInitRequest {
+        pgp_keys,
+        secret_shares,
+        secret_threshold,
+    };
+
+    if admin_threshold < 1 {
+        return Ok(HttpResponse::from_error(ErrorBadRequest(
+            json!({"error": "admin_threshold must be at least 1"}),
+        )));
+    }
+
+    if admin_threshold > admin_pgp_keys.len() {
+        return Ok(HttpResponse::from_error(ErrorBadRequest(
+            json!({"error": "admin_threshold must be less than or equal to the number of admin_pgp_keys"}),
+        )));
+    }
+
+    loop {
+        let current_state = worker.state.read().unwrap().clone();
+        match current_state {
+            UnsealServerState::VaultUninitialized => {
+                break;
+            }
+            UnsealServerState::VaultUnsealed => {
+                return Err(anyhow!("Vault already unsealed")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultInitialized { .. } => {
+                return Err(anyhow!("Vault already initialized")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultInitializedAndConfigured => {
+                return Err(anyhow!("Vault already initialized")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::Undefined => {
+                let state = get_vault_status(vault_url, client.clone()).await;
+                *worker.state.write().unwrap() = state;
+                continue;
+            }
+        }
+    }
+
+    trace!(
+        "Sending init request to Vault {}",
+        serde_json::to_string(&vault_init).unwrap()
+    );
+    let mut response = client
+        .post(format!("{}/v1/sys/init", vault_url))
+        .send_json(&vault_init)
+        .await?;
+
+    let status_code = response.status();
+    if !status_code.is_success() {
+        error!("Vault returned server error: {}", status_code);
+        return Err(HttpResponseError::from_proxy(response).await);
+    }
+
+    let response = response
+        .json::<serde_json::Value>()
+        .await
+        .context("Failed to convert to json")
+        .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+    info!("Vault initialized");
+    trace!("response {}", response);
+
+    let root_token = response["root_token"]
+        .as_str()
+        .ok_or(anyhow!("No `root_token` field"))
+        .status(StatusCode::BAD_GATEWAY)?
+        .to_string();
+
+    debug!("Root token: {root_token}");
+
+    let unseal_keys = response["keys_base64"]
+        .as_array()
+        .ok_or(anyhow!("No `keys_base64` field"))
+        .status(StatusCode::BAD_GATEWAY)?
+        .iter()
+        .map(|v| v.as_str().unwrap().to_string())
+        .collect::<Vec<_>>();
+
+    debug!("Unseal keys: {}", unseal_keys.join(", "));
+
+    /*
+    FIXME: use unseal keys to create new token
+        let mut output = File::create("/opt/vault/data/root_token")
+            .context("Failed to create `/opt/vault/data/root_token`")?;
+        output
+            .write_all(root_token.as_bytes())
+            .context("Failed to write root_token")?;
+    */
+
+    *worker.state.write().unwrap() = UnsealServerState::VaultInitialized {
+        admin_config: AdminConfig {
+            admin_pgp_keys,
+            admin_threshold,
+        },
+        admin_tee_mrenclave,
+        root_token,
+    };
+
+    let response = InitResponse { unseal_keys };
+
+    Ok(HttpResponse::Ok().json(response)) // <- send response
+}

bin/tee-vault-unseal/src/main.rs

@@ -0,0 +1,366 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+//! Server to initialize and unseal the Vault TEE.
+
+#![deny(missing_docs)]
+#![deny(clippy::all)]
+
+mod attestation;
+mod init;
+mod unseal;
+
+use actix_web::http::header;
+use actix_web::rt::time::sleep;
+use actix_web::web::Data;
+use actix_web::{web, App, HttpServer};
+use anyhow::{Context, Result};
+use attestation::get_attestation;
+use awc::{Client, Connector};
+use clap::Parser;
+use init::post_init;
+use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};
+use rustls::client::WebPkiServerVerifier;
+use rustls::pki_types::{CertificateDer, ServerName, UnixTime};
+use rustls::{ClientConfig, DigitallySignedStruct, Error, ServerConfig, SignatureScheme};
+use rustls_pemfile::{certs, read_one};
+use sha2::{Digest, Sha256};
+use std::fmt::Debug;
+use std::net::Ipv6Addr;
+use std::sync::{Arc, RwLock};
+use std::time::Duration;
+use std::{fs::File, io::BufReader};
+use teepot::json::http::{Init, Unseal, ATTESTATION_URL};
+use teepot::json::secrets::AdminConfig;
+use teepot::server::attestation::get_quote_and_collateral;
+use teepot::server::new_json_cfg;
+use teepot::sgx::{parse_tcb_levels, EnumSet, TcbLevel};
+use tracing::{error, info, trace};
+use tracing_log::LogTracer;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter, Registry};
+use unseal::post_unseal;
+use x509_cert::der::Decode as _;
+use x509_cert::der::Encode as _;
+use x509_cert::Certificate;
+
+const VAULT_AUTH_TEE_SHA256: &str = include_str!("../../../assets/vault-auth-tee.sha256");
+const VAULT_TOKEN_HEADER: &str = "X-Vault-Token";
+
+/// Worker thread state and data
+pub struct Worker {
+    /// TLS config for the HTTPS client
+    pub client_tls_config: Arc<ClientConfig>,
+    /// Server config
+    pub config: Arc<UnsealServerConfig>,
+    /// Server state
+    pub state: Arc<RwLock<UnsealServerState>>,
+}
+
+/// Global Server config
+#[derive(Debug, Default)]
+pub struct UnsealServerConfig {
+    /// Vault URL
+    pub vault_url: String,
+    /// The expected report_data for the Vault TEE
+    pub report_data: Vec<u8>,
+    /// allowed TCB levels
+    pub allowed_tcb_levels: Option<EnumSet<TcbLevel>>,
+}
+
+/// Server state
+#[derive(Debug, Clone)]
+pub enum UnsealServerState {
+    /// Undefined
+    Undefined,
+    /// Vault is not yet initialized
+    VaultUninitialized,
+    /// Vault is initialized but not unsealed
+    VaultInitialized {
+        /// config for the admin TEE
+        admin_config: AdminConfig,
+        /// initial admin TEE mrenclave
+        admin_tee_mrenclave: String,
+        /// Vault root token
+        root_token: String,
+    },
+    /// Vault is already initialized but not unsealed
+    /// and should already be configured
+    VaultInitializedAndConfigured,
+    /// Vault is unsealed
+    VaultUnsealed,
+}
+
+impl UnsealServerConfig {
+    /// Create a new ServerState
+    pub fn new(
+        vault_url: String,
+        report_data: [u8; 64],
+        allowed_tcb_levels: Option<EnumSet<TcbLevel>>,
+    ) -> Self {
+        Self {
+            report_data: report_data.to_vec(),
+            vault_url,
+            allowed_tcb_levels,
+        }
+    }
+}
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+    /// allowed TCB levels, comma separated
+    #[arg(long, value_parser = parse_tcb_levels, env = "ALLOWED_TCB_LEVELS", default_value = "Ok")]
+    allowed_tcb_levels: EnumSet<TcbLevel>,
+    /// port to listen on
+    #[arg(long, env = "PORT", default_value = "8443")]
+    port: u16,
+    /// vault url
+    #[arg(long, env = "VAULT_ADDR", default_value = "https://vault:8210")]
+    vault_url: String,
+}
+
+#[actix_web::main]
+async fn main() -> Result<()> {
+    LogTracer::init().context("Failed to set logger")?;
+
+    let subscriber = Registry::default()
+        .with(EnvFilter::from_default_env())
+        .with(
+            fmt::layer()
+                .with_span_events(fmt::format::FmtSpan::NEW)
+                .with_writer(std::io::stderr),
+        );
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    let args = Args::parse();
+
+    let tls_ok = std::path::Path::new("/opt/vault/tls/tls.ok");
+    loop {
+        info!("Waiting for TLS key/cert files to be generated");
+
+        // Wait for the file `data/tls.key` to exist
+        if tls_ok.exists() {
+            break;
+        }
+        sleep(Duration::from_secs(1)).await;
+    }
+
+    info!("Starting up");
+
+    let (config, client_tls_config, report_data) = load_rustls_config().or_else(|e| {
+        error!("failed to load rustls config: {e:?}");
+        Err(e).context("Failed to load rustls config")
+    })?;
+
+    if let Err(e) = get_quote_and_collateral(Some(args.allowed_tcb_levels), &report_data) {
+        error!("failed to get quote and collateral: {e:?}");
+        // don't return for now, we can still serve requests but we won't be able to attest
+    }
+
+    let client = create_https_client(client_tls_config.clone());
+
+    let server_state = get_vault_status(&args.vault_url, client).await;
+
+    info!("Starting HTTPS server at port {}", args.port);
+    let server_config = Arc::new(UnsealServerConfig::new(
+        args.vault_url,
+        report_data,
+        Some(args.allowed_tcb_levels),
+    ));
+
+    let server_state = Arc::new(RwLock::new(server_state));
+
+    let server = match HttpServer::new(move || {
+        let worker = Worker {
+            client_tls_config: client_tls_config.clone(),
+            config: server_config.clone(),
+            state: server_state.clone(),
+        };
+
+        App::new()
+            // enable logger
+            //.wrap(TracingLogger::default())
+            .app_data(new_json_cfg())
+            .app_data(Data::new(worker))
+            .service(web::resource(ATTESTATION_URL).route(web::get().to(get_attestation)))
+            .service(web::resource(Init::URL).route(web::post().to(post_init)))
+            .service(web::resource(Unseal::URL).route(web::post().to(post_unseal)))
+    })
+    .bind_rustls_0_22((Ipv6Addr::UNSPECIFIED, args.port), config)
+    {
+        Ok(c) => c,
+        Err(e) => {
+            error!("Failed to bind to port {}: {e:?}", args.port);
+            return Err(e).context(format!("Failed to bind to port {}", args.port));
+        }
+    };
+
+    if let Err(e) = server.worker_max_blocking_threads(2).workers(8).run().await {
+        error!("failed to start HTTPS server: {e:?}");
+        return Err(e).context("Failed to start HTTPS server");
+    }
+
+    Ok(())
+}
+
+async fn get_vault_status(vault_url: &str, client: Client) -> UnsealServerState {
+    loop {
+        let r = client
+            .get(format!("{}/v1/sys/health", vault_url))
+            .send()
+            .await;
+
+        if let Ok(r) = r {
+            // https://developer.hashicorp.com/vault/api-docs/system/health
+            match r.status().as_u16() {
+                200 | 429 | 472 | 473 => {
+                    info!("Vault is initialized and unsealed");
+                    break UnsealServerState::VaultUnsealed;
+                }
+                501 => {
+                    info!("Vault is not initialized");
+                    break UnsealServerState::VaultUninitialized;
+                }
+                503 => {
+                    info!("Vault is initialized but not unsealed");
+                    break UnsealServerState::VaultInitializedAndConfigured;
+                }
+                s => {
+                    error!("Vault is not ready: status code {s}");
+                }
+            }
+        }
+        info!("Waiting for vault to be ready");
+        sleep(Duration::from_secs(1)).await;
+    }
+}
+
+// Save the hash of the public server key to `REPORT_DATA` to check
+// the attestations against it and it does not change on reconnect.
+fn make_verifier(server_cert: Box<[u8]>) -> impl ServerCertVerifier {
+    #[derive(Debug)]
+    struct V {
+        server_cert: Box<[u8]>,
+        server_verifier: Arc<WebPkiServerVerifier>,
+    }
+    impl ServerCertVerifier for V {
+        fn verify_server_cert(
+            &self,
+            end_entity: &CertificateDer,
+            _intermediates: &[CertificateDer],
+            _server_name: &ServerName,
+            _ocsp_response: &[u8],
+            _now: UnixTime,
+        ) -> std::result::Result<ServerCertVerified, Error> {
+            let data = &self.server_cert;
+
+            if data.as_ref() == end_entity.as_ref() {
+                info!("Server certificate matches expected certificate");
+                Ok(ServerCertVerified::assertion())
+            } else {
+                error!("Server certificate does not match expected certificate");
+                Err(rustls::Error::General(
+                    "Server certificate does not match expected certificate".to_string(),
+                ))
+            }
+        }
+        fn verify_tls12_signature(
+            &self,
+            message: &[u8],
+            cert: &CertificateDer<'_>,
+            dss: &DigitallySignedStruct,
+        ) -> std::result::Result<HandshakeSignatureValid, Error> {
+            self.server_verifier
+                .verify_tls12_signature(message, cert, dss)
+        }
+
+        fn verify_tls13_signature(
+            &self,
+            message: &[u8],
+            cert: &CertificateDer<'_>,
+            dss: &DigitallySignedStruct,
+        ) -> std::result::Result<HandshakeSignatureValid, Error> {
+            self.server_verifier
+                .verify_tls13_signature(message, cert, dss)
+        }
+
+        fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
+            self.server_verifier.supported_verify_schemes()
+        }
+    }
+    let root_store = Arc::new(rustls::RootCertStore::empty());
+    let server_verifier = WebPkiServerVerifier::builder(root_store).build().unwrap();
+    V {
+        server_cert,
+        server_verifier,
+    }
+}
+
+/// Load TLS key/cert files
+pub fn load_rustls_config() -> Result<(ServerConfig, Arc<ClientConfig>, [u8; 64])> {
+    // init server config builder with safe defaults
+    let config = ServerConfig::builder().with_no_client_auth();
+
+    // load TLS key/cert files
+    let cert_file = &mut BufReader::new(
+        File::open("/opt/vault/tls/tls.crt").context("Failed to open TLS cert file")?,
+    );
+    let key_file = &mut BufReader::new(
+        File::open("/opt/vault/tls/tls.key").context("Failed to open TLS key file")?,
+    );
+
+    // convert files to key/cert objects
+    let cert_chain: Vec<_> = certs(cert_file)
+        .unwrap()
+        .into_iter()
+        .map(rustls::pki_types::CertificateDer::from)
+        .collect();
+    let priv_key: rustls::pki_types::PrivateKeyDer = match read_one(key_file).unwrap() {
+        Some(rustls_pemfile::Item::RSAKey(key)) => {
+            rustls::pki_types::PrivatePkcs1KeyDer::from(key).into()
+        }
+        Some(rustls_pemfile::Item::PKCS8Key(key)) => {
+            rustls::pki_types::PrivatePkcs8KeyDer::from(key).into()
+        }
+        _ => panic!("no keys found"),
+    };
+
+    let tls_config = Arc::new(
+        rustls::ClientConfig::builder()
+            .dangerous()
+            .with_custom_certificate_verifier(Arc::new(make_verifier(
+                cert_chain[0].as_ref().into(),
+            )))
+            .with_no_client_auth(),
+    );
+
+    let cert = Certificate::from_der(cert_chain[0].as_ref()).unwrap();
+    let pub_key = cert
+        .tbs_certificate
+        .subject_public_key_info
+        .to_der()
+        .unwrap();
+
+    let hash = Sha256::digest(pub_key);
+    let mut report_data = [0u8; 64];
+    report_data[..32].copy_from_slice(&hash[..32]);
+
+    let report_data_hex = hex::encode(report_data);
+    trace!(report_data_hex);
+
+    let config = config
+        .with_single_cert(cert_chain, priv_key)
+        .context("Failed to load TLS key/cert files")?;
+
+    Ok((config, tls_config, report_data))
+}
+
+/// Create an HTTPS client with the default headers and config
+pub fn create_https_client(client_tls_config: Arc<ClientConfig>) -> Client {
+    Client::builder()
+        .add_default_header((header::USER_AGENT, "teepot/1.0"))
+        // a "connector" wraps the stream into an encrypted connection
+        .connector(Connector::new().rustls_0_22(client_tls_config))
+        .timeout(Duration::from_secs(12000))
+        .finish()
+}

bin/tee-vault-unseal/src/unseal.rs

@@ -0,0 +1,412 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+use crate::{
+    create_https_client, get_vault_status, UnsealServerConfig, UnsealServerState, Worker,
+    VAULT_AUTH_TEE_SHA256, VAULT_TOKEN_HEADER,
+};
+use actix_web::http::StatusCode;
+use actix_web::rt::time::sleep;
+use actix_web::{web, HttpResponse};
+use anyhow::{anyhow, Context, Result};
+use awc::{Client, ClientRequest, SendClientRequest};
+use serde_json::{json, Value};
+use std::fs::File;
+use std::future::Future;
+use std::io::Read;
+use std::time::Duration;
+use teepot::client::vault::VaultConnection;
+use teepot::json::http::Unseal;
+use teepot::json::secrets::{AdminConfig, AdminState};
+use teepot::server::{HttpResponseError, Status};
+use tracing::{debug, error, info, instrument, trace};
+
+#[instrument(level = "info", name = "/v1/sys/unseal", skip_all)]
+pub async fn post_unseal(
+    worker: web::Data<Worker>,
+    item: web::Json<Unseal>,
+) -> Result<HttpResponse, HttpResponseError> {
+    let client = create_https_client(worker.client_tls_config.clone());
+    let app = &worker.config;
+    let vault_url = &app.vault_url;
+
+    loop {
+        let current_state = worker.state.read().unwrap().clone();
+        match current_state {
+            UnsealServerState::VaultUninitialized => {
+                return Err(anyhow!("Vault not yet initialized")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultUnsealed => {
+                return Err(anyhow!("Vault already unsealed")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultInitialized { .. } => {
+                break;
+            }
+            UnsealServerState::VaultInitializedAndConfigured => {
+                break;
+            }
+            UnsealServerState::Undefined => {
+                let state = get_vault_status(vault_url, client.clone()).await;
+                *worker.state.write().unwrap() = state;
+                continue;
+            }
+        }
+    }
+
+    let mut response = client
+        .post(format!("{}/v1/sys/unseal", vault_url))
+        .send_json(&item.0)
+        .await?;
+
+    let status_code = response.status();
+    if !status_code.is_success() {
+        error!("Vault returned server error: {}", status_code);
+        let mut client_resp = HttpResponse::build(status_code);
+        for (header_name, header_value) in response.headers().iter() {
+            client_resp.insert_header((header_name.clone(), header_value.clone()));
+        }
+        return Ok(client_resp.streaming(response));
+    }
+
+    let response: Value = response
+        .json()
+        .await
+        .context("parsing unseal response")
+        .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+    debug!("unseal: {:?}", response);
+
+    if response.get("errors").is_some() {
+        return Ok(HttpResponse::Ok().json(response));
+    }
+
+    let sealed = response
+        .get("sealed")
+        .map(|v| v.as_bool().unwrap_or(true))
+        .unwrap_or(true);
+
+    debug!(sealed);
+
+    // if unsealed
+    if !sealed {
+        let mut state = UnsealServerState::VaultUnsealed;
+        std::mem::swap(&mut *worker.state.write().unwrap(), &mut state);
+
+        match state {
+            UnsealServerState::VaultUninitialized => {
+                return Err(anyhow!("Invalid internal state")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultUnsealed => {
+                return Err(anyhow!("Invalid internal state")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultInitialized {
+                admin_config,
+                admin_tee_mrenclave,
+                root_token,
+            } => {
+                debug!(root_token);
+                info!("Vault is unsealed");
+                let app = &worker.config;
+                let client = create_https_client(worker.client_tls_config.clone());
+
+                vault_configure_unsealed(
+                    app,
+                    &admin_config,
+                    &root_token,
+                    &admin_tee_mrenclave,
+                    &client,
+                )
+                .await
+                .context("Failed to configure unsealed vault")
+                .status(StatusCode::BAD_GATEWAY)?;
+
+                // destroy root token
+                let _response = client
+                    .post(format!("{}/v1/auth/token/revoke-self", app.vault_url))
+                    .insert_header((VAULT_TOKEN_HEADER, root_token.to_string()))
+                    .send()
+                    .await;
+
+                info!("Vault unsealed and configured!");
+            }
+            UnsealServerState::VaultInitializedAndConfigured => {
+                info!("Vault is unsealed and hopefully configured!");
+                info!("Initiating raft join");
+                // load TLS cert chain
+                let mut cert_file = File::open("/opt/vault/tls/cacert.pem")
+                    .context("Failed to open TLS cert chain")
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+                let mut cert_buf = Vec::new();
+                cert_file
+                    .read_to_end(&mut cert_buf)
+                    .context("Failed to read TLS cert chain")
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+                let cert_chain = std::str::from_utf8(&cert_buf)
+                    .context("Failed to parse TLS cert chain as UTF-8")
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)?
+                    .to_string();
+
+                let payload = json!({"leader_ca_cert": cert_chain, "retry": true });
+
+                let mut response = client
+                    .post(format!("{}/v1/sys/storage/raft/join", vault_url))
+                    .send_json(&payload)
+                    .await?;
+
+                let status_code = response.status();
+                if !status_code.is_success() {
+                    error!("Vault returned server error: {}", status_code);
+                    let mut client_resp = HttpResponse::build(status_code);
+                    for (header_name, header_value) in response.headers().iter() {
+                        client_resp.insert_header((header_name.clone(), header_value.clone()));
+                    }
+                    return Ok(client_resp.streaming(response));
+                }
+
+                let response: Value = response
+                    .json()
+                    .await
+                    .context("parsing raft join response")
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+                debug!("raft join: {:?}", response);
+
+                if response.get("errors").is_some() {
+                    return Ok(HttpResponse::Ok().json(response));
+                }
+            }
+            UnsealServerState::Undefined => {
+                unreachable!("Invalid internal state");
+            }
+        }
+    }
+
+    Ok(HttpResponse::Accepted().json(response)) // <- send response
+}
+
+pub async fn vault_configure_unsealed(
+    app: &UnsealServerConfig,
+    admin_config: &AdminConfig,
+    root_token: &str,
+    admin_tee_mrenclave: &str,
+    c: &Client,
+) -> Result<(), HttpResponseError> {
+    wait_for_plugins_catalog(app, root_token, c).await;
+
+    if !plugin_is_already_running(app, root_token, c).await? {
+        let r = vault(
+            "Installing vault-auth-tee plugin",
+            c.put(format!(
+                "{}/v1/sys/plugins/catalog/auth/vault-auth-tee",
+                app.vault_url
+            )),
+            root_token,
+            json!({
+                "sha256": VAULT_AUTH_TEE_SHA256,
+                "command": "vault-auth-tee",
+                "version": "0.1.0+dev"
+            }),
+        )
+        .await
+        .map_err(|e| anyhow!("{:?}", e))
+        .status(StatusCode::BAD_GATEWAY)?;
+        if !r.status().is_success() {
+            let err = HttpResponseError::from_proxy(r).await;
+            return Err(err);
+        }
+    } else {
+        info!("vault-auth-tee plugin already installed");
+    }
+
+    if !plugin_is_already_running(app, root_token, c).await? {
+        let r = vault(
+            "Activating vault-auth-tee plugin",
+            c.post(format!("{}/v1/sys/auth/tee", app.vault_url)),
+            root_token,
+            json!({"type": "vault-auth-tee"}),
+        )
+        .await
+        .map_err(|e| anyhow!("{:?}", e))
+        .status(StatusCode::BAD_GATEWAY)?;
+        if !r.status().is_success() {
+            let err = HttpResponseError::from_proxy(r).await;
+            return Err(err);
+        }
+    } else {
+        info!("vault-auth-tee plugin already activated");
+    }
+
+    if let Ok(mut r) = c
+        .get(format!("{}/v1/auth/tee/tees?list=true", app.vault_url))
+        .insert_header((VAULT_TOKEN_HEADER, root_token))
+        .send()
+        .await
+    {
+        let r: Value = r
+            .json()
+            .await
+            .map_err(|e| anyhow!("{:?}", e))
+            .status(StatusCode::BAD_GATEWAY)?;
+        trace!("{:?}", r);
+        if let Some(tees) = r.get("data").and_then(|v| v.get("keys")) {
+            if let Some(tees) = tees.as_array() {
+                if tees.contains(&json!("root")) {
+                    info!("root TEE already installed");
+                    return Ok(());
+                }
+            }
+        }
+    }
+
+    vault(
+        "Installing root TEE",
+        c.put(format!("{}/v1/auth/tee/tees/admin", app.vault_url)),
+        root_token,
+        json!({
+            "lease": "1000",
+            "name": "admin",
+            "types": "sgx",
+            "sgx_allowed_tcb_levels": "Ok,SwHardeningNeeded",
+            "sgx_mrenclave": &admin_tee_mrenclave,
+            "token_policies": "admin"
+        }),
+    )
+    .await
+    .map_err(|e| anyhow!("{:?}", e))
+    .status(StatusCode::BAD_GATEWAY)?;
+
+    // Install admin policies
+    let admin_policy = include_str!("admin-policy.hcl");
+    vault(
+        "Installing admin policy",
+        c.put(format!("{}/v1/sys/policies/acl/admin", app.vault_url)),
+        root_token,
+        json!({ "policy": admin_policy }),
+    )
+    .await
+    .map_err(|e| anyhow!("{:?}", e))
+    .status(StatusCode::BAD_GATEWAY)?;
+
+    vault(
+        "Enable the key/value secrets engine v1 at secret/.",
+        c.put(format!("{}/v1/sys/mounts/secret", app.vault_url)),
+        root_token,
+        json!({ "type": "kv", "description": "K/V v1" } ),
+    )
+    .await
+    .map_err(|e| anyhow!("{:?}", e))
+    .status(StatusCode::BAD_GATEWAY)?;
+
+    // Create a `VaultConnection` for the `admin` tee to initialize the secrets for it.
+    // Safety: the connection was already attested
+    let admin_vcon = unsafe {
+        VaultConnection::new_from_client_without_attestation(
+            app.vault_url.clone(),
+            c.clone(),
+            "admin".into(),
+            root_token.to_string(),
+        )
+    };
+
+    // initialize the admin config
+    admin_vcon.store_secret(admin_config, "config").await?;
+    admin_vcon
+        .store_secret(AdminState::default(), "state")
+        .await?;
+
+    Ok(())
+}
+
+async fn wait_for_plugins_catalog(app: &UnsealServerConfig, root_token: &str, c: &Client) {
+    info!("Waiting for plugins to be loaded");
+    loop {
+        let r = c
+            .get(format!("{}/v1/sys/plugins/catalog", app.vault_url))
+            .insert_header((VAULT_TOKEN_HEADER, root_token))
+            .send()
+            .await;
+
+        match r {
+            Ok(r) => {
+                if r.status().is_success() {
+                    break;
+                } else {
+                    debug!("/v1/sys/plugins/catalog status: {:#?}", r)
+                }
+            }
+            Err(e) => {
+                debug!("/v1/sys/plugins/catalog error: {}", e)
+            }
+        }
+
+        info!("Waiting for plugins to be loaded");
+        sleep(Duration::from_secs(1)).await;
+    }
+}
+
+async fn plugin_is_already_running(
+    app: &UnsealServerConfig,
+    root_token: &str,
+    c: &Client,
+) -> std::result::Result<bool, HttpResponseError> {
+    if let Ok(mut r) = c
+        .get(format!("{}/v1/sys/auth", app.vault_url))
+        .insert_header((VAULT_TOKEN_HEADER, root_token))
+        .send()
+        .await
+    {
+        if !r.status().is_success() {
+            return Ok(false);
+        }
+        let r: Value = r
+            .json()
+            .await
+            .map_err(|e| anyhow!("{:?}", e))
+            .status(StatusCode::BAD_GATEWAY)?;
+        trace!("{}", r.to_string());
+
+        let is_running = r
+            .get("data")
+            .and_then(|v| v.get("tee/"))
+            .and_then(|v| v.get("running_sha256"))
+            .and_then(|v| v.as_str())
+            .and_then(|v| if v.is_empty() { None } else { Some(v) })
+            .and_then(|v| {
+                if v == VAULT_AUTH_TEE_SHA256 {
+                    Some(v)
+                } else {
+                    None
+                }
+            })
+            .is_some();
+        Ok(is_running)
+    } else {
+        Ok(false)
+    }
+}
+
+async fn vault(
+    action: &str,
+    req: ClientRequest,
+    token: &str,
+    json: Value,
+) -> <SendClientRequest as Future>::Output {
+    info!("{}", action);
+    debug!("json: {:?}", json);
+    match req
+        .insert_header((VAULT_TOKEN_HEADER, token))
+        .send_json(&json)
+        .await
+    {
+        Ok(r) => {
+            debug!("response {:?}", r);
+            Ok(r)
+        }
+        Err(e) => {
+            error!("{}: {}", action, e);
+            Err(e)
+        }
+    }
+}

bin/tee-vault-unseal/tee-vault-unseal.manifest.template

@@ -0,0 +1,62 @@
+libos.entrypoint = "/app/tee-vault-unseal"
+
+[loader]
+argv = [ "/app/tee-vault-unseal" ]
+entrypoint = "file:{{ gramine.libos }}"
+env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}:/usr{{ arch_libdir }}:/lib"
+env.HOME = "/app"
+env.MALLOC_ARENA_MAX = "1"
+env.AZDCAP_DEBUG_LOG_LEVEL = "ignore"
+env.AZDCAP_COLLATERAL_VERSION = "v4"
+
+### Required configuration ###
+env.ALLOWED_TCB_LEVELS = { passthrough = true }
+env.VAULT_ADDR = { passthrough = true }
+
+### DEBUG ###
+env.RUST_BACKTRACE = "1"
+env.RUST_LOG="info,tee_vault_unseal=trace,teepot=trace,awc=debug"
+
+[fs]
+root.uri = "file:/"
+start_dir = "/app"
+mounts = [
+  { path = "{{ execdir }}", uri = "file:{{ execdir }}" },
+  { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
+  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
+  { path = "/etc", uri = "file:/etc" },
+  { type = "tmpfs", path = "/var/tmp" },
+  { type = "tmpfs", path = "/tmp" },
+  { type = "tmpfs", path = "/app/.dcap-qcnl" },
+  { type = "tmpfs", path = "/app/.az-dcap-client" },
+  { type = "encrypted", path = "/opt/vault/tls", uri = "file:/opt/vault/tls", key_name = "_sgx_mrsigner" },
+  { path = "/lib/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" },
+]
+
+[sgx]
+trusted_files = [
+  "file:/etc/ld.so.cache",
+  "file:/app/",
+  "file:{{ execdir }}/",
+  "file:{{ arch_libdir }}/",
+  "file:/usr/{{ arch_libdir }}/",
+  "file:{{ gramine.libos }}",
+  "file:{{ gramine.runtimedir() }}/",
+  "file:/usr/lib/ssl/openssl.cnf",
+  "file:/etc/ssl/",
+  "file:/etc/sgx_default_qcnl.conf",
+  "file:/lib/libdcap_quoteprov.so",
+]
+remote_attestation = "dcap"
+max_threads = 64
+edmm_enable = false
+## max enclave size
+enclave_size = "2G"
+
+[sys]
+enable_extra_runtime_domain_names_conf = true
+enable_sigterm_injection = true
+
+# possible tweak option, if problems with mio
+# currently mio is compiled with `mio_unsupported_force_waker_pipe`
+# insecure__allow_eventfd = true

bin/teepot-read/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "teepot-read"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+actix-web.workspace = true
+anyhow.workspace = true
+awc.workspace = true
+clap.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+teepot.workspace = true
+tracing-actix-web.workspace = true
+tracing-log.workspace = true
+tracing-subscriber.workspace = true
+tracing.workspace = true

bin/teepot-read/src/main.rs

@@ -0,0 +1,111 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+//! Get the secrets from a Vault TEE and pass them as environment variables to a command
+
+#![deny(missing_docs)]
+#![deny(clippy::all)]
+
+use anyhow::{Context, Result};
+use clap::Parser;
+use serde_json::Value;
+use std::collections::HashMap;
+use std::os::unix::process::CommandExt;
+use std::process::Command;
+use teepot::client::vault::VaultConnection;
+use teepot::server::attestation::VaultAttestationArgs;
+use tracing::{debug, info, warn};
+use tracing_log::LogTracer;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter, Registry};
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Arguments {
+    /// turn on test mode
+    #[arg(long, hide = true)]
+    pub test: bool,
+    /// vault token
+    #[arg(long, env = "VAULT_TOKEN", hide = true)]
+    pub vault_token: String,
+    #[clap(flatten)]
+    pub attestation: VaultAttestationArgs,
+    /// name of this TEE to login to vault
+    #[arg(long, required = true)]
+    pub name: String,
+    /// secrets to get from vault and pass as environment variables
+    #[arg(long, required = true)]
+    pub secrets: Vec<String>,
+    /// command to run
+    pub command: Vec<String>,
+}
+
+#[actix_web::main]
+async fn main() -> Result<()> {
+    LogTracer::init().context("Failed to set logger")?;
+
+    let subscriber = Registry::default()
+        .with(EnvFilter::from_default_env())
+        .with(fmt::layer().with_writer(std::io::stderr));
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    let args = Arguments::parse();
+
+    // Split every string with a ',' into a vector of strings, flatten them and collect them.
+    let secrets = args
+        .secrets
+        .iter()
+        .flat_map(|s| s.split(','))
+        .collect::<Vec<_>>();
+
+    info!("args: {:?}", args);
+
+    let conn = if args.test {
+        warn!("TEST MODE");
+        let client = awc::Client::builder()
+            .add_default_header((actix_web::http::header::USER_AGENT, "teepot/1.0"))
+            .finish();
+        // SAFETY: TEST MODE
+        unsafe {
+            VaultConnection::new_from_client_without_attestation(
+                args.attestation.vault_addr.clone(),
+                client,
+                args.name.clone(),
+                args.vault_token.clone(),
+            )
+        }
+    } else {
+        VaultConnection::new(&args.attestation.clone().into(), args.name.clone())
+            .await
+            .expect("connecting to vault")
+    };
+
+    let mut env: HashMap<String, String> = HashMap::new();
+
+    for secret_name in secrets {
+        debug!("getting secret {secret_name}");
+        let secret_val: serde_json::Value = match conn.load_secret(secret_name).await? {
+            Some(val) => val,
+            None => {
+                debug!("secret {secret_name} not found");
+                continue;
+            }
+        };
+
+        debug!("got secret {secret_name}: {secret_val}");
+
+        // Plain strings can be converted to strings.
+        let env_val = match secret_val {
+            Value::String(s) => s,
+            _ => secret_val.to_string(),
+        };
+
+        env.insert(secret_name.to_string(), env_val);
+    }
+
+    let err = Command::new(&args.command[0])
+        .args(&args.command[1..])
+        .envs(env)
+        .exec();
+
+    Err(err).context("exec failed")
+}

bin/teepot-write/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "teepot-write"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
+
+[dependencies]
+actix-web.workspace = true
+anyhow.workspace = true
+awc.workspace = true
+clap.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+teepot.workspace = true
+tracing-actix-web.workspace = true
+tracing-log.workspace = true
+tracing-subscriber.workspace = true
+tracing.workspace = true

bin/teepot-write/src/main.rs

@@ -0,0 +1,98 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+//! Write secrets to a Vault TEE from environment variables
+
+#![deny(missing_docs)]
+#![deny(clippy::all)]
+
+use anyhow::{Context, Result};
+use clap::Parser;
+use serde_json::Value;
+use std::collections::HashMap;
+use std::env;
+use teepot::client::vault::VaultConnection;
+use teepot::server::attestation::VaultAttestationArgs;
+use tracing::{debug, info, warn};
+use tracing_log::LogTracer;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter, Registry};
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Arguments {
+    /// turn on test mode
+    #[arg(long, hide = true)]
+    pub test: bool,
+    /// vault token
+    #[arg(long, env = "VAULT_TOKEN", hide = true)]
+    pub vault_token: String,
+    #[clap(flatten)]
+    pub attestation: VaultAttestationArgs,
+    /// name of this TEE to login to vault
+    #[arg(long, required = true)]
+    pub name: String,
+    /// name of this TEE to login to vault
+    #[arg(long)]
+    pub store_name: Option<String>,
+    /// secrets to write to vault with the value of the environment variables
+    #[arg(long, required = true)]
+    pub secrets: Vec<String>,
+}
+
+#[actix_web::main]
+async fn main() -> Result<()> {
+    LogTracer::init().context("Failed to set logger")?;
+
+    let subscriber = Registry::default()
+        .with(EnvFilter::from_default_env())
+        .with(fmt::layer().with_writer(std::io::stderr));
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    let args = Arguments::parse();
+
+    // Split every string with a ',' into a vector of strings, flatten them and collect them.
+    let secrets = args
+        .secrets
+        .iter()
+        .flat_map(|s| s.split(','))
+        .collect::<Vec<_>>();
+
+    info!("args: {:?}", args);
+
+    let conn = if args.test {
+        warn!("TEST MODE");
+        let client = awc::Client::builder()
+            .add_default_header((actix_web::http::header::USER_AGENT, "teepot/1.0"))
+            .finish();
+        // SAFETY: TEST MODE
+        unsafe {
+            VaultConnection::new_from_client_without_attestation(
+                args.attestation.vault_addr.clone(),
+                client,
+                args.name.clone(),
+                args.vault_token.clone(),
+            )
+        }
+    } else {
+        VaultConnection::new(&args.attestation.clone().into(), args.name.clone())
+            .await
+            .expect("connecting to vault")
+    };
+
+    let tee_name = args.store_name.unwrap_or(args.name.clone());
+
+    let env = env::vars()
+        .filter(|(k, _)| secrets.contains(&k.as_str()))
+        .collect::<HashMap<_, _>>();
+
+    for (secret_name, secret_val) in env {
+        debug!("storing secret {secret_name}: {secret_val}");
+        let secret_val = Value::String(secret_val);
+        conn.store_secret_for_tee(&tee_name, &secret_val, &secret_name)
+            .await
+            .expect("storing secret");
+        info!("stored secret {secret_name}");
+    }
+
+    Ok(())
+}

bin/vault-admin/Cargo.toml

@@ -0,0 +1,23 @@
+[package]
+name = "vault-admin"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+actix-web.workspace = true
+anyhow.workspace = true
+awc.workspace = true
+bytemuck.workspace = true
+clap.workspace = true
+hex.workspace = true
+pgp.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+teepot.workspace = true
+tracing-log.workspace = true
+tracing-subscriber.workspace = true
+tracing.workspace = true

bin/vault-admin/README.md

@@ -0,0 +1,46 @@
+```bash
+❯ idents=( tests/data/pub*.asc )
+❯ cargo run --bin vault-admin -p vault-admin -- \
+  verify \
+    ${idents[@]/#/-i } \
+    tests/data/test.json \
+    tests/data/test.json.asc
+
+Verified signature for `81A312C59D679D930FA9E8B06D728F29A2DBABF8`
+
+❯ RUST_LOG=info cargo run -p vault-admin -- \
+  send \
+   --sgx-mrsigner c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d \
+   --sgx-allowed-tcb-levels SwHardeningNeeded \
+   --server https://127.0.0.1:8444 \
+  bin/tee-vault-admin/tests/data/test.json \
+  bin/tee-vault-admin/tests/data/test.json.asc
+
+2023-08-04T10:51:14.919941Z  INFO vault_admin: Quote verified! Connection secure!
+2023-08-04T10:51:14.920430Z  INFO tee_client: Getting attestation report
+2023-08-04T10:51:15.020459Z  INFO tee_client: Checked or set server certificate public key hash `f6dc06b9f2a14fa16a94c076a85eab8513f99ec0091801cc62c8761e42908fc1`
+2023-08-04T10:51:15.024310Z  INFO tee_client: Verifying attestation report
+2023-08-04T10:51:15.052712Z  INFO tee_client: TcbLevel is allowed: SwHardeningNeeded: Software hardening is needed
+2023-08-04T10:51:15.054508Z  WARN tee_client: Info: Advisory ID: INTEL-SA-00615
+2023-08-04T10:51:15.054572Z  INFO tee_client: Report data matches `f6dc06b9f2a14fa16a94c076a85eab8513f99ec0091801cc62c8761e42908fc1`
+2023-08-04T10:51:15.054602Z  INFO tee_client: mrsigner `c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d` matches
+[
+  {
+    "request": {
+      "data": {
+        "lease": "1000",
+        "name": "test",
+        "sgx_allowed_tcb_levels": "Ok,SwHardeningNeeded",
+        "sgx_mrsigner": "c5591a72b8b86e0d8814d6e8750e3efe66aea2d102b8ba2405365559b858697d",
+        "token_policies": "test",
+        "types": "sgx"
+      },
+      "url": "/v1/auth/tee/tees/test"
+    },
+    "response": {
+      "status_code": 204,
+      "value": null
+    }
+  }
+]
+```

bin/vault-admin/src/main.rs

@@ -0,0 +1,366 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+use anyhow::{anyhow, bail, Context, Result};
+use clap::{Args, Parser, Subcommand};
+use pgp::types::KeyTrait;
+use pgp::{Deserializable, SignedPublicKey};
+use serde_json::Value;
+use std::default::Default;
+use std::fs::{File, OpenOptions};
+use std::io::{Read, Write};
+use std::path::{Path, PathBuf};
+use teepot::client::{AttestationArgs, TeeConnection};
+use teepot::json::http::{
+    SignRequest, SignRequestData, SignResponse, VaultCommandRequest, VaultCommands,
+    VaultCommandsResponse, ATTESTATION_URL, DIGEST_URL,
+};
+use teepot::server::signatures::verify_sig;
+use teepot::sgx::sign::Signature;
+use tracing::{error, info};
+use tracing_log::LogTracer;
+use tracing_subscriber::Registry;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter};
+
+#[derive(Args, Debug)]
+struct SendArgs {
+    #[clap(flatten)]
+    pub attestation: AttestationArgs,
+    /// Vault command file
+    #[arg(required = true)]
+    pub command_file: PathBuf,
+    /// GPG signature files
+    #[arg(required = true)]
+    pub sigs: Vec<PathBuf>,
+}
+
+#[derive(Args, Debug)]
+struct SignTeeArgs {
+    #[clap(flatten)]
+    pub attestation: AttestationArgs,
+    /// output file
+    #[arg(short, long, required = true)]
+    pub out: PathBuf,
+    /// signature request file
+    #[arg(required = true)]
+    pub sig_request_file: PathBuf,
+    /// GPG signature files
+    #[arg(required = true)]
+    pub sigs: Vec<PathBuf>,
+}
+
+#[derive(Args, Debug)]
+struct DigestArgs {
+    #[clap(flatten)]
+    pub attestation: AttestationArgs,
+}
+
+#[derive(Args, Debug)]
+struct VerifyArgs {
+    /// GPG identity files
+    #[arg(short, long, required = true)]
+    pub idents: Vec<PathBuf>,
+    /// Vault command file
+    #[arg(required = true)]
+    pub command_file: PathBuf,
+    /// GPG signature files
+    #[arg(required = true)]
+    pub sigs: Vec<PathBuf>,
+}
+
+#[derive(Args, Debug)]
+struct CreateSignRequestArgs {
+    /// Last digest
+    #[arg(long)]
+    pub last_digest: Option<String>,
+    /// TEE name
+    #[arg(long)]
+    pub tee_name: Option<String>,
+    /// Vault command file
+    #[arg(required = true)]
+    pub sig_file: PathBuf,
+}
+
+#[derive(Subcommand, Debug)]
+enum SubCommands {
+    /// Send the signed commands to execute to the vault
+    Command(SendArgs),
+    /// Verify the signature(s) for the commands to send
+    Verify(VerifyArgs),
+    /// Get the digest of the last executed commands
+    Digest(DigestArgs),
+    /// Send the signed commands to execute to the vault
+    SignTee(SignTeeArgs),
+    /// Create a sign request
+    CreateSignRequest(CreateSignRequestArgs),
+}
+
+/// Admin tool for the vault
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Arguments {
+    #[clap(subcommand)]
+    cmd: SubCommands,
+}
+
+#[actix_web::main]
+async fn main() -> Result<()> {
+    LogTracer::init().context("Failed to set logger")?;
+
+    let subscriber = Registry::default()
+        .with(EnvFilter::from_default_env())
+        .with(fmt::layer().with_writer(std::io::stderr));
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    let args = Arguments::parse();
+    info!("Quote verified! Connection secure!");
+
+    match args.cmd {
+        SubCommands::Command(args) => send_commands(args).await?,
+        SubCommands::SignTee(args) => send_sig_request(args).await?,
+        SubCommands::Verify(args) => {
+            verify(args.command_file, args.idents.iter(), args.sigs.iter())?
+        }
+        SubCommands::Digest(args) => digest(args).await?,
+        SubCommands::CreateSignRequest(args) => create_sign_request(args)?,
+    }
+
+    Ok(())
+}
+
+fn create_sign_request(args: CreateSignRequestArgs) -> Result<()> {
+    let mut sigstruct_file = File::open(&args.sig_file)?;
+    let mut sigstruct_bytes = Vec::new();
+    sigstruct_file.read_to_end(&mut sigstruct_bytes)?;
+
+    let sigstruct = bytemuck::try_from_bytes::<Signature>(&sigstruct_bytes)
+        .context(format!("parsing signature file {:?}", &args.sig_file))?;
+
+    let body = sigstruct.body();
+    let data = bytemuck::bytes_of(&body).to_vec();
+
+    let sign_request_data = SignRequestData {
+        data,
+        last_digest: args.last_digest.unwrap_or_default(),
+        tee_name: args.tee_name.unwrap_or_default(),
+        tee_type: "sgx".to_string(),
+        ..Default::default()
+    };
+
+    println!("{}", serde_json::to_string_pretty(&sign_request_data)?);
+    Ok(())
+}
+
+fn verify(
+    msg: impl AsRef<Path>,
+    idents_file_paths: impl Iterator<Item = impl AsRef<Path>>,
+    sig_paths: impl Iterator<Item = impl AsRef<Path>>,
+) -> Result<()> {
+    let mut cmd_file = File::open(msg.as_ref())?;
+    let mut cmd_buf = Vec::new();
+    cmd_file
+        .read_to_end(&mut cmd_buf)
+        .context(format!("reading command file {:?}", &cmd_file))?;
+
+    let mut idents = Vec::new();
+    for ident_file_path in idents_file_paths {
+        let ident_file = File::open(ident_file_path.as_ref()).context(format!(
+            "reading identity file {:?}",
+            ident_file_path.as_ref()
+        ))?;
+        idents.push(
+            SignedPublicKey::from_armor_single(ident_file)
+                .context(format!(
+                    "reading identity file {:?}",
+                    ident_file_path.as_ref()
+                ))?
+                .0,
+        );
+    }
+
+    for sig_path in sig_paths {
+        let mut sig_file = File::open(&sig_path)
+            .context(format!("reading signature file {:?}", sig_path.as_ref()))?;
+        let mut sig = String::new();
+        sig_file
+            .read_to_string(&mut sig)
+            .context(format!("reading signature file {:?}", sig_path.as_ref()))?;
+        let ident_pos = verify_sig(&sig, &cmd_buf, &idents)?;
+        println!(
+            "Verified signature for `{}`",
+            hex::encode_upper(idents.get(ident_pos).unwrap().fingerprint())
+        );
+        // Remove the identity from the list of identities to verify
+        idents.remove(ident_pos);
+    }
+
+    Ok(())
+}
+
+async fn send_commands(args: SendArgs) -> Result<()> {
+    // Read the command file into a string
+    let mut cmd_file = File::open(&args.command_file)?;
+    let mut commands = String::new();
+    cmd_file.read_to_string(&mut commands)?;
+
+    // Check that the command file is valid JSON
+    let vault_commands: VaultCommands = serde_json::from_str(&commands)
+        .context(format!("parsing command file {:?}", &args.command_file))?;
+
+    let mut signatures = Vec::new();
+
+    for sig in args.sigs {
+        let mut sig_file = File::open(sig)?;
+        let mut sig = String::new();
+        sig_file.read_to_string(&mut sig)?;
+        signatures.push(sig);
+    }
+
+    let send_req = VaultCommandRequest {
+        commands,
+        signatures,
+    };
+
+    let conn = TeeConnection::new(&args.attestation, ATTESTATION_URL).await?;
+
+    let mut response = conn
+        .client()
+        .post(&format!(
+            "{server}{url}",
+            server = conn.server(),
+            url = VaultCommandRequest::URL
+        ))
+        .send_json(&send_req)
+        .await
+        .map_err(|e| anyhow!("sending command request: {}", e))?;
+
+    let status_code = response.status();
+    if !status_code.is_success() {
+        error!("sending command request: {}", status_code);
+        if let Ok(r) = response.json::<Value>().await {
+            eprintln!(
+                "Error sending command request: {}",
+                serde_json::to_string(&r).unwrap_or_default()
+            );
+        }
+        bail!("sending command request: {}", status_code);
+    }
+
+    let cmd_responses: VaultCommandsResponse = response
+        .json()
+        .await
+        .context("failed parsing command response")?;
+
+    println!("digest: {}", &cmd_responses.digest);
+
+    let pairs = cmd_responses
+        .results
+        .iter()
+        .zip(vault_commands.commands.iter())
+        .map(|(resp, cmd)| {
+            let mut pair = serde_json::Map::new();
+            pair.insert("request".to_string(), serde_json::to_value(cmd).unwrap());
+            pair.insert("response".to_string(), serde_json::to_value(resp).unwrap());
+            pair
+        })
+        .collect::<Vec<_>>();
+
+    println!("{}", serde_json::to_string_pretty(&pairs)?);
+    Ok(())
+}
+
+async fn send_sig_request(args: SignTeeArgs) -> Result<()> {
+    // Read the command file into a string
+    let mut cmd_file = File::open(&args.sig_request_file)?;
+    let mut sign_request_data_str = String::new();
+    cmd_file.read_to_string(&mut sign_request_data_str)?;
+
+    // Check that the command file is valid JSON
+    let _sign_request_data: SignRequestData = serde_json::from_str(&sign_request_data_str)
+        .context(format!("parsing command file {:?}", &args.sig_request_file))?;
+
+    let mut signatures = Vec::new();
+
+    for sig in args.sigs {
+        let mut sig_file = File::open(sig)?;
+        let mut sig = String::new();
+        sig_file.read_to_string(&mut sig)?;
+        signatures.push(sig);
+    }
+
+    // open out_file early to fail fast if it is not writable
+    let mut out_file = OpenOptions::new()
+        .create(true)
+        .write(true)
+        .open(&args.out)?;
+
+    let send_req = SignRequest {
+        sign_request_data: sign_request_data_str,
+        signatures,
+    };
+
+    let conn = TeeConnection::new(&args.attestation, ATTESTATION_URL).await?;
+
+    let mut response = conn
+        .client()
+        .post(&format!(
+            "{server}{url}",
+            server = conn.server(),
+            url = SignRequest::URL
+        ))
+        .send_json(&send_req)
+        .await
+        .map_err(|e| anyhow!("sending sign request: {}", e))?;
+
+    let status_code = response.status();
+    if !status_code.is_success() {
+        error!("sending sign request: {}", status_code);
+        if let Ok(r) = response.json::<Value>().await {
+            eprintln!(
+                "Error sending sign request: {}",
+                serde_json::to_string(&r).unwrap_or_default()
+            );
+        }
+        bail!("sending sign request: {}", status_code);
+    }
+
+    let sign_response: SignResponse = response
+        .json()
+        .await
+        .context("failed parsing sign response")?;
+
+    println!("digest: {}", &sign_response.digest);
+
+    out_file.write_all(&sign_response.signed_data)?;
+
+    println!("{{ \"digest\": \"{}\" }}", sign_response.digest);
+    Ok(())
+}
+
+async fn digest(args: DigestArgs) -> Result<()> {
+    let conn = TeeConnection::new(&args.attestation, ATTESTATION_URL).await?;
+
+    let mut response = conn
+        .client()
+        .get(&format!("{server}{DIGEST_URL}", server = conn.server()))
+        .send()
+        .await
+        .map_err(|e| anyhow!("sending digest request: {}", e))?;
+
+    let status_code = response.status();
+    if !status_code.is_success() {
+        error!("sending digest request: {}", status_code);
+        if let Ok(r) = response.json::<Value>().await {
+            eprintln!("Error sending digest request: {}", r);
+        }
+        bail!("sending digest request: {}", status_code);
+    }
+
+    let digest_response: Value = response
+        .json()
+        .await
+        .context("failed parsing digest response")?;
+
+    println!("{}", serde_json::to_string_pretty(&digest_response)?);
+    Ok(())
+}

bin/vault-unseal/Cargo.toml

@@ -0,0 +1,22 @@
+[package]
+name = "vault-unseal"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+actix-web.workspace = true
+anyhow.workspace = true
+awc.workspace = true
+base64.workspace = true
+clap.workspace = true
+hex.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+teepot.workspace = true
+tracing-log.workspace = true
+tracing-subscriber.workspace = true
+tracing.workspace = true

bin/vault-unseal/src/main.rs

@@ -0,0 +1,220 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+use anyhow::{anyhow, bail, Context, Result};
+use base64::{engine::general_purpose, Engine as _};
+use clap::{Args, Parser, Subcommand};
+use serde_json::Value;
+use std::fs::File;
+use std::io::Read;
+use teepot::client::{AttestationArgs, TeeConnection};
+use teepot::json::http::{Init, InitResponse, Unseal, ATTESTATION_URL};
+use tracing::{error, info, trace, warn};
+use tracing_log::LogTracer;
+use tracing_subscriber::Registry;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter};
+
+#[derive(Args, Debug)]
+pub struct InitArgs {
+    /// admin threshold
+    #[arg(long)]
+    admin_threshold: usize,
+    /// PGP keys to sign commands for the admin tee
+    #[arg(short, long)]
+    admin_pgp_key_file: Vec<String>,
+    /// admin TEE mrenclave
+    #[arg(long)]
+    admin_tee_mrenclave: String,
+    /// secret threshold
+    #[arg(long)]
+    unseal_threshold: usize,
+    /// PGP keys to encrypt the unseal keys with
+    #[arg(short, long)]
+    unseal_pgp_key_file: Vec<String>,
+}
+
+/// subcommands and their options/arguments.
+#[derive(Subcommand, Debug)]
+enum SubCommands {
+    Init(InitArgs),
+    Unseal,
+}
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Arguments {
+    #[clap(flatten)]
+    pub attestation: AttestationArgs,
+    /// Subcommands (with their own options)
+    #[clap(subcommand)]
+    cmd: SubCommands,
+}
+
+#[actix_web::main]
+async fn main() -> Result<()> {
+    LogTracer::init().context("Failed to set logger")?;
+
+    let subscriber = Registry::default()
+        .with(EnvFilter::from_default_env())
+        .with(fmt::layer().with_writer(std::io::stderr));
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    let args = Arguments::parse();
+
+    match args.cmd {
+        SubCommands::Init(_) => init(args).await?,
+        SubCommands::Unseal => unseal(args).await?,
+    }
+
+    Ok(())
+}
+
+async fn init(args: Arguments) -> Result<()> {
+    let conn = TeeConnection::new(&args.attestation, ATTESTATION_URL).await?;
+
+    info!("Quote verified! Connection secure!");
+
+    let SubCommands::Init(init_args) = args.cmd else {
+        unreachable!()
+    };
+
+    if init_args.admin_threshold == 0 {
+        bail!("admin threshold must be greater than 0");
+    }
+
+    if init_args.unseal_threshold == 0 {
+        bail!("unseal threshold must be greater than 0");
+    }
+
+    if init_args.admin_threshold > init_args.admin_pgp_key_file.len() {
+        bail!("admin threshold must be less than or equal to the number of admin pgp keys");
+    }
+
+    if init_args.unseal_threshold > init_args.unseal_pgp_key_file.len() {
+        bail!("unseal threshold must be less than or equal to the number of unseal pgp keys");
+    }
+
+    let mut pgp_keys = Vec::new();
+
+    for filename in init_args.unseal_pgp_key_file {
+        let mut file =
+            File::open(&filename).context(format!("Failed to open pgp key file {}", &filename))?;
+        let mut buf = Vec::new();
+        file.read_to_end(&mut buf)?;
+        let key = std::str::from_utf8(&buf)?.trim().to_string();
+        pgp_keys.push(key);
+    }
+
+    let mut admin_pgp_keys = Vec::new();
+
+    for filename in init_args.admin_pgp_key_file {
+        let mut file =
+            File::open(&filename).context(format!("Failed to open pgp key file {}", &filename))?;
+        // read all lines from file and concatenate them
+        let mut key = String::new();
+        file.read_to_string(&mut key)
+            .context(format!("Failed to read pgp key file {}", &filename))?;
+        key.retain(|c| !c.is_ascii_whitespace());
+
+        let bytes = general_purpose::STANDARD.decode(key).context(format!(
+            "Failed to base64 decode pgp key file {}",
+            &filename
+        ))?;
+        admin_pgp_keys.push(bytes.into_boxed_slice());
+    }
+
+    let init = Init {
+        secret_shares: pgp_keys.len() as _,
+        secret_threshold: init_args.unseal_threshold,
+        admin_threshold: init_args.admin_threshold,
+        admin_tee_mrenclave: init_args.admin_tee_mrenclave,
+        admin_pgp_keys: admin_pgp_keys.into_boxed_slice(),
+        pgp_keys,
+    };
+
+    info!("Inititalizing vault");
+
+    let mut response = conn
+        .client()
+        .post(&format!(
+            "{server}{url}",
+            server = conn.server(),
+            url = Init::URL
+        ))
+        .send_json(&init)
+        .await
+        .map_err(|e| anyhow!("Error sending init request: {}", e))?;
+
+    let status_code = response.status();
+    if !status_code.is_success() {
+        error!("Failed to init vault: {}", status_code);
+        if let Ok(r) = response.json::<Value>().await {
+            eprintln!("Failed to init vault: {}", r);
+        }
+        bail!("failed to init vault: {}", status_code);
+    }
+
+    let init_response: Value = response.json().await.context("failed to init vault")?;
+
+    info!("Got Response: {}", init_response.to_string());
+
+    let resp: InitResponse =
+        serde_json::from_value(init_response).context("Failed to parse init response")?;
+    println!("{}", serde_json::to_string(&resp).unwrap());
+    Ok(())
+}
+
+async fn unseal(args: Arguments) -> Result<()> {
+    info!("Reading unencrypted key from stdin");
+
+    // read all bytes from stdin
+    let mut stdin = std::io::stdin();
+    let mut buf = Vec::new();
+    stdin.read_to_end(&mut buf)?;
+    let key = std::str::from_utf8(&buf)?.trim().to_string();
+
+    if key.is_empty() {
+        bail!("Error reading key from stdin");
+    }
+
+    let conn = TeeConnection::new(&args.attestation, ATTESTATION_URL).await?;
+
+    info!("Quote verified! Connection secure!");
+
+    info!("Unsealing vault");
+
+    let unseal_data = Unseal { key };
+
+    let mut response = conn
+        .client()
+        .post(&format!(
+            "{server}{url}",
+            server = conn.server(),
+            url = Unseal::URL
+        ))
+        .send_json(&unseal_data)
+        .await
+        .map_err(|e| anyhow!("Error sending unseal request: {}", e))?;
+
+    let status_code = response.status();
+    if !status_code.is_success() {
+        error!("Failed to unseal vault: {}", status_code);
+        if let Ok(r) = response.json::<Value>().await {
+            eprintln!("Failed to unseal vault: {}", r);
+        }
+        bail!("failed to unseal vault: {}", status_code);
+    }
+
+    let unseal_response: Value = response.json().await.context("failed to unseal vault")?;
+
+    trace!("Got Response: {}", unseal_response.to_string());
+
+    if matches!(unseal_response["sealed"].as_bool(), Some(true)) {
+        warn!("Vault is still sealed!");
+        println!("Vault is still sealed!");
+    } else {
+        info!("Vault is unsealed!");
+        println!("Vault is unsealed!");
+    }
+    Ok(())
+}

bin/verify-attestation/Cargo.toml

@@ -0,0 +1,13 @@
+[package]
+name = "verify-attestation"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+anyhow.workspace = true
+hex.workspace = true
+intel-tee-quote-verification-rs.workspace = true
+teepot.workspace = true

bin/verify-attestation/Dockerfile

@@ -0,0 +1,46 @@
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-vault-admin.manifest.template' -av /data/ ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p verify-attestation --bin verify-attestation \
+    && mv ./target/x86_64-unknown-linux-gnu/release/verify-attestation ./
+
+FROM docker.io/ubuntu:20.04
+
+RUN apt-get update \
+    && apt-get install -y curl
+
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        libsgx-dcap-default-qpl \
+        libsgx-urts \
+        libsgx-enclave-common \
+        libsgx-dcap-quote-verify
+RUN apt purge -y libsgx-ae-qve
+RUN rm -rf /var/lib/apt/lists/*
+
+COPY --from=buildtee /build/verify-attestation /bin/verify-attestation
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "verify-attestation" ]

bin/verify-attestation/src/main.rs

@@ -0,0 +1,58 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+//! Simple TEE attestation verification test
+
+#![deny(missing_docs)]
+#![deny(clippy::all)]
+
+use anyhow::{bail, Context, Result};
+use std::io::Read;
+use std::time::UNIX_EPOCH;
+use teepot::client::TcbLevel;
+use teepot::sgx::{tee_qv_get_collateral, verify_quote_with_collateral, QuoteVerificationResult};
+
+fn main() -> Result<()> {
+    // read myquote from stdin
+    let mut myquote = Vec::new();
+    std::io::stdin()
+        .read_to_end(&mut myquote)
+        .context("Failed to read quote from stdin")?;
+
+    let collateral = tee_qv_get_collateral(&myquote).context("Failed to get collateral")?;
+
+    let unix_time: i64 = std::time::SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .unwrap()
+        .as_secs() as _;
+
+    let QuoteVerificationResult {
+        collateral_expired,
+        result,
+
+        quote,
+        advisories,
+        ..
+    } = verify_quote_with_collateral(&myquote, Some(&collateral), unix_time.saturating_add(60))
+        .context("Failed to verify quote with collateral")?;
+
+    if collateral_expired {
+        bail!("Freshly fetched collateral expired");
+    }
+
+    let tcblevel = TcbLevel::from(result);
+    if tcblevel != TcbLevel::Ok {
+        println!("Quote verification result: {}", tcblevel);
+    }
+
+    for advisory in advisories {
+        println!("\tInfo: Advisory ID: {advisory}");
+    }
+
+    println!("Quote verified successfully: {}", tcblevel);
+    println!("mrsigner: {}", hex::encode(quote.report_body.mrsigner));
+    println!("mrenclave: {}", hex::encode(quote.report_body.mrenclave));
+    println!("reportdata: {}", hex::encode(quote.report_body.reportdata));
+
+    Ok(())
+}