feat: initial commit

Signed-off-by: Harald Hoyer <harald@matterlabs.dev>

bin/tee-stress-client/Cargo.toml

@@ -0,0 +1,26 @@
+[package]
+name = "tee-stress-client"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+actix-web.workspace = true
+anyhow.workspace = true
+awc.workspace = true
+base64.workspace = true
+bytemuck.workspace = true
+clap.workspace = true
+hex.workspace = true
+mio.workspace = true
+rustls.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+teepot.workspace = true
+tracing-actix-web.workspace = true
+tracing-log.workspace = true
+tracing-subscriber.workspace = true
+tracing.workspace = true

bin/tee-stress-client/Dockerfile-azure

@@ -0,0 +1,85 @@
+FROM docker.io/ubuntu:20.04 AS azuredcap
+WORKDIR /build
+ADD https://github.com/microsoft/Azure-DCAP-Client/archive/refs/tags/1.12.0.tar.gz ./Azure-DCAP-Client.tar.gz
+RUN tar -xvf Azure-DCAP-Client.tar.gz
+COPY assets/Azure-DCAP-Client.patch ./Azure-DCAP-Client.patch
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y software-properties-common; \
+    add-apt-repository ppa:team-xbmc/ppa -y; \
+    apt-get update; \
+    apt-get install -y \
+        build-essential \
+        cmake \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        pkg-config \
+        nlohmann-json3-dev \
+        wget \
+        dos2unix \
+        ;
+
+WORKDIR /build/Azure-DCAP-Client-1.12.0
+RUN dos2unix src/dcap_provider.cpp && patch -p1 < ../Azure-DCAP-Client.patch
+WORKDIR /build/Azure-DCAP-Client-1.12.0/src/Linux
+RUN ./configure && make && make install
+
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-stress-client.manifest.template' -av /data/ ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-stress-client --bin tee-stress-client \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-stress-client ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/microsoft.asc https://packages.microsoft.com/keys/microsoft.asc \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.asc] https://packages.microsoft.com/ubuntu/20.04/prod focal main" > /etc/apt/sources.list.d/msprod.list \
+    && apt-get update \
+    && apt purge -y libsgx-dcap-default-qpl \
+    && apt-get install -y az-dcap-client
+
+RUN apt purge -y libsgx-ae-qve
+#    libsgx-urts
+
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /etc/sgx_default_qcnl.conf
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-stress-client .
+COPY ./bin/tee-stress-client/tee-stress-client.manifest.template .
+COPY vault/enclave-key.pem .
+
+# The original Azure library is still delivering expired collateral, so we have to use a patched version
+COPY --from=azuredcap /usr/local/lib/libdcap_quoteprov.so /usr/lib/
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-stress-client.manifest.template tee-stress-client.manifest \
+    && gramine-sgx-sign --manifest tee-stress-client.manifest --output tee-stress-client.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-stress-client" ]

bin/tee-stress-client/Dockerfile-intel

@@ -0,0 +1,58 @@
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-stress-client.manifest.template' -av /data/ ./
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-stress-client --bin tee-stress-client \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-stress-client ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        libsgx-dcap-default-qpl \
+        libsgx-urts \
+        libsgx-enclave-common \
+        libsgx-dcap-quote-verify
+RUN apt purge -y libsgx-ae-qve
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /lib/libdcap_quoteprov.so
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-stress-client .
+COPY ./bin/tee-stress-client/tee-stress-client.manifest.template .
+COPY vault/enclave-key.pem .
+
+COPY assets/sgx_default_qcnl.conf.json /etc/sgx_default_qcnl.conf
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-stress-client.manifest.template tee-stress-client.manifest \
+    && gramine-sgx-sign --manifest tee-stress-client.manifest --output tee-stress-client.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-stress-client" ]

bin/tee-stress-client/src/main.rs

@@ -0,0 +1,101 @@
+// SPDX-License-Identifier: Apache-2.0
+
+//! Server to handle requests to the Vault TEE
+
+#![deny(missing_docs)]
+#![deny(clippy::all)]
+
+use actix_web::rt::time::sleep;
+use anyhow::{Context, Result};
+use clap::Parser;
+use serde::{Deserialize, Serialize};
+use std::time::Duration;
+use teepot::client::vault::VaultConnection;
+use teepot::server::attestation::{get_quote_and_collateral, VaultAttestationArgs};
+use teepot::server::pki::make_self_signed_cert;
+use teepot::sgx::{parse_tcb_levels, EnumSet, TcbLevel};
+use tracing::{error, trace};
+use tracing_log::LogTracer;
+use tracing_subscriber::Registry;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter};
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Arguments {
+    /// allowed TCB levels, comma separated
+    #[arg(long, value_parser = parse_tcb_levels, env = "ALLOWED_TCB_LEVELS", default_value = "Ok")]
+    my_sgx_allowed_tcb_levels: EnumSet<TcbLevel>,
+    #[clap(flatten)]
+    pub attestation: VaultAttestationArgs,
+}
+
+#[derive(Debug, Serialize, Deserialize)]
+struct MySecret {
+    val: usize,
+}
+
+#[actix_web::main]
+async fn main() -> Result<()> {
+    LogTracer::init().context("Failed to set logger")?;
+
+    let subscriber = Registry::default()
+        .with(EnvFilter::from_default_env())
+        .with(fmt::layer().with_writer(std::io::stderr));
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    let args = Arguments::parse();
+
+    let (report_data, _cert_chain, _priv_key) = make_self_signed_cert()?;
+    if let Err(e) = get_quote_and_collateral(Some(args.my_sgx_allowed_tcb_levels), &report_data) {
+        error!("failed to get quote and collateral: {e:?}");
+        // don't return for now, we can still serve requests but we won't be able to attest
+    }
+
+    let mut vault_1 = args.attestation.clone();
+    let mut vault_2 = args.attestation.clone();
+    let mut vault_3 = args.attestation.clone();
+
+    vault_1.vault_addr = "https://vault-1:8210".to_string();
+    vault_2.vault_addr = "https://vault-2:8210".to_string();
+    vault_3.vault_addr = "https://vault-3:8210".to_string();
+
+    let servers = vec![vault_1.clone(), vault_2.clone(), vault_3.clone()];
+
+    let mut val: usize = 1;
+
+    loop {
+        let mut conns = Vec::new();
+        for server in &servers {
+            match VaultConnection::new(&server.into(), "stress".to_string()).await {
+                Ok(conn) => conns.push(conn),
+                Err(e) => {
+                    error!("connecting to {}: {}", server.vault_addr, e);
+                    continue;
+                }
+            }
+        }
+
+        if conns.is_empty() {
+            error!("no connections");
+            sleep(Duration::from_secs(1)).await;
+            continue;
+        }
+
+        let i = val % conns.len();
+        trace!("storing secret");
+        conns[i]
+            .store_secret(MySecret { val }, "val")
+            .await
+            .context("storing secret")?;
+        for conn in conns {
+            let got: MySecret = conn
+                .load_secret("val")
+                .await
+                .context("loading secret")?
+                .context("loading secret")?;
+            assert_eq!(got.val, val,);
+        }
+        val += 1;
+        sleep(Duration::from_secs(1)).await;
+    }
+}

bin/tee-stress-client/tee-stress-client.manifest.template

@@ -0,0 +1,66 @@
+libos.entrypoint = "/app/tee-stress-client"
+
+[loader]
+argv = [ "/app/tee-stress-client" ]
+entrypoint = "file:{{ gramine.libos }}"
+env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}:/usr{{ arch_libdir }}:/lib"
+env.HOME = "/app"
+env.MALLOC_ARENA_MAX = "1"
+env.AZDCAP_DEBUG_LOG_LEVEL = "ignore"
+env.AZDCAP_COLLATERAL_VERSION = "v4"
+
+### Admin Config ###
+env.PORT  = { passthrough = true }
+
+### VAULT attestation ###
+env.VAULT_ADDR = { passthrough = true }
+env.VAULT_SGX_MRENCLAVE = { passthrough = true }
+env.VAULT_SGX_MRSIGNER = { passthrough = true }
+env.VAULT_SGX_ALLOWED_TCB_LEVELS = { passthrough = true }
+
+### DEBUG ###
+env.RUST_BACKTRACE = "1"
+env.RUST_LOG="info"
+
+[fs]
+root.uri = "file:/"
+start_dir = "/app"
+mounts = [
+  { path = "{{ execdir }}", uri = "file:{{ execdir }}" },
+  { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
+  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
+  { path = "/etc", uri = "file:/etc" },
+  { type = "tmpfs", path = "/var/tmp" },
+  { type = "tmpfs", path = "/tmp" },
+  { type = "tmpfs", path = "/app/.dcap-qcnl" },
+  { type = "tmpfs", path = "/app/.az-dcap-client" },
+  { path = "/lib/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" },
+]
+
+[sgx]
+trusted_files = [
+  "file:/etc/ld.so.cache",
+  "file:/app/",
+  "file:{{ execdir }}/",
+  "file:{{ arch_libdir }}/",
+  "file:/usr/{{ arch_libdir }}/",
+  "file:{{ gramine.libos }}",
+  "file:{{ gramine.runtimedir() }}/",
+  "file:/usr/lib/ssl/openssl.cnf",
+  "file:/etc/ssl/",
+  "file:/etc/sgx_default_qcnl.conf",
+  "file:/lib/libdcap_quoteprov.so",
+]
+remote_attestation = "dcap"
+max_threads = 64
+edmm_enable = false
+## max enclave size
+enclave_size = "8G"
+
+[sys]
+enable_extra_runtime_domain_names_conf = true
+enable_sigterm_injection = true
+
+# possible tweak option, if problems with mio
+# currently mio is compiled with `mio_unsupported_force_waker_pipe`
+# insecure__allow_eventfd = true