feat: initial commit

Signed-off-by: Harald Hoyer <harald@matterlabs.dev>

bin/tee-vault-unseal/Cargo.toml

@@ -0,0 +1,28 @@
+[package]
+name = "tee-vault-unseal"
+version.workspace = true
+edition.workspace = true
+authors.workspace = true
+license.workspace = true
+repository.workspace = true
+
+[dependencies]
+actix-tls.workspace = true
+actix-web.workspace = true
+anyhow.workspace = true
+awc.workspace = true
+clap.workspace = true
+hex.workspace = true
+mio.workspace = true
+rustls-pemfile.workspace = true
+rustls.workspace = true
+serde.workspace = true
+serde_json.workspace = true
+sha2.workspace = true
+teepot.workspace = true
+thiserror.workspace = true
+tokio.workspace = true
+tracing-log.workspace = true
+tracing-subscriber.workspace = true
+tracing.workspace = true
+x509-cert.workspace = true

bin/tee-vault-unseal/Dockerfile-azure

@@ -0,0 +1,92 @@
+FROM ghcr.io/matter-labs/vault-auth-tee:latest AS vault-auth-tee
+
+FROM docker.io/ubuntu:20.04 AS azuredcap
+WORKDIR /build
+ADD https://github.com/microsoft/Azure-DCAP-Client/archive/refs/tags/1.12.0.tar.gz ./Azure-DCAP-Client.tar.gz
+RUN tar -xvf Azure-DCAP-Client.tar.gz
+COPY assets/Azure-DCAP-Client.patch ./Azure-DCAP-Client.patch
+RUN set -eux; \
+    apt-get update; \
+    apt-get install -y software-properties-common; \
+    add-apt-repository ppa:team-xbmc/ppa -y; \
+    apt-get update; \
+    apt-get install -y \
+        build-essential \
+        cmake \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        pkg-config \
+        nlohmann-json3-dev \
+        wget \
+        dos2unix \
+        ;
+
+WORKDIR /build/Azure-DCAP-Client-1.12.0
+RUN dos2unix src/dcap_provider.cpp && patch -p1 < ../Azure-DCAP-Client.patch
+WORKDIR /build/Azure-DCAP-Client-1.12.0/src/Linux
+RUN ./configure && make && make install
+
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+COPY --from=vault-auth-tee /opt/vault/plugins/vault-auth-tee ./
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-vault-unseal.manifest.template' -av /data/ ./
+RUN sha256sum /opt/vault/plugins/vault-auth-tee | ( read a _ ; echo -n $a ) | tee assets/vault-auth-tee.sha256
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-vault-unseal --bin tee-vault-unseal \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-vault-unseal ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/microsoft.asc https://packages.microsoft.com/keys/microsoft.asc \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/microsoft.asc] https://packages.microsoft.com/ubuntu/20.04/prod focal main" > /etc/apt/sources.list.d/msprod.list \
+    && apt-get update \
+    && apt purge -y libsgx-dcap-default-qpl \
+    && apt-get install -y az-dcap-client
+
+RUN apt purge -y libsgx-ae-qve
+#    libsgx-urts
+
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /etc/sgx_default_qcnl.conf
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-vault-unseal .
+COPY ./bin/tee-vault-unseal/tee-vault-unseal.manifest.template .
+COPY vault/enclave-key.pem .
+RUN mkdir -p /opt/vault/tls && rm -rf /opt/vault/tls/*
+
+# The original Azure library is still delivering expired collateral, so we have to use a patched version
+COPY --from=azuredcap /usr/local/lib/libdcap_quoteprov.so /usr/lib/
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-vault-unseal.manifest.template tee-vault-unseal.manifest \
+    && gramine-sgx-sign --manifest tee-vault-unseal.manifest --output tee-vault-unseal.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+VOLUME /opt/vault/tls
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-vault-unseal" ]

bin/tee-vault-unseal/Dockerfile-intel

@@ -0,0 +1,65 @@
+FROM ghcr.io/matter-labs/vault-auth-tee:latest AS vault-auth-tee
+
+FROM docker.io/rust:1-bullseye AS buildtee
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        cmake \
+        rsync \
+        pkg-config \
+        libssl-dev \
+        libcurl4-openssl-dev \
+        libprotobuf-dev \
+        protobuf-compiler \
+        clang \
+        libsgx-headers \
+        libsgx-dcap-quote-verify-dev
+
+WORKDIR /opt/vault/plugins
+COPY --from=vault-auth-tee /opt/vault/plugins/vault-auth-tee ./
+
+WORKDIR /build
+RUN --mount=type=bind,target=/data rsync --exclude='/.git' --filter="dir-merge,- .gitignore" --exclude "Dockerfile-*" --exclude 'tee-vault-unseal.manifest.template' -av /data/ ./
+RUN sha256sum /opt/vault/plugins/vault-auth-tee | ( read a _ ; echo -n $a ) | tee assets/vault-auth-tee.sha256
+RUN --mount=type=cache,target=/usr/local/cargo/registry --mount=type=cache,target=target \
+    RUSTFLAGS="-C target-cpu=icelake-server --cfg mio_unsupported_force_waker_pipe" \
+    cargo build --locked --target x86_64-unknown-linux-gnu --release -p tee-vault-unseal --bin tee-vault-unseal \
+    && mv ./target/x86_64-unknown-linux-gnu/release/tee-vault-unseal ./
+
+FROM docker.io/gramineproject/gramine:v1.5
+
+RUN curl -fsSLo /usr/share/keyrings/intel.asc https://download.01.org/intel-sgx/sgx_repo/ubuntu/intel-sgx-deb.key \
+    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/intel.asc] https://download.01.org/intel-sgx/sgx_repo/ubuntu focal main" > /etc/apt/sources.list.d/intel-sgx.list \
+    && apt-get update \
+    && apt-get install -y --no-install-recommends \
+        libsgx-dcap-default-qpl \
+        libsgx-urts \
+        libsgx-enclave-common \
+        libsgx-dcap-quote-verify
+RUN apt purge -y libsgx-ae-qve
+RUN rm -rf /var/lib/apt/lists/*
+
+# So we only have to use one gramine template
+RUN touch /lib/libdcap_quoteprov.so
+
+WORKDIR /app
+
+COPY --from=buildtee /build/tee-vault-unseal .
+COPY ./bin/tee-vault-unseal/tee-vault-unseal.manifest.template .
+COPY vault/enclave-key.pem .
+RUN mkdir -p /opt/vault/tls && rm -rf /opt/vault/tls/*
+
+COPY assets/sgx_default_qcnl.conf.json /etc/sgx_default_qcnl.conf
+
+RUN gramine-manifest -Darch_libdir=/lib/x86_64-linux-gnu -Dexecdir=/usr/bin -Dlog_level=warning tee-vault-unseal.manifest.template tee-vault-unseal.manifest \
+    && gramine-sgx-sign --manifest tee-vault-unseal.manifest --output tee-vault-unseal.manifest.sgx --key enclave-key.pem \
+    && rm enclave-key.pem
+
+VOLUME /opt/vault/tls
+
+EXPOSE 8443
+
+ENTRYPOINT ["/bin/sh", "-c"]
+CMD [ "/restart_aesm.sh ; exec gramine-sgx tee-vault-unseal" ]

bin/tee-vault-unseal/src/admin-policy.hcl

@@ -0,0 +1,80 @@
+# Read system health check
+path "sys/health"
+{
+  capabilities = ["read", "sudo"]
+}
+
+# Create and manage ACL policies broadly across Vault
+
+# List existing policies
+path "sys/policies/acl"
+{
+  capabilities = ["list"]
+}
+
+# Create and manage ACL policies
+path "sys/policies/acl/*"
+{
+  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+}
+
+# Enable and manage authentication methods broadly across Vault
+
+# Manage auth methods broadly across Vault
+path "auth/*"
+{
+  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+}
+
+# Create, update, and delete auth methods
+path "sys/auth/*"
+{
+  capabilities = ["create", "update", "delete", "sudo"]
+}
+
+# List auth methods
+path "sys/auth"
+{
+  capabilities = ["read"]
+}
+
+# Enable and manage the key/value secrets engine at `secret/` path
+
+# List, create, update, and delete key/value secrets
+path "secret/*"
+{
+  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+}
+
+# Manage secrets engines
+path "sys/mounts/*"
+{
+  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+}
+
+# List existing secrets engines.
+path "sys/mounts"
+{
+  capabilities = ["read"]
+}
+
+# Manage plugins
+# https://developer.hashicorp.com/vault/api-docs/system/plugins-catalog
+path "sys/plugins/catalog/*"
+{
+  capabilities = ["create", "read", "update", "delete", "list", "sudo"]
+}
+
+# List existing plugins
+# https://developer.hashicorp.com/vault/api-docs/system/plugins-catalog
+path "sys/plugins/catalog"
+{
+  capabilities = ["list"]
+}
+
+# Reload plugins
+# https://developer.hashicorp.com/vault/api-docs/system/plugins-reload-backend
+path "sys/plugins/reload/backend"
+{
+  capabilities = ["create", "update", "sudo"]
+}

bin/tee-vault-unseal/src/attestation.rs

@@ -0,0 +1,27 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+use crate::Worker;
+use actix_web::http::StatusCode;
+use actix_web::web::{Data, Json};
+use anyhow::{Context, Result};
+use teepot::json::http::AttestationResponse;
+use teepot::server::attestation::get_quote_and_collateral;
+use teepot::server::{HttpResponseError, Status};
+use tracing::instrument;
+
+#[instrument(level = "info", name = "/v1/sys/attestation", skip_all)]
+pub async fn get_attestation(
+    worker: Data<Worker>,
+) -> Result<Json<AttestationResponse>, HttpResponseError> {
+    let report_data: [u8; 64] = worker
+        .config
+        .report_data
+        .clone()
+        .try_into()
+        .map_err(|_| "Error getting attestation")?;
+    get_quote_and_collateral(None, &report_data)
+        .context("Error getting attestation")
+        .map(Json)
+        .status(StatusCode::INTERNAL_SERVER_ERROR)
+}

bin/tee-vault-unseal/src/init.rs

@@ -0,0 +1,135 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+use crate::{create_https_client, get_vault_status, UnsealServerState, Worker};
+use actix_web::error::ErrorBadRequest;
+use actix_web::{web, HttpResponse};
+use anyhow::{anyhow, Context, Result};
+use awc::http::StatusCode;
+use serde_json::json;
+use teepot::json::http::{Init, InitResponse, VaultInitRequest};
+use teepot::json::secrets::AdminConfig;
+use teepot::server::{HttpResponseError, Status};
+use tracing::{debug, error, info, instrument, trace};
+
+#[instrument(level = "info", name = "/v1/sys/init", skip_all)]
+pub async fn post_init(
+    worker: web::Data<Worker>,
+    init: web::Json<Init>,
+) -> Result<HttpResponse, HttpResponseError> {
+    let Init {
+        pgp_keys,
+        secret_shares,
+        secret_threshold,
+        admin_pgp_keys,
+        admin_threshold,
+        admin_tee_mrenclave,
+    } = init.into_inner();
+    let client = create_https_client(worker.client_tls_config.clone());
+    let vault_url = &worker.config.vault_url;
+
+    let vault_init = VaultInitRequest {
+        pgp_keys,
+        secret_shares,
+        secret_threshold,
+    };
+
+    if admin_threshold < 1 {
+        return Ok(HttpResponse::from_error(ErrorBadRequest(
+            json!({"error": "admin_threshold must be at least 1"}),
+        )));
+    }
+
+    if admin_threshold > admin_pgp_keys.len() {
+        return Ok(HttpResponse::from_error(ErrorBadRequest(
+            json!({"error": "admin_threshold must be less than or equal to the number of admin_pgp_keys"}),
+        )));
+    }
+
+    loop {
+        let current_state = worker.state.read().unwrap().clone();
+        match current_state {
+            UnsealServerState::VaultUninitialized => {
+                break;
+            }
+            UnsealServerState::VaultUnsealed => {
+                return Err(anyhow!("Vault already unsealed")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultInitialized { .. } => {
+                return Err(anyhow!("Vault already initialized")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultInitializedAndConfigured => {
+                return Err(anyhow!("Vault already initialized")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::Undefined => {
+                let state = get_vault_status(vault_url, client.clone()).await;
+                *worker.state.write().unwrap() = state;
+                continue;
+            }
+        }
+    }
+
+    trace!(
+        "Sending init request to Vault {}",
+        serde_json::to_string(&vault_init).unwrap()
+    );
+    let mut response = client
+        .post(format!("{}/v1/sys/init", vault_url))
+        .send_json(&vault_init)
+        .await?;
+
+    let status_code = response.status();
+    if !status_code.is_success() {
+        error!("Vault returned server error: {}", status_code);
+        return Err(HttpResponseError::from_proxy(response).await);
+    }
+
+    let response = response
+        .json::<serde_json::Value>()
+        .await
+        .context("Failed to convert to json")
+        .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+    info!("Vault initialized");
+    trace!("response {}", response);
+
+    let root_token = response["root_token"]
+        .as_str()
+        .ok_or(anyhow!("No `root_token` field"))
+        .status(StatusCode::BAD_GATEWAY)?
+        .to_string();
+
+    debug!("Root token: {root_token}");
+
+    let unseal_keys = response["keys_base64"]
+        .as_array()
+        .ok_or(anyhow!("No `keys_base64` field"))
+        .status(StatusCode::BAD_GATEWAY)?
+        .iter()
+        .map(|v| v.as_str().unwrap().to_string())
+        .collect::<Vec<_>>();
+
+    debug!("Unseal keys: {}", unseal_keys.join(", "));
+
+    /*
+    FIXME: use unseal keys to create new token
+        let mut output = File::create("/opt/vault/data/root_token")
+            .context("Failed to create `/opt/vault/data/root_token`")?;
+        output
+            .write_all(root_token.as_bytes())
+            .context("Failed to write root_token")?;
+    */
+
+    *worker.state.write().unwrap() = UnsealServerState::VaultInitialized {
+        admin_config: AdminConfig {
+            admin_pgp_keys,
+            admin_threshold,
+        },
+        admin_tee_mrenclave,
+        root_token,
+    };
+
+    let response = InitResponse { unseal_keys };
+
+    Ok(HttpResponse::Ok().json(response)) // <- send response
+}

bin/tee-vault-unseal/src/main.rs

@@ -0,0 +1,366 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+//! Server to initialize and unseal the Vault TEE.
+
+#![deny(missing_docs)]
+#![deny(clippy::all)]
+
+mod attestation;
+mod init;
+mod unseal;
+
+use actix_web::http::header;
+use actix_web::rt::time::sleep;
+use actix_web::web::Data;
+use actix_web::{web, App, HttpServer};
+use anyhow::{Context, Result};
+use attestation::get_attestation;
+use awc::{Client, Connector};
+use clap::Parser;
+use init::post_init;
+use rustls::client::danger::{HandshakeSignatureValid, ServerCertVerified, ServerCertVerifier};
+use rustls::client::WebPkiServerVerifier;
+use rustls::pki_types::{CertificateDer, ServerName, UnixTime};
+use rustls::{ClientConfig, DigitallySignedStruct, Error, ServerConfig, SignatureScheme};
+use rustls_pemfile::{certs, read_one};
+use sha2::{Digest, Sha256};
+use std::fmt::Debug;
+use std::net::Ipv6Addr;
+use std::sync::{Arc, RwLock};
+use std::time::Duration;
+use std::{fs::File, io::BufReader};
+use teepot::json::http::{Init, Unseal, ATTESTATION_URL};
+use teepot::json::secrets::AdminConfig;
+use teepot::server::attestation::get_quote_and_collateral;
+use teepot::server::new_json_cfg;
+use teepot::sgx::{parse_tcb_levels, EnumSet, TcbLevel};
+use tracing::{error, info, trace};
+use tracing_log::LogTracer;
+use tracing_subscriber::{fmt, prelude::*, EnvFilter, Registry};
+use unseal::post_unseal;
+use x509_cert::der::Decode as _;
+use x509_cert::der::Encode as _;
+use x509_cert::Certificate;
+
+const VAULT_AUTH_TEE_SHA256: &str = include_str!("../../../assets/vault-auth-tee.sha256");
+const VAULT_TOKEN_HEADER: &str = "X-Vault-Token";
+
+/// Worker thread state and data
+pub struct Worker {
+    /// TLS config for the HTTPS client
+    pub client_tls_config: Arc<ClientConfig>,
+    /// Server config
+    pub config: Arc<UnsealServerConfig>,
+    /// Server state
+    pub state: Arc<RwLock<UnsealServerState>>,
+}
+
+/// Global Server config
+#[derive(Debug, Default)]
+pub struct UnsealServerConfig {
+    /// Vault URL
+    pub vault_url: String,
+    /// The expected report_data for the Vault TEE
+    pub report_data: Vec<u8>,
+    /// allowed TCB levels
+    pub allowed_tcb_levels: Option<EnumSet<TcbLevel>>,
+}
+
+/// Server state
+#[derive(Debug, Clone)]
+pub enum UnsealServerState {
+    /// Undefined
+    Undefined,
+    /// Vault is not yet initialized
+    VaultUninitialized,
+    /// Vault is initialized but not unsealed
+    VaultInitialized {
+        /// config for the admin TEE
+        admin_config: AdminConfig,
+        /// initial admin TEE mrenclave
+        admin_tee_mrenclave: String,
+        /// Vault root token
+        root_token: String,
+    },
+    /// Vault is already initialized but not unsealed
+    /// and should already be configured
+    VaultInitializedAndConfigured,
+    /// Vault is unsealed
+    VaultUnsealed,
+}
+
+impl UnsealServerConfig {
+    /// Create a new ServerState
+    pub fn new(
+        vault_url: String,
+        report_data: [u8; 64],
+        allowed_tcb_levels: Option<EnumSet<TcbLevel>>,
+    ) -> Self {
+        Self {
+            report_data: report_data.to_vec(),
+            vault_url,
+            allowed_tcb_levels,
+        }
+    }
+}
+
+#[derive(Parser, Debug)]
+#[command(author, version, about, long_about = None)]
+struct Args {
+    /// allowed TCB levels, comma separated
+    #[arg(long, value_parser = parse_tcb_levels, env = "ALLOWED_TCB_LEVELS", default_value = "Ok")]
+    allowed_tcb_levels: EnumSet<TcbLevel>,
+    /// port to listen on
+    #[arg(long, env = "PORT", default_value = "8443")]
+    port: u16,
+    /// vault url
+    #[arg(long, env = "VAULT_ADDR", default_value = "https://vault:8210")]
+    vault_url: String,
+}
+
+#[actix_web::main]
+async fn main() -> Result<()> {
+    LogTracer::init().context("Failed to set logger")?;
+
+    let subscriber = Registry::default()
+        .with(EnvFilter::from_default_env())
+        .with(
+            fmt::layer()
+                .with_span_events(fmt::format::FmtSpan::NEW)
+                .with_writer(std::io::stderr),
+        );
+    tracing::subscriber::set_global_default(subscriber).unwrap();
+
+    let args = Args::parse();
+
+    let tls_ok = std::path::Path::new("/opt/vault/tls/tls.ok");
+    loop {
+        info!("Waiting for TLS key/cert files to be generated");
+
+        // Wait for the file `data/tls.key` to exist
+        if tls_ok.exists() {
+            break;
+        }
+        sleep(Duration::from_secs(1)).await;
+    }
+
+    info!("Starting up");
+
+    let (config, client_tls_config, report_data) = load_rustls_config().or_else(|e| {
+        error!("failed to load rustls config: {e:?}");
+        Err(e).context("Failed to load rustls config")
+    })?;
+
+    if let Err(e) = get_quote_and_collateral(Some(args.allowed_tcb_levels), &report_data) {
+        error!("failed to get quote and collateral: {e:?}");
+        // don't return for now, we can still serve requests but we won't be able to attest
+    }
+
+    let client = create_https_client(client_tls_config.clone());
+
+    let server_state = get_vault_status(&args.vault_url, client).await;
+
+    info!("Starting HTTPS server at port {}", args.port);
+    let server_config = Arc::new(UnsealServerConfig::new(
+        args.vault_url,
+        report_data,
+        Some(args.allowed_tcb_levels),
+    ));
+
+    let server_state = Arc::new(RwLock::new(server_state));
+
+    let server = match HttpServer::new(move || {
+        let worker = Worker {
+            client_tls_config: client_tls_config.clone(),
+            config: server_config.clone(),
+            state: server_state.clone(),
+        };
+
+        App::new()
+            // enable logger
+            //.wrap(TracingLogger::default())
+            .app_data(new_json_cfg())
+            .app_data(Data::new(worker))
+            .service(web::resource(ATTESTATION_URL).route(web::get().to(get_attestation)))
+            .service(web::resource(Init::URL).route(web::post().to(post_init)))
+            .service(web::resource(Unseal::URL).route(web::post().to(post_unseal)))
+    })
+    .bind_rustls_0_22((Ipv6Addr::UNSPECIFIED, args.port), config)
+    {
+        Ok(c) => c,
+        Err(e) => {
+            error!("Failed to bind to port {}: {e:?}", args.port);
+            return Err(e).context(format!("Failed to bind to port {}", args.port));
+        }
+    };
+
+    if let Err(e) = server.worker_max_blocking_threads(2).workers(8).run().await {
+        error!("failed to start HTTPS server: {e:?}");
+        return Err(e).context("Failed to start HTTPS server");
+    }
+
+    Ok(())
+}
+
+async fn get_vault_status(vault_url: &str, client: Client) -> UnsealServerState {
+    loop {
+        let r = client
+            .get(format!("{}/v1/sys/health", vault_url))
+            .send()
+            .await;
+
+        if let Ok(r) = r {
+            // https://developer.hashicorp.com/vault/api-docs/system/health
+            match r.status().as_u16() {
+                200 | 429 | 472 | 473 => {
+                    info!("Vault is initialized and unsealed");
+                    break UnsealServerState::VaultUnsealed;
+                }
+                501 => {
+                    info!("Vault is not initialized");
+                    break UnsealServerState::VaultUninitialized;
+                }
+                503 => {
+                    info!("Vault is initialized but not unsealed");
+                    break UnsealServerState::VaultInitializedAndConfigured;
+                }
+                s => {
+                    error!("Vault is not ready: status code {s}");
+                }
+            }
+        }
+        info!("Waiting for vault to be ready");
+        sleep(Duration::from_secs(1)).await;
+    }
+}
+
+// Save the hash of the public server key to `REPORT_DATA` to check
+// the attestations against it and it does not change on reconnect.
+fn make_verifier(server_cert: Box<[u8]>) -> impl ServerCertVerifier {
+    #[derive(Debug)]
+    struct V {
+        server_cert: Box<[u8]>,
+        server_verifier: Arc<WebPkiServerVerifier>,
+    }
+    impl ServerCertVerifier for V {
+        fn verify_server_cert(
+            &self,
+            end_entity: &CertificateDer,
+            _intermediates: &[CertificateDer],
+            _server_name: &ServerName,
+            _ocsp_response: &[u8],
+            _now: UnixTime,
+        ) -> std::result::Result<ServerCertVerified, Error> {
+            let data = &self.server_cert;
+
+            if data.as_ref() == end_entity.as_ref() {
+                info!("Server certificate matches expected certificate");
+                Ok(ServerCertVerified::assertion())
+            } else {
+                error!("Server certificate does not match expected certificate");
+                Err(rustls::Error::General(
+                    "Server certificate does not match expected certificate".to_string(),
+                ))
+            }
+        }
+        fn verify_tls12_signature(
+            &self,
+            message: &[u8],
+            cert: &CertificateDer<'_>,
+            dss: &DigitallySignedStruct,
+        ) -> std::result::Result<HandshakeSignatureValid, Error> {
+            self.server_verifier
+                .verify_tls12_signature(message, cert, dss)
+        }
+
+        fn verify_tls13_signature(
+            &self,
+            message: &[u8],
+            cert: &CertificateDer<'_>,
+            dss: &DigitallySignedStruct,
+        ) -> std::result::Result<HandshakeSignatureValid, Error> {
+            self.server_verifier
+                .verify_tls13_signature(message, cert, dss)
+        }
+
+        fn supported_verify_schemes(&self) -> Vec<SignatureScheme> {
+            self.server_verifier.supported_verify_schemes()
+        }
+    }
+    let root_store = Arc::new(rustls::RootCertStore::empty());
+    let server_verifier = WebPkiServerVerifier::builder(root_store).build().unwrap();
+    V {
+        server_cert,
+        server_verifier,
+    }
+}
+
+/// Load TLS key/cert files
+pub fn load_rustls_config() -> Result<(ServerConfig, Arc<ClientConfig>, [u8; 64])> {
+    // init server config builder with safe defaults
+    let config = ServerConfig::builder().with_no_client_auth();
+
+    // load TLS key/cert files
+    let cert_file = &mut BufReader::new(
+        File::open("/opt/vault/tls/tls.crt").context("Failed to open TLS cert file")?,
+    );
+    let key_file = &mut BufReader::new(
+        File::open("/opt/vault/tls/tls.key").context("Failed to open TLS key file")?,
+    );
+
+    // convert files to key/cert objects
+    let cert_chain: Vec<_> = certs(cert_file)
+        .unwrap()
+        .into_iter()
+        .map(rustls::pki_types::CertificateDer::from)
+        .collect();
+    let priv_key: rustls::pki_types::PrivateKeyDer = match read_one(key_file).unwrap() {
+        Some(rustls_pemfile::Item::RSAKey(key)) => {
+            rustls::pki_types::PrivatePkcs1KeyDer::from(key).into()
+        }
+        Some(rustls_pemfile::Item::PKCS8Key(key)) => {
+            rustls::pki_types::PrivatePkcs8KeyDer::from(key).into()
+        }
+        _ => panic!("no keys found"),
+    };
+
+    let tls_config = Arc::new(
+        rustls::ClientConfig::builder()
+            .dangerous()
+            .with_custom_certificate_verifier(Arc::new(make_verifier(
+                cert_chain[0].as_ref().into(),
+            )))
+            .with_no_client_auth(),
+    );
+
+    let cert = Certificate::from_der(cert_chain[0].as_ref()).unwrap();
+    let pub_key = cert
+        .tbs_certificate
+        .subject_public_key_info
+        .to_der()
+        .unwrap();
+
+    let hash = Sha256::digest(pub_key);
+    let mut report_data = [0u8; 64];
+    report_data[..32].copy_from_slice(&hash[..32]);
+
+    let report_data_hex = hex::encode(report_data);
+    trace!(report_data_hex);
+
+    let config = config
+        .with_single_cert(cert_chain, priv_key)
+        .context("Failed to load TLS key/cert files")?;
+
+    Ok((config, tls_config, report_data))
+}
+
+/// Create an HTTPS client with the default headers and config
+pub fn create_https_client(client_tls_config: Arc<ClientConfig>) -> Client {
+    Client::builder()
+        .add_default_header((header::USER_AGENT, "teepot/1.0"))
+        // a "connector" wraps the stream into an encrypted connection
+        .connector(Connector::new().rustls_0_22(client_tls_config))
+        .timeout(Duration::from_secs(12000))
+        .finish()
+}

bin/tee-vault-unseal/src/unseal.rs

@@ -0,0 +1,412 @@
+// SPDX-License-Identifier: Apache-2.0
+// Copyright (c) 2023 Matter Labs
+
+use crate::{
+    create_https_client, get_vault_status, UnsealServerConfig, UnsealServerState, Worker,
+    VAULT_AUTH_TEE_SHA256, VAULT_TOKEN_HEADER,
+};
+use actix_web::http::StatusCode;
+use actix_web::rt::time::sleep;
+use actix_web::{web, HttpResponse};
+use anyhow::{anyhow, Context, Result};
+use awc::{Client, ClientRequest, SendClientRequest};
+use serde_json::{json, Value};
+use std::fs::File;
+use std::future::Future;
+use std::io::Read;
+use std::time::Duration;
+use teepot::client::vault::VaultConnection;
+use teepot::json::http::Unseal;
+use teepot::json::secrets::{AdminConfig, AdminState};
+use teepot::server::{HttpResponseError, Status};
+use tracing::{debug, error, info, instrument, trace};
+
+#[instrument(level = "info", name = "/v1/sys/unseal", skip_all)]
+pub async fn post_unseal(
+    worker: web::Data<Worker>,
+    item: web::Json<Unseal>,
+) -> Result<HttpResponse, HttpResponseError> {
+    let client = create_https_client(worker.client_tls_config.clone());
+    let app = &worker.config;
+    let vault_url = &app.vault_url;
+
+    loop {
+        let current_state = worker.state.read().unwrap().clone();
+        match current_state {
+            UnsealServerState::VaultUninitialized => {
+                return Err(anyhow!("Vault not yet initialized")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultUnsealed => {
+                return Err(anyhow!("Vault already unsealed")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultInitialized { .. } => {
+                break;
+            }
+            UnsealServerState::VaultInitializedAndConfigured => {
+                break;
+            }
+            UnsealServerState::Undefined => {
+                let state = get_vault_status(vault_url, client.clone()).await;
+                *worker.state.write().unwrap() = state;
+                continue;
+            }
+        }
+    }
+
+    let mut response = client
+        .post(format!("{}/v1/sys/unseal", vault_url))
+        .send_json(&item.0)
+        .await?;
+
+    let status_code = response.status();
+    if !status_code.is_success() {
+        error!("Vault returned server error: {}", status_code);
+        let mut client_resp = HttpResponse::build(status_code);
+        for (header_name, header_value) in response.headers().iter() {
+            client_resp.insert_header((header_name.clone(), header_value.clone()));
+        }
+        return Ok(client_resp.streaming(response));
+    }
+
+    let response: Value = response
+        .json()
+        .await
+        .context("parsing unseal response")
+        .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+    debug!("unseal: {:?}", response);
+
+    if response.get("errors").is_some() {
+        return Ok(HttpResponse::Ok().json(response));
+    }
+
+    let sealed = response
+        .get("sealed")
+        .map(|v| v.as_bool().unwrap_or(true))
+        .unwrap_or(true);
+
+    debug!(sealed);
+
+    // if unsealed
+    if !sealed {
+        let mut state = UnsealServerState::VaultUnsealed;
+        std::mem::swap(&mut *worker.state.write().unwrap(), &mut state);
+
+        match state {
+            UnsealServerState::VaultUninitialized => {
+                return Err(anyhow!("Invalid internal state")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultUnsealed => {
+                return Err(anyhow!("Invalid internal state")).status(StatusCode::BAD_REQUEST);
+            }
+            UnsealServerState::VaultInitialized {
+                admin_config,
+                admin_tee_mrenclave,
+                root_token,
+            } => {
+                debug!(root_token);
+                info!("Vault is unsealed");
+                let app = &worker.config;
+                let client = create_https_client(worker.client_tls_config.clone());
+
+                vault_configure_unsealed(
+                    app,
+                    &admin_config,
+                    &root_token,
+                    &admin_tee_mrenclave,
+                    &client,
+                )
+                .await
+                .context("Failed to configure unsealed vault")
+                .status(StatusCode::BAD_GATEWAY)?;
+
+                // destroy root token
+                let _response = client
+                    .post(format!("{}/v1/auth/token/revoke-self", app.vault_url))
+                    .insert_header((VAULT_TOKEN_HEADER, root_token.to_string()))
+                    .send()
+                    .await;
+
+                info!("Vault unsealed and configured!");
+            }
+            UnsealServerState::VaultInitializedAndConfigured => {
+                info!("Vault is unsealed and hopefully configured!");
+                info!("Initiating raft join");
+                // load TLS cert chain
+                let mut cert_file = File::open("/opt/vault/tls/cacert.pem")
+                    .context("Failed to open TLS cert chain")
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+                let mut cert_buf = Vec::new();
+                cert_file
+                    .read_to_end(&mut cert_buf)
+                    .context("Failed to read TLS cert chain")
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+                let cert_chain = std::str::from_utf8(&cert_buf)
+                    .context("Failed to parse TLS cert chain as UTF-8")
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)?
+                    .to_string();
+
+                let payload = json!({"leader_ca_cert": cert_chain, "retry": true });
+
+                let mut response = client
+                    .post(format!("{}/v1/sys/storage/raft/join", vault_url))
+                    .send_json(&payload)
+                    .await?;
+
+                let status_code = response.status();
+                if !status_code.is_success() {
+                    error!("Vault returned server error: {}", status_code);
+                    let mut client_resp = HttpResponse::build(status_code);
+                    for (header_name, header_value) in response.headers().iter() {
+                        client_resp.insert_header((header_name.clone(), header_value.clone()));
+                    }
+                    return Ok(client_resp.streaming(response));
+                }
+
+                let response: Value = response
+                    .json()
+                    .await
+                    .context("parsing raft join response")
+                    .status(StatusCode::INTERNAL_SERVER_ERROR)?;
+
+                debug!("raft join: {:?}", response);
+
+                if response.get("errors").is_some() {
+                    return Ok(HttpResponse::Ok().json(response));
+                }
+            }
+            UnsealServerState::Undefined => {
+                unreachable!("Invalid internal state");
+            }
+        }
+    }
+
+    Ok(HttpResponse::Accepted().json(response)) // <- send response
+}
+
+pub async fn vault_configure_unsealed(
+    app: &UnsealServerConfig,
+    admin_config: &AdminConfig,
+    root_token: &str,
+    admin_tee_mrenclave: &str,
+    c: &Client,
+) -> Result<(), HttpResponseError> {
+    wait_for_plugins_catalog(app, root_token, c).await;
+
+    if !plugin_is_already_running(app, root_token, c).await? {
+        let r = vault(
+            "Installing vault-auth-tee plugin",
+            c.put(format!(
+                "{}/v1/sys/plugins/catalog/auth/vault-auth-tee",
+                app.vault_url
+            )),
+            root_token,
+            json!({
+                "sha256": VAULT_AUTH_TEE_SHA256,
+                "command": "vault-auth-tee",
+                "version": "0.1.0+dev"
+            }),
+        )
+        .await
+        .map_err(|e| anyhow!("{:?}", e))
+        .status(StatusCode::BAD_GATEWAY)?;
+        if !r.status().is_success() {
+            let err = HttpResponseError::from_proxy(r).await;
+            return Err(err);
+        }
+    } else {
+        info!("vault-auth-tee plugin already installed");
+    }
+
+    if !plugin_is_already_running(app, root_token, c).await? {
+        let r = vault(
+            "Activating vault-auth-tee plugin",
+            c.post(format!("{}/v1/sys/auth/tee", app.vault_url)),
+            root_token,
+            json!({"type": "vault-auth-tee"}),
+        )
+        .await
+        .map_err(|e| anyhow!("{:?}", e))
+        .status(StatusCode::BAD_GATEWAY)?;
+        if !r.status().is_success() {
+            let err = HttpResponseError::from_proxy(r).await;
+            return Err(err);
+        }
+    } else {
+        info!("vault-auth-tee plugin already activated");
+    }
+
+    if let Ok(mut r) = c
+        .get(format!("{}/v1/auth/tee/tees?list=true", app.vault_url))
+        .insert_header((VAULT_TOKEN_HEADER, root_token))
+        .send()
+        .await
+    {
+        let r: Value = r
+            .json()
+            .await
+            .map_err(|e| anyhow!("{:?}", e))
+            .status(StatusCode::BAD_GATEWAY)?;
+        trace!("{:?}", r);
+        if let Some(tees) = r.get("data").and_then(|v| v.get("keys")) {
+            if let Some(tees) = tees.as_array() {
+                if tees.contains(&json!("root")) {
+                    info!("root TEE already installed");
+                    return Ok(());
+                }
+            }
+        }
+    }
+
+    vault(
+        "Installing root TEE",
+        c.put(format!("{}/v1/auth/tee/tees/admin", app.vault_url)),
+        root_token,
+        json!({
+            "lease": "1000",
+            "name": "admin",
+            "types": "sgx",
+            "sgx_allowed_tcb_levels": "Ok,SwHardeningNeeded",
+            "sgx_mrenclave": &admin_tee_mrenclave,
+            "token_policies": "admin"
+        }),
+    )
+    .await
+    .map_err(|e| anyhow!("{:?}", e))
+    .status(StatusCode::BAD_GATEWAY)?;
+
+    // Install admin policies
+    let admin_policy = include_str!("admin-policy.hcl");
+    vault(
+        "Installing admin policy",
+        c.put(format!("{}/v1/sys/policies/acl/admin", app.vault_url)),
+        root_token,
+        json!({ "policy": admin_policy }),
+    )
+    .await
+    .map_err(|e| anyhow!("{:?}", e))
+    .status(StatusCode::BAD_GATEWAY)?;
+
+    vault(
+        "Enable the key/value secrets engine v1 at secret/.",
+        c.put(format!("{}/v1/sys/mounts/secret", app.vault_url)),
+        root_token,
+        json!({ "type": "kv", "description": "K/V v1" } ),
+    )
+    .await
+    .map_err(|e| anyhow!("{:?}", e))
+    .status(StatusCode::BAD_GATEWAY)?;
+
+    // Create a `VaultConnection` for the `admin` tee to initialize the secrets for it.
+    // Safety: the connection was already attested
+    let admin_vcon = unsafe {
+        VaultConnection::new_from_client_without_attestation(
+            app.vault_url.clone(),
+            c.clone(),
+            "admin".into(),
+            root_token.to_string(),
+        )
+    };
+
+    // initialize the admin config
+    admin_vcon.store_secret(admin_config, "config").await?;
+    admin_vcon
+        .store_secret(AdminState::default(), "state")
+        .await?;
+
+    Ok(())
+}
+
+async fn wait_for_plugins_catalog(app: &UnsealServerConfig, root_token: &str, c: &Client) {
+    info!("Waiting for plugins to be loaded");
+    loop {
+        let r = c
+            .get(format!("{}/v1/sys/plugins/catalog", app.vault_url))
+            .insert_header((VAULT_TOKEN_HEADER, root_token))
+            .send()
+            .await;
+
+        match r {
+            Ok(r) => {
+                if r.status().is_success() {
+                    break;
+                } else {
+                    debug!("/v1/sys/plugins/catalog status: {:#?}", r)
+                }
+            }
+            Err(e) => {
+                debug!("/v1/sys/plugins/catalog error: {}", e)
+            }
+        }
+
+        info!("Waiting for plugins to be loaded");
+        sleep(Duration::from_secs(1)).await;
+    }
+}
+
+async fn plugin_is_already_running(
+    app: &UnsealServerConfig,
+    root_token: &str,
+    c: &Client,
+) -> std::result::Result<bool, HttpResponseError> {
+    if let Ok(mut r) = c
+        .get(format!("{}/v1/sys/auth", app.vault_url))
+        .insert_header((VAULT_TOKEN_HEADER, root_token))
+        .send()
+        .await
+    {
+        if !r.status().is_success() {
+            return Ok(false);
+        }
+        let r: Value = r
+            .json()
+            .await
+            .map_err(|e| anyhow!("{:?}", e))
+            .status(StatusCode::BAD_GATEWAY)?;
+        trace!("{}", r.to_string());
+
+        let is_running = r
+            .get("data")
+            .and_then(|v| v.get("tee/"))
+            .and_then(|v| v.get("running_sha256"))
+            .and_then(|v| v.as_str())
+            .and_then(|v| if v.is_empty() { None } else { Some(v) })
+            .and_then(|v| {
+                if v == VAULT_AUTH_TEE_SHA256 {
+                    Some(v)
+                } else {
+                    None
+                }
+            })
+            .is_some();
+        Ok(is_running)
+    } else {
+        Ok(false)
+    }
+}
+
+async fn vault(
+    action: &str,
+    req: ClientRequest,
+    token: &str,
+    json: Value,
+) -> <SendClientRequest as Future>::Output {
+    info!("{}", action);
+    debug!("json: {:?}", json);
+    match req
+        .insert_header((VAULT_TOKEN_HEADER, token))
+        .send_json(&json)
+        .await
+    {
+        Ok(r) => {
+            debug!("response {:?}", r);
+            Ok(r)
+        }
+        Err(e) => {
+            error!("{}: {}", action, e);
+            Err(e)
+        }
+    }
+}

bin/tee-vault-unseal/tee-vault-unseal.manifest.template

@@ -0,0 +1,62 @@
+libos.entrypoint = "/app/tee-vault-unseal"
+
+[loader]
+argv = [ "/app/tee-vault-unseal" ]
+entrypoint = "file:{{ gramine.libos }}"
+env.LD_LIBRARY_PATH = "/lib:{{ arch_libdir }}:/usr{{ arch_libdir }}:/lib"
+env.HOME = "/app"
+env.MALLOC_ARENA_MAX = "1"
+env.AZDCAP_DEBUG_LOG_LEVEL = "ignore"
+env.AZDCAP_COLLATERAL_VERSION = "v4"
+
+### Required configuration ###
+env.ALLOWED_TCB_LEVELS = { passthrough = true }
+env.VAULT_ADDR = { passthrough = true }
+
+### DEBUG ###
+env.RUST_BACKTRACE = "1"
+env.RUST_LOG="info,tee_vault_unseal=trace,teepot=trace,awc=debug"
+
+[fs]
+root.uri = "file:/"
+start_dir = "/app"
+mounts = [
+  { path = "{{ execdir }}", uri = "file:{{ execdir }}" },
+  { path = "/lib", uri = "file:{{ gramine.runtimedir() }}" },
+  { path = "{{ arch_libdir }}", uri = "file:{{ arch_libdir }}" },
+  { path = "/etc", uri = "file:/etc" },
+  { type = "tmpfs", path = "/var/tmp" },
+  { type = "tmpfs", path = "/tmp" },
+  { type = "tmpfs", path = "/app/.dcap-qcnl" },
+  { type = "tmpfs", path = "/app/.az-dcap-client" },
+  { type = "encrypted", path = "/opt/vault/tls", uri = "file:/opt/vault/tls", key_name = "_sgx_mrsigner" },
+  { path = "/lib/libdcap_quoteprov.so", uri = "file:/lib/libdcap_quoteprov.so" },
+]
+
+[sgx]
+trusted_files = [
+  "file:/etc/ld.so.cache",
+  "file:/app/",
+  "file:{{ execdir }}/",
+  "file:{{ arch_libdir }}/",
+  "file:/usr/{{ arch_libdir }}/",
+  "file:{{ gramine.libos }}",
+  "file:{{ gramine.runtimedir() }}/",
+  "file:/usr/lib/ssl/openssl.cnf",
+  "file:/etc/ssl/",
+  "file:/etc/sgx_default_qcnl.conf",
+  "file:/lib/libdcap_quoteprov.so",
+]
+remote_attestation = "dcap"
+max_threads = 64
+edmm_enable = false
+## max enclave size
+enclave_size = "2G"
+
+[sys]
+enable_extra_runtime_domain_names_conf = true
+enable_sigterm_injection = true
+
+# possible tweak option, if problems with mio
+# currently mio is compiled with `mio_unsupported_force_waker_pipe`
+# insecure__allow_eventfd = true