From fb6f7e34c7236f22c67bebdeb8dc3bef4ced78d7 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Thu, 14 Dec 2023 16:55:33 -0500 Subject: [PATCH 01/31] Add Quorum Key Resharding Service wip wip get to compile Initial reshard provision and unit test code clean up fix n choose k refactor n choose k Finish tests for reshard add test for boot reshard Add generate reshard input wip wip wip wip Get reshard-renencrypt working Get post share working lint wip Build get reshard output Add new secrets for file key get full thing working e2e refactor wip get qos core to compile Get all of qos core working refactor to not use quorumpubkey wrapper wip Get e2e tests working with new input Add logic for checking that e2e share recombination works finish integration test lint stuff Improve human verifiactions clean up --- .gitignore | 1 + src/Cargo.lock | 2 + src/integration/Cargo.toml | 1 + .../mock/keys/new-share-set/quorum_threshold | 1 + .../mock/keys/new-share-set/reshard-1.pub | 1 + .../mock/keys/new-share-set/reshard-2.pub | 1 + .../mock/keys/new-share-set/reshard-3.pub | 1 + .../mock/keys/new-share-set/reshard-4.pub | 1 + .../new-share-set-secrets/reshard-1.secret | 1 + .../new-share-set-secrets/reshard-2.secret | 1 + .../new-share-set-secrets/reshard-3.secret | 1 + .../new-share-set-secrets/reshard-4.secret | 1 + .../mock/reshard/user1/qkey1/quorum_key.pub | 1 + .../mock/reshard/user2/qkey1/quorum_key.pub | 1 + .../mock/reshard/user3/qkey1/quorum_key.pub | 1 + src/integration/src/lib.rs | 2 + src/integration/tests/boot.rs | 18 +- src/integration/tests/genesis.rs | 6 +- src/integration/tests/key.rs | 12 +- src/integration/tests/reshard.rs | 337 +++++++++ src/qos_client/Cargo.toml | 1 + src/qos_client/RESHARD_GUIDE.md | 122 ++++ src/qos_client/src/cli/mod.rs | 335 ++++++++- src/qos_client/src/cli/services.rs | 550 ++++++++++++++- src/qos_core/src/protocol/error.rs | 23 + src/qos_core/src/protocol/msg.rs | 45 ++ .../src/protocol/services/attestation.rs | 23 +- src/qos_core/src/protocol/services/genesis.rs | 11 +- src/qos_core/src/protocol/services/mod.rs | 1 + .../src/protocol/services/provision.rs | 2 +- src/qos_core/src/protocol/services/reshard.rs | 657 ++++++++++++++++++ src/qos_core/src/protocol/state.rs | 200 +++++- src/qos_crypto/mock/rsa_private.mock.pem | 51 -- src/qos_crypto/mock/rsa_public.mock.pem | 14 - src/qos_crypto/src/lib.rs | 1 + src/qos_crypto/src/n_choose_k.rs | 140 ++++ src/qos_crypto/src/shamir.rs | 5 + src/qos_host/src/lib.rs | 9 +- src/toolchain | 1 + 39 files changed, 2476 insertions(+), 106 deletions(-) create mode 100644 src/integration/mock/keys/new-share-set/quorum_threshold create mode 100644 src/integration/mock/keys/new-share-set/reshard-1.pub create mode 100644 src/integration/mock/keys/new-share-set/reshard-2.pub create mode 100644 src/integration/mock/keys/new-share-set/reshard-3.pub create mode 100644 src/integration/mock/keys/new-share-set/reshard-4.pub create mode 100644 src/integration/mock/new-share-set-secrets/reshard-1.secret create mode 100644 src/integration/mock/new-share-set-secrets/reshard-2.secret create mode 100644 src/integration/mock/new-share-set-secrets/reshard-3.secret create mode 100644 src/integration/mock/new-share-set-secrets/reshard-4.secret create mode 100644 src/integration/mock/reshard/user1/qkey1/quorum_key.pub create mode 100644 src/integration/mock/reshard/user2/qkey1/quorum_key.pub create mode 100644 src/integration/mock/reshard/user3/qkey1/quorum_key.pub create mode 100644 src/integration/tests/reshard.rs create mode 100644 src/qos_client/RESHARD_GUIDE.md create mode 100644 src/qos_core/src/protocol/services/reshard.rs delete mode 100644 src/qos_crypto/mock/rsa_private.mock.pem delete mode 100644 src/qos_crypto/mock/rsa_public.mock.pem create mode 100644 src/qos_crypto/src/n_choose_k.rs create mode 160000 src/toolchain diff --git a/.gitignore b/.gitignore index e589aef0..198ea39d 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ target/ !src/integration/mock/boot-e2e/all-personal-dir/user2-dir/* !src/integration/mock/boot-e2e/all-personal-dir/user3-dir/* !src/integration/mock/boot-e2e/genesis-dir/* +!src/integration/mock/new-share-set-secrets/* src/integration/mock/pivot-build-fingerprints.txt src/integration/pivot_ok2_works src/integration/pivot_ok_works diff --git a/src/Cargo.lock b/src/Cargo.lock index 31ee6da7..1f8137cc 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -872,6 +872,7 @@ dependencies = [ "qos_test_primitives", "rand", "serde", + "serde_json", "tokio", "ureq", ] @@ -1342,6 +1343,7 @@ dependencies = [ "qos_test_primitives", "rand_core", "rpassword", + "serde", "serde_json", "ureq", "x509", diff --git a/src/integration/Cargo.toml b/src/integration/Cargo.toml index 9da9b8fd..3216c07a 100644 --- a/src/integration/Cargo.toml +++ b/src/integration/Cargo.toml @@ -25,3 +25,4 @@ aws-nitro-enclaves-nsm-api = { version = "0.3", default-features = false } rand = "0.8" ureq = { version = "2.9", features = ["json"], default-features = false } serde = { version = "1", features = ["derive"] } +serde_json = "1.0" diff --git a/src/integration/mock/keys/new-share-set/quorum_threshold b/src/integration/mock/keys/new-share-set/quorum_threshold new file mode 100644 index 00000000..e440e5c8 --- /dev/null +++ b/src/integration/mock/keys/new-share-set/quorum_threshold @@ -0,0 +1 @@ +3 \ No newline at end of file diff --git a/src/integration/mock/keys/new-share-set/reshard-1.pub b/src/integration/mock/keys/new-share-set/reshard-1.pub new file mode 100644 index 00000000..9e6fad97 --- /dev/null +++ b/src/integration/mock/keys/new-share-set/reshard-1.pub @@ -0,0 +1 @@ +040ee9045f3718bd1345dccf88693c993626d08448fdeba8ecaf1b867f4d0572d439852ef460963a9e8fab08864a55994c0779216b44a165b4eaced98722ed3778041646e59014eaec046b2636d3943f446282363c26cf995320d5944b8b4d7af0aa588c208c13ded5c86c3e9a31af687c4027d4636173f405503e7b1baeeee7eaa5 \ No newline at end of file diff --git a/src/integration/mock/keys/new-share-set/reshard-2.pub b/src/integration/mock/keys/new-share-set/reshard-2.pub new file mode 100644 index 00000000..42103a6b --- /dev/null +++ b/src/integration/mock/keys/new-share-set/reshard-2.pub @@ -0,0 +1 @@ +04c82672b2f8c4d520c5c7cda207b4a05f433e4db7f0daed9bbde6f54d42814af5aeabec191d2dda32ba4cdc6616aa3fda0a6711affa0d42efbe11144043028622044810d6d24626abfe6c31e884e674c870a2197c9e9cd80786b2fd3a087e2c38cad8376d9b7086901915d261ecb92bde5a757d27bbf1a20904120ff079b8a8ef71 \ No newline at end of file diff --git a/src/integration/mock/keys/new-share-set/reshard-3.pub b/src/integration/mock/keys/new-share-set/reshard-3.pub new file mode 100644 index 00000000..8b39c7af --- /dev/null +++ b/src/integration/mock/keys/new-share-set/reshard-3.pub @@ -0,0 +1 @@ +049872acc56bca90eea07e1e1185e3015be3b7295b4ba484299702489bf4858b1374928b335d3405a16221ec240e80817fbfd783c7052446a31bd1821a9a10ff9c0469361a228e22e7cad34774a50f7cd8f97e7d6542f3903bf9d14647302691ef9195ae2c08ec62dcd0e845bc75e94ef8b9fa45925199a2f7d94d00981d6d2e0d85 \ No newline at end of file diff --git a/src/integration/mock/keys/new-share-set/reshard-4.pub b/src/integration/mock/keys/new-share-set/reshard-4.pub new file mode 100644 index 00000000..f4b9c0a3 --- /dev/null +++ b/src/integration/mock/keys/new-share-set/reshard-4.pub @@ -0,0 +1 @@ +0442993076a3b8345cb58b860477bce9db21bb6caceae8df298860410594ea08d4fc2ffec944fd7623a893b57037e0f20c44ff8eee6eff03110717efb9269181ed04bb495296212027597e2eb93ffbba07f0c41ae3018409b9ad2177e87b53a2729806f52ad6d0f6399ca3d37edddc81a687cd2a0a9f8aab914d76be2930ff8f5bba \ No newline at end of file diff --git a/src/integration/mock/new-share-set-secrets/reshard-1.secret b/src/integration/mock/new-share-set-secrets/reshard-1.secret new file mode 100644 index 00000000..38887d3a --- /dev/null +++ b/src/integration/mock/new-share-set-secrets/reshard-1.secret @@ -0,0 +1 @@ +60dd1d44decfa12be68c49abdb47b02c7d03e63de8f6d61ac7d9c4a59e2bf381 \ No newline at end of file diff --git a/src/integration/mock/new-share-set-secrets/reshard-2.secret b/src/integration/mock/new-share-set-secrets/reshard-2.secret new file mode 100644 index 00000000..7f06d1f4 --- /dev/null +++ b/src/integration/mock/new-share-set-secrets/reshard-2.secret @@ -0,0 +1 @@ +1b28ba3a047709e4bac8f5911bd213dbeca7b7023a702ea5333837a80c2ed170 \ No newline at end of file diff --git a/src/integration/mock/new-share-set-secrets/reshard-3.secret b/src/integration/mock/new-share-set-secrets/reshard-3.secret new file mode 100644 index 00000000..ded6e818 --- /dev/null +++ b/src/integration/mock/new-share-set-secrets/reshard-3.secret @@ -0,0 +1 @@ +f37186894abb1f45ce0eb5b24b5184334d7d85278037d28af11423f50043d83b \ No newline at end of file diff --git a/src/integration/mock/new-share-set-secrets/reshard-4.secret b/src/integration/mock/new-share-set-secrets/reshard-4.secret new file mode 100644 index 00000000..e008df7d --- /dev/null +++ b/src/integration/mock/new-share-set-secrets/reshard-4.secret @@ -0,0 +1 @@ +ccb796f57e4a5f52f2ebd81af50a7c98d7576b5503b5dddc337e67b6217d1fa3 \ No newline at end of file diff --git a/src/integration/mock/reshard/user1/qkey1/quorum_key.pub b/src/integration/mock/reshard/user1/qkey1/quorum_key.pub new file mode 100644 index 00000000..3ecb87a8 --- /dev/null +++ b/src/integration/mock/reshard/user1/qkey1/quorum_key.pub @@ -0,0 +1 @@ +04c9434ba0a681ee7c21e17c7ce4f668360803686b198774c9362dac090f9995eeb68961319370969bd0d657167d9cfce13a7466ec47aba9845fbfc4fe9277866d04043daa777f57c1ebef21ff3eb71e00a681921da56186ac96b5d3b06b645c88c512fe8072d12971ce1f9592ef6bafd98b4982f8cf73cb6e80c8f6424294e54c71 \ No newline at end of file diff --git a/src/integration/mock/reshard/user2/qkey1/quorum_key.pub b/src/integration/mock/reshard/user2/qkey1/quorum_key.pub new file mode 100644 index 00000000..3ecb87a8 --- /dev/null +++ b/src/integration/mock/reshard/user2/qkey1/quorum_key.pub @@ -0,0 +1 @@ +04c9434ba0a681ee7c21e17c7ce4f668360803686b198774c9362dac090f9995eeb68961319370969bd0d657167d9cfce13a7466ec47aba9845fbfc4fe9277866d04043daa777f57c1ebef21ff3eb71e00a681921da56186ac96b5d3b06b645c88c512fe8072d12971ce1f9592ef6bafd98b4982f8cf73cb6e80c8f6424294e54c71 \ No newline at end of file diff --git a/src/integration/mock/reshard/user3/qkey1/quorum_key.pub b/src/integration/mock/reshard/user3/qkey1/quorum_key.pub new file mode 100644 index 00000000..3ecb87a8 --- /dev/null +++ b/src/integration/mock/reshard/user3/qkey1/quorum_key.pub @@ -0,0 +1 @@ +04c9434ba0a681ee7c21e17c7ce4f668360803686b198774c9362dac090f9995eeb68961319370969bd0d657167d9cfce13a7466ec47aba9845fbfc4fe9277866d04043daa777f57c1ebef21ff3eb71e00a681921da56186ac96b5d3b06b645c88c512fe8072d12971ce1f9592ef6bafd98b4982f8cf73cb6e80c8f6424294e54c71 \ No newline at end of file diff --git a/src/integration/src/lib.rs b/src/integration/src/lib.rs index 4250b56f..cc0542d5 100644 --- a/src/integration/src/lib.rs +++ b/src/integration/src/lib.rs @@ -34,6 +34,8 @@ pub const LOCAL_HOST: &str = "127.0.0.1"; pub const PCR3: &str = "78fce75db17cd4e0a3fb8dad3ad128ca5e77edbb2b2c7f75329dccd99aa5f6ef4fc1f1a452e315b9e98f9e312e6921e6"; /// QOS dist directory. pub const QOS_DIST_DIR: &str = "./mock/dist"; +/// Mock pcr3 pre-image. +pub const PCR3_PRE_IMAGE_PATH: &str = "./mock/namespaces/pcr3-preimage.txt"; const MSG: &str = "msg"; diff --git a/src/integration/tests/boot.rs b/src/integration/tests/boot.rs index e5f0bd05..8df54b02 100644 --- a/src/integration/tests/boot.rs +++ b/src/integration/tests/boot.rs @@ -7,7 +7,8 @@ use std::{ use borsh::de::BorshDeserialize; use integration::{ - LOCAL_HOST, PIVOT_OK2_PATH, PIVOT_OK2_SUCCESS_FILE, QOS_DIST_DIR, + LOCAL_HOST, PCR3_PRE_IMAGE_PATH, PIVOT_OK2_PATH, PIVOT_OK2_SUCCESS_FILE, + QOS_DIST_DIR, }; use qos_core::protocol::{ services::{ @@ -51,7 +52,6 @@ async fn standard_boot_e2e() { let namespace = "quit-coding-to-vape"; let personal_dir = |user: &str| format!("{all_personal_dir}/{user}-dir"); - let user1 = "user1"; let user2 = "user2"; let user3 = "user3"; @@ -81,7 +81,7 @@ async fn standard_boot_e2e() { "--qos-release-dir", QOS_DIST_DIR, "--pcr3-preimage-path", - "./mock/namespaces/pcr3-preimage.txt", + PCR3_PRE_IMAGE_PATH, "--manifest-path", &cli_manifest_path, "--pivot-args", @@ -157,7 +157,7 @@ async fn standard_boot_e2e() { "--manifest-approvals-dir", &*boot_dir, "--pcr3-preimage-path", - "./mock/namespaces/pcr3-preimage.txt", + PCR3_PRE_IMAGE_PATH, "--pivot-hash-path", PIVOT_HASH_PATH, "--qos-release-dir", @@ -306,7 +306,7 @@ async fn standard_boot_e2e() { "--host-ip", LOCAL_HOST, "--pcr3-preimage-path", - "./mock/pcr3-preimage.txt", + PCR3_PRE_IMAGE_PATH, "--unsafe-skip-attestation", ]) .spawn() @@ -361,7 +361,7 @@ async fn standard_boot_e2e() { "--manifest-envelope-path", &manifest_envelope_path, "--pcr3-preimage-path", - "./mock/namespaces/pcr3-preimage.txt", + PCR3_PRE_IMAGE_PATH, "--manifest-set-dir", "./mock/keys/manifest-set", "--alias", @@ -400,9 +400,9 @@ async fn standard_boot_e2e() { stdin.write_all("yes\n".as_bytes()).expect("Failed to write to stdin"); assert_eq!( - &stdout.next().unwrap().unwrap(), - "Does this AWS IAM role belong to the intended organization: arn:aws:iam::123456789012:role/Webserver? (yes/no)" - ); + &stdout.next().unwrap().unwrap(), + "Does this AWS IAM role belong to the intended organization: arn:aws:iam::123456789012:role/Webserver? (yes/no)" + ); stdin.write_all("yes\n".as_bytes()).expect("Failed to write to stdin"); assert_eq!( diff --git a/src/integration/tests/genesis.rs b/src/integration/tests/genesis.rs index 10518b16..d08259cf 100644 --- a/src/integration/tests/genesis.rs +++ b/src/integration/tests/genesis.rs @@ -6,7 +6,7 @@ use std::{ }; use borsh::de::BorshDeserialize; -use integration::{LOCAL_HOST, QOS_DIST_DIR}; +use integration::{LOCAL_HOST, PCR3_PRE_IMAGE_PATH, QOS_DIST_DIR}; use qos_core::protocol::services::genesis::GenesisOutput; use qos_crypto::{sha_512, shamir::shares_reconstruct}; use qos_nsm::nitro::unsafe_attestation_doc_from_der; @@ -153,7 +153,7 @@ async fn genesis_e2e() { "--qos-release-dir", QOS_DIST_DIR, "--pcr3-preimage-path", - "./mock/pcr3-preimage.txt", + PCR3_PRE_IMAGE_PATH, "--dr-key-path", DR_KEY_PUBLIC_PATH, "--unsafe-skip-attestation" @@ -225,7 +225,7 @@ async fn genesis_e2e() { "--qos-release-dir", QOS_DIST_DIR, "--pcr3-preimage-path", - "./mock/pcr3-preimage.txt", + PCR3_PRE_IMAGE_PATH, "--unsafe-skip-attestation" ]) .spawn() diff --git a/src/integration/tests/key.rs b/src/integration/tests/key.rs index 3769c83b..0568948f 100644 --- a/src/integration/tests/key.rs +++ b/src/integration/tests/key.rs @@ -1,6 +1,8 @@ use std::{fs, process::Command}; -use integration::{LOCAL_HOST, PIVOT_LOOP_PATH, QOS_DIST_DIR}; +use integration::{ + LOCAL_HOST, PCR3_PRE_IMAGE_PATH, PIVOT_LOOP_PATH, QOS_DIST_DIR, +}; use qos_crypto::sha_256; use qos_p256::{P256Pair, P256Public}; use qos_test_primitives::{ChildWrapper, PathWrapper}; @@ -158,7 +160,7 @@ fn generate_manifest_envelope() { "--restart-policy", "always", "--pcr3-preimage-path", - "./mock/namespaces/pcr3-preimage.txt", + PCR3_PRE_IMAGE_PATH, "--pivot-hash-path", PIVOT_HASH_PATH, "--qos-release-dir", @@ -196,7 +198,7 @@ fn generate_manifest_envelope() { "--manifest-approvals-dir", BOOT_DIR, "--pcr3-preimage-path", - "./mock/namespaces/pcr3-preimage.txt", + PCR3_PRE_IMAGE_PATH, "--pivot-hash-path", PIVOT_HASH_PATH, "--qos-release-dir", @@ -293,7 +295,7 @@ fn boot_old_enclave(old_host_port: u16) -> (ChildWrapper, ChildWrapper) { "--host-ip", LOCAL_HOST, "--pcr3-preimage-path", - "./mock/namespaces/pcr3-preimage.txt", + PCR3_PRE_IMAGE_PATH, "--unsafe-skip-attestation", ]) .spawn() @@ -343,7 +345,7 @@ fn boot_old_enclave(old_host_port: u16) -> (ChildWrapper, ChildWrapper) { "--manifest-envelope-path", MANIFEST_ENVELOPE_PATH, "--pcr3-preimage-path", - "./mock/namespaces/pcr3-preimage.txt", + PCR3_PRE_IMAGE_PATH, "--manifest-set-dir", "./mock/keys/manifest-set", "--alias", diff --git a/src/integration/tests/reshard.rs b/src/integration/tests/reshard.rs new file mode 100644 index 00000000..5845b0fb --- /dev/null +++ b/src/integration/tests/reshard.rs @@ -0,0 +1,337 @@ +use std::{ + collections::HashMap, + fs, + io::{BufRead, BufReader, Write}, + path::Path, + process::{Command, Stdio}, +}; + +use integration::{LOCAL_HOST, PCR3_PRE_IMAGE_PATH, QOS_DIST_DIR}; +use qos_crypto::n_choose_k; +use qos_p256::P256Pair; +use qos_test_primitives::{ChildWrapper, PathWrapper}; + +#[tokio::test] +async fn reshard_e2e() { + let tmp: PathWrapper = "/tmp/reshard_e2e".into(); + drop(fs::create_dir_all(&*tmp)); + let _eph_path: PathWrapper = "/tmp/reshard_e2e/eph.secret".into(); + let usock: PathWrapper = "/tmp/reshard_e2e/usock.sock".into(); + let secret_path: PathWrapper = "/tmp/reshard_e2e/quorum.secret".into(); + let attestation_doc_path: PathWrapper = "/tmp/reshard_e2e/att_doc".into(); + let reshard_input_path: PathWrapper = + "/tmp/reshard_e2e/reshard_input.json".into(); + let reshard_output_path: PathWrapper = + "/tmp/reshard_e2e/reshard_output.json".into(); + let eph_path: PathWrapper = "/tmp/reshard_e2e/ephemeral_key.secret".into(); + + let all_personal_dir = "./mock/boot-e2e/all-personal-dir"; + let personal_dir = |user: &str| format!("{all_personal_dir}/{user}-dir"); + let user1 = "user1"; + let user2 = "user2"; + + let host_port = qos_test_primitives::find_free_port().unwrap(); + + // Start Enclave + let mut _enclave_child_process: ChildWrapper = + Command::new("../target/debug/qos_core") + .args([ + "--usock", + &*usock, + "--quorum-file", + &*secret_path, + "--pivot-file", + "/tmp/reshard_e2e/never_write_pivot_file", + "--ephemeral-file", + &*eph_path, + "--mock", + "--manifest-file", + "/tmp/reshard_e2e/never_write_manifest", + ]) + .spawn() + .unwrap() + .into(); + + // Start Host + let mut _host_child_process: ChildWrapper = + Command::new("../target/debug/qos_host") + .args([ + "--host-port", + &host_port.to_string(), + "--host-ip", + LOCAL_HOST, + "--usock", + &*usock, + ]) + .spawn() + .unwrap() + .into(); + + assert!(Command::new("../target/debug/qos_client") + .args([ + "generate-reshard-input", + "--qos-release-dir", + QOS_DIST_DIR, + "--pcr3-preimage-path", + PCR3_PRE_IMAGE_PATH, + "--quorum-key-path-multiple", + "./mock/namespaces/quit-coding-to-vape/quorum_key.pub", + "--old-share-set-dir", + "./mock/keys/share-set", + "--new-share-set-dir", + "./mock/keys/new-share-set", + "--reshard-input-path", + &*reshard_input_path, + ]) + .spawn() + .unwrap() + .wait() + .unwrap() + .success()); + + qos_test_primitives::wait_until_port_is_bound(host_port); + + assert!(Command::new("../target/debug/qos_client") + .args([ + "boot-reshard", + "--reshard-input-path", + &*reshard_input_path, + "--host-port", + &host_port.to_string(), + "--host-ip", + LOCAL_HOST, + ]) + .spawn() + .unwrap() + .wait() + .unwrap() + .success()); + + assert!(Command::new("../target/debug/qos_client") + .args([ + "get-reshard-attestation-doc", + "--attestation-doc-path", + &*attestation_doc_path, + "--host-port", + &host_port.to_string(), + "--host-ip", + LOCAL_HOST, + ]) + .spawn() + .unwrap() + .wait() + .unwrap() + .success()); + + for user in [&user1, &user2] { + let secret_path = format!("{}/{}.secret", &personal_dir(user), user); + let provision_input_path = + format!("{}/{}.provision_input.json", &*tmp, user); + let quorum_share_dir1 = format!("./mock/reshard/{}/qkey1", user); + + let mut child = Command::new("../target/debug/qos_client") + .args([ + "reshard-re-encrypt-share", + "--secret-path", + &secret_path, + "--quorum-share-dir-multiple", + &*quorum_share_dir1, + "--attestation-doc-path", + &*attestation_doc_path, + "--provision-input-path", + &*provision_input_path, + "--reshard-input-path", + &*reshard_input_path, + "--qos-release-dir", + QOS_DIST_DIR, + "--pcr3-preimage-path", + PCR3_PRE_IMAGE_PATH, + "--new-share-set-dir", + "./mock/keys/new-share-set", + "--old-share-set-dir", + "./mock/keys/share-set", + "--alias", + user, + "--unsafe-skip-attestation", + "--unsafe-eph-path-override", + &*eph_path, + ]) + .stdin(Stdio::piped()) + .stdout(Stdio::piped()) + .spawn() + .unwrap(); + + let mut stdin = child.stdin.take().expect("Failed to open stdin"); + let mut stdout = { + let stdout = child.stdout.as_mut().unwrap(); + let stdout_reader = BufReader::new(stdout); + stdout_reader.lines() + }; + assert_eq!( + &stdout.next().unwrap().unwrap(), + "**WARNING:** Skipping attestation document verification.", + ); + assert_eq!( + &stdout.next().unwrap().unwrap(), + "Does this AWS IAM role belong to the intended organization: arn:aws:iam::123456789012:role/Webserver? (yes/no)", + ); + stdin.write_all("yes\n".as_bytes()).expect("Failed to write to stdin"); + + assert_eq!( + &stdout.next().unwrap().unwrap(), + "Does this new share set look correct? (yes/no)" + ); + assert_eq!( + &stdout.next().unwrap().unwrap(), + "040ee9045f3718bd1345dccf88693c993626d08448fdeba8ecaf1b867f4d0572d439852ef460963a9e8fab08864a55994c0779216b44a165b4eaced98722ed3778041646e59014eaec046b2636d3943f446282363c26cf995320d5944b8b4d7af0aa588c208c13ded5c86c3e9a31af687c4027d4636173f405503e7b1baeeee7eaa5" + ); + assert_eq!( + &stdout.next().unwrap().unwrap(), + "04c82672b2f8c4d520c5c7cda207b4a05f433e4db7f0daed9bbde6f54d42814af5aeabec191d2dda32ba4cdc6616aa3fda0a6711affa0d42efbe11144043028622044810d6d24626abfe6c31e884e674c870a2197c9e9cd80786b2fd3a087e2c38cad8376d9b7086901915d261ecb92bde5a757d27bbf1a20904120ff079b8a8ef71" + ); + assert_eq!( + &stdout.next().unwrap().unwrap(), + "049872acc56bca90eea07e1e1185e3015be3b7295b4ba484299702489bf4858b1374928b335d3405a16221ec240e80817fbfd783c7052446a31bd1821a9a10ff9c0469361a228e22e7cad34774a50f7cd8f97e7d6542f3903bf9d14647302691ef9195ae2c08ec62dcd0e845bc75e94ef8b9fa45925199a2f7d94d00981d6d2e0d85" + ); + assert_eq!( + &stdout.next().unwrap().unwrap(), + "0442993076a3b8345cb58b860477bce9db21bb6caceae8df298860410594ea08d4fc2ffec944fd7623a893b57037e0f20c44ff8eee6eff03110717efb9269181ed04bb495296212027597e2eb93ffbba07f0c41ae3018409b9ad2177e87b53a2729806f52ad6d0f6399ca3d37edddc81a687cd2a0a9f8aab914d76be2930ff8f5bba" + ); + stdin.write_all("yes\n".as_bytes()).expect("Failed to write to stdin"); + assert_eq!( + &stdout.next().unwrap().unwrap(), + "Is this the correct reconstruction threshold for the new share set: 3? (yes/no)" + ); + stdin.write_all("yes\n".as_bytes()).expect("Failed to write to stdin"); + assert_eq!( + &stdout.next().unwrap().unwrap(), + "Are these the correct quorum keys to reshard? (yes/no)" + ); + assert_eq!( + &stdout.next().unwrap().unwrap(), + "04c9434ba0a681ee7c21e17c7ce4f668360803686b198774c9362dac090f9995eeb68961319370969bd0d657167d9cfce13a7466ec47aba9845fbfc4fe9277866d04043daa777f57c1ebef21ff3eb71e00a681921da56186ac96b5d3b06b645c88c512fe8072d12971ce1f9592ef6bafd98b4982f8cf73cb6e80c8f6424294e54c71" + ); + stdin.write_all("yes\n".as_bytes()).expect("Failed to write to stdin"); + + assert_eq!( + stdout.next().unwrap().unwrap(), + format!("Reshard provision input written to: /tmp/reshard_e2e/{}.provision_input.json", user), + ); + assert!(child.wait().unwrap().success()); + + // Post the encrypted shares and approval over reshard input + assert!(Command::new("../target/debug/qos_client") + .args([ + "reshard-post-share", + "--host-port", + &host_port.to_string(), + "--host-ip", + LOCAL_HOST, + "--provision-input-path", + &*provision_input_path, + ]) + .spawn() + .unwrap() + .wait() + .unwrap() + .success()); + } + + assert!(Command::new("../target/debug/qos_client") + .args([ + "get-reshard-output", + "--host-port", + &host_port.to_string(), + "--host-ip", + LOCAL_HOST, + "--reshard-output-path", + &reshard_output_path, + ]) + .spawn() + .unwrap() + .wait() + .unwrap() + .success()); + + let secret_path_fn = |user: &str| { + format!("./mock/new-share-set-secrets/reshard-{}.secret", user) + }; + + type Share = Vec; + type QuorumPubKey = Vec; + let mut seen_shares = HashMap::>::new(); + for user in ["1", "2", "3", "4"] { + let secret_path = secret_path_fn(user); + let share_dir: PathWrapper = + format!("{}/{}/quorum_shares", &*tmp, user).into(); + fs::create_dir_all(&*share_dir).unwrap(); + let pair = P256Pair::from_hex_file(&secret_path).unwrap(); + + assert!(Command::new("../target/debug/qos_client") + .args([ + "verify-reshard-output", + "--reshard-output-path", + &reshard_output_path, + "--secret-path", + &secret_path, + "--share-dir", + &share_dir, + ]) + .spawn() + .unwrap() + .wait() + .unwrap() + .success()); + + for entry in fs::read_dir(&*share_dir).unwrap() { + let path = entry.unwrap().path(); + + if path.is_dir() { + let mut quorum_key = None; + let mut share = None; + for inner_entry in fs::read_dir(&*path).unwrap() { + let inner_path = inner_entry.unwrap().path(); + if inner_path.is_file() { + let split_name = split_file_name(&inner_path); + let buf = fs::read(inner_path).unwrap(); + + if split_name.first().unwrap() == "quorum_key" { + quorum_key = Some(buf) + } else if split_name.last().unwrap() == "share" { + share = Some(buf) + } + } + } + + let share = share.unwrap(); + let decrypted_share = pair.decrypt(&share).unwrap(); + let quorum_key_pub_bytes = + qos_hex::decode_from_vec(quorum_key.unwrap()).unwrap(); + seen_shares + .entry(quorum_key_pub_bytes) + .or_insert_with(Vec::new) + .push(decrypted_share) + } + } + } + + for (pub_key, shares) in seen_shares.iter() { + for combo in n_choose_k::combinations( + shares, 3, /* new share set threshold */ + ) { + let secret: [u8; 32] = + qos_crypto::shamir::shares_reconstruct(&combo) + .try_into() + .unwrap(); + + let quorum_key = P256Pair::from_master_seed(&secret).unwrap(); + assert_eq!(*pub_key, quorum_key.public_key().to_bytes()); + } + } +} + +fn split_file_name(p: &Path) -> Vec { + let file_name = + p.file_name().map(std::ffi::OsStr::to_string_lossy).unwrap(); + file_name.split('.').map(String::from).collect() +} diff --git a/src/qos_client/Cargo.toml b/src/qos_client/Cargo.toml index 57d9bace..62a06fbc 100644 --- a/src/qos_client/Cargo.toml +++ b/src/qos_client/Cargo.toml @@ -20,6 +20,7 @@ rand_core = { version = "0.6", default-features = false } zeroize = { version = "1.6", default-features = false } rpassword = { version = "7", default-features = false } serde_json = { version = "1" } +serde = { version = "1", default-features = false } x509 = { version = "0.2", default-features = false, optional = true } yubikey = { version = "*", features = ["untested"], default-features = false, optional = true } diff --git a/src/qos_client/RESHARD_GUIDE.md b/src/qos_client/RESHARD_GUIDE.md new file mode 100644 index 00000000..95fe5771 --- /dev/null +++ b/src/qos_client/RESHARD_GUIDE.md @@ -0,0 +1,122 @@ +# Quorum Key Resharding Guide + +This guide covers how to reshard a quorum key using the qos_client CLI. + +## Overview + +Flow: + +1) Generate the configuration for how to reshard the given quorum keys. This configuration is called the `ReshardInput`. +2) Boot the enclave in reshard mode using the `ReshardInput`. +3) A threshold of the _old_ share holders query the enclave for an attestation document. +4) A threshold of the _old_ share holders re-encrypt their shares to the enclaves ephemeral key and post those shares in a single message. The data structure used to group a users shares and their corresponding quorum keys is called the `ReshardProvisionInput`. +5) All of the _new_ share holders fetch the `ReshardOutput` to verify they can decrypt their shares. + +## Steps + +### 1 - Generate ReshardInput (Lead) + +Generate the configuration for resharding the quorum keys. + +```sh +qos_client generate-reshard-input \ + --qos-release-dir \ + --pcr3-preimage-path \ + --quorum-key-path-multiple \ + --old-share-set-dir \ + --new-share-set-dir \ + --reshard-input-path +``` + +### 2 - Reshard Boot (Lead) + +Post the reshard boot instruction with the reshard input. + +```sh +qos_client boot-reshard \ + --reshard-input-path \ + --host-port 3001 \ + --host-ip localhost +``` + +### 3 - Get attestation doc (Old Share Holder) + +Get the attestation doc from the enclave. The attestation doc contains a refference to the reshard input and the ephemeral key which shares will be encrypted. + +```sh +qos_client get-reshard-attestation-doc \ + --attestation-doc-path \ + --host-port 3001 \ + --host-ip localhost +``` + +### 4 - Re-encrypt share to ephemeral key (Old Share Holder) + +Use the attestation doc to verify that the enclave is properly setup and running the expected code and re-encrypt relevant shares to the ephemeral key of the enclave. This step should be done on an airgapped machine as unencrypted shares will be exposed to memory. + +For each quorum key being resharded, the user will need a separate directory with just the quorum key and their targeted share. Each directory must be organized like: + +``` +- quorum-share-dir + - quorum_key.pub + - my_alias.share +``` + +Note that the logic looks at the extension of the file to determine if its a share or the quorum key. + +```sh +qos_client reshard-re-encrypt-share \ + --yubikey \ + --quorum-share-dir-multiple \ + --attestation-doc-path \ + --provision-input-path \ + --reshard-input-path \ + --qos-release-dir \ + --pcr3-preimage-path \ + --new-share-set-dir \ + --old-share-set-dir \ + --alias +``` + +### 5 - Post reshard input (Old Share Holder) + +Post the re-encrypted shares from last step in order to reconstruct the quorum keys. + +```sh +qos_client reshard-post-share + --provision-input-path \ + --host-port 3001 \ + --host-ip localhost +``` + +### 6 - Get the new shares (New Share Holder) + +```sh +qos_client get-reshard-output \ + --reshard-output-path \ + --host-port 3001 \ + --host-ip localhost +``` + +### 7 - Verify shares (New Share Holder) + +Verify that we can decrypt our shares. This step should be done on an airgapped machine as the unencrypted share will be exposed to memory. + +The new shares will be written to subdirectories generated within the given share dir. The subdirectories will be named with the first four bytes of the quorum key. Each subdirectory will contain a new share and the quorum key it targets. After running the command against an empty `share-dir` and two sharded quorum, keys, the layout would look like: + +``` +- share-dir + - 04009fd6 + - quorum_key.pub + - my_alias.share + - 041acdf2 + - quorum_key.pub + - my_alias.share +``` + +```sh +qos_client verify-reshard-output \ + --yubikey \ + --reshard-output-path \ + --share-dir +``` diff --git a/src/qos_client/src/cli/mod.rs b/src/qos_client/src/cli/mod.rs index 437fd714..e9e6a24d 100644 --- a/src/qos_client/src/cli/mod.rs +++ b/src/qos_client/src/cli/mod.rs @@ -34,6 +34,8 @@ const QOS_REALEASE_DIR: &str = "qos-release-dir"; const PCR3_PREIMAGE_PATH: &str = "pcr3-preimage-path"; const PIVOT_HASH_PATH: &str = "pivot-hash-path"; const SHARE_SET_DIR: &str = "share-set-dir"; +const NEW_SHARE_SET_DIR: &str = "new-share-set-dir"; +const OLD_SHARE_SET_DIR: &str = "old-share-set-dir"; const MANIFEST_SET_DIR: &str = "manifest-set-dir"; const PATCH_SET_DIR: &str = "patch-set-dir"; const NAMESPACE_DIR: &str = "namespace-dir"; @@ -51,6 +53,7 @@ const APPROVAL_PATH: &str = "approval-path"; const EPH_WRAPPED_SHARE_PATH: &str = "eph-wrapped-share-path"; const ATTESTATION_DOC_PATH: &str = "attestation-doc-path"; const MASTER_SEED_PATH: &str = "master-seed-path"; +const RESHARD_OUTPUT_PATH: &str = "reshard-output-path"; const SHARE: &str = "share"; const OUTPUT_DIR: &str = "output-dir"; const THRESHOLD: &str = "threshold"; @@ -69,6 +72,11 @@ const PLAINTEXT_PATH: &str = "plaintext-path"; const OUTPUT_HEX: &str = "output-hex"; const VALIDATION_TIME_OVERRIDE: &str = "validation-time-override"; const JSON: &str = "json"; +const RESHARD_INPUT_PATH: &str = "reshard-input-path"; +const QUORUM_KEY_PATH_MULTIPLE: &str = "quorum-key-path-multiple"; +const QUORUM_SHARE_DIR_MULTIPLE: &str = "quorum-share-dir-multiple"; +const SHARE_DIR: &str = "share-dir"; +const PROVISION_INPUT_PATH: &str = "provision-input-path"; pub(crate) enum DisplayType { Manifest, @@ -211,6 +219,24 @@ pub enum Command { P256AsymmetricEncrypt, /// Decrypt a payload encrypted to a qos_p256 public key. P256AsymmetricDecrypt, + /// Generate the input for the reshard ceremony. + GenerateReshardInput, + /// Broadcast the boot reshard instruction. + BootReshard, + /// Get the attestation doc with the reshard input hash as the user + /// data and the ephemeral key as the public key. + GetReshardAttestationDoc, + /// Reencrypt a quorum key share to the ephemeral key of an enclave booted + /// for resharding. + ReshardReEncryptShare, + /// Submit an encrypted share along with the associated approval to an + /// enclave booted for resharding. + ReshardPostShare, + /// Fetch the reshard output after the quorum key has been provisioned for + /// resharding. + GetReshardOutput, + /// Verify the reshard output + VerifyReshardOutput, } impl From<&str> for Command { @@ -247,6 +273,13 @@ impl From<&str> for Command { "p256-sign" => Self::P256Sign, "p256-asymmetric-encrypt" => Self::P256AsymmetricEncrypt, "p256-asymmetric-decrypt" => Self::P256AsymmetricDecrypt, + "generate-reshard-input" => Self::GenerateReshardInput, + "boot-reshard" => Self::BootReshard, + "get-reshard-attestation-doc" => Self::GetReshardAttestationDoc, + "reshard-re-encrypt-share" => Self::ReshardReEncryptShare, + "reshard-post-share" => Self::ReshardPostShare, + "get-reshard-output" => Self::GetReshardOutput, + "verify-reshard-output" => Self::VerifyReshardOutput, _ => panic!( "Unrecognized command, try something like `host-health --help`" ), @@ -333,7 +366,39 @@ impl Command { fn share_set_dir_token() -> Token { Token::new( SHARE_SET_DIR, - "Director with public keys for members of the share set.", + "Directory with public keys for members of the share set.", + ) + .takes_value(true) + .required(true) + } + fn old_share_set_dir_token() -> Token { + Token::new( + OLD_SHARE_SET_DIR, + "Directory with public keys for members of the OLD share set.", + ) + .takes_value(true) + .required(true) + } + fn new_share_set_dir_token() -> Token { + Token::new( + NEW_SHARE_SET_DIR, + "Directory with public keys for members of the NEW share set.", + ) + .takes_value(true) + .required(true) + } + fn reshard_input_path_token() -> Token { + Token::new( + RESHARD_INPUT_PATH, + "Path to the file to read/write the reshard input.", + ) + .takes_value(true) + .required(true) + } + fn reshard_output_path_token() -> Token { + Token::new( + RESHARD_OUTPUT_PATH, + "Path to the file to read/write the reshard output.", ) .takes_value(true) .required(true) @@ -362,6 +427,14 @@ impl Command { .takes_value(true) .required(true) } + fn share_dir_token() -> Token { + Token::new( + SHARE_DIR, + "directory to read/write subdirectories that contain quorum key and your associated share.", + ) + .takes_value(true) + .required(true) + } fn alias_token() -> Token { Token::new(ALIAS, "Alias for identifying the key pair") .takes_value(true) @@ -380,7 +453,6 @@ impl Command { .takes_value(true) .required(true) } - fn yubikey_token() -> Token { Token::new(YUBIKEY, "Flag to indicate using a yubikey for signing") .takes_value(false) @@ -556,6 +628,32 @@ impl Command { .required(false) .takes_value(false) } + fn quorum_key_path_multiple_token() -> Token { + Token::new( + QUORUM_KEY_PATH_MULTIPLE, + "use multiple times to specify multiple quorum public key files.", + ) + .required(true) + .takes_value(true) + .allow_multiple(true) + } + fn quorum_share_dir_multiple_token() -> Token { + Token::new( + QUORUM_SHARE_DIR_MULTIPLE, + "path to directory with just your share and the associated quorum key." + ) + .required(true) + .takes_value(true) + .allow_multiple(true) + } + fn provision_input_path_token() -> Token { + Token::new( + PROVISION_INPUT_PATH, + "path to file to read/write ReshardProvisionInput.", + ) + .required(true) + .takes_value(true) + } fn base() -> Parser { Parser::new() @@ -643,6 +741,16 @@ impl Command { .token(Self::pivot_args_token()) } + fn generate_reshard_input() -> Parser { + Parser::new() + .token(Self::qos_release_dir_token()) + .token(Self::quorum_key_path_multiple_token()) + .token(Self::pcr3_preimage_path_token()) + .token(Self::old_share_set_dir_token()) + .token(Self::new_share_set_dir_token()) + .token(Self::reshard_input_path_token()) + } + fn approve_manifest() -> Parser { Parser::new() .token(Self::yubikey_token()) @@ -668,12 +776,20 @@ impl Command { .token(Self::unsafe_skip_attestation_token()) } + fn boot_reshard() -> Parser { + Self::base().token(Self::reshard_input_path_token()) + } + fn get_attestation_doc() -> Parser { Self::base() .token(Self::attestation_doc_path_token()) .token(Self::manifest_envelope_path_token()) } + fn get_reshard_attestation_doc() -> Parser { + Self::base().token(Self::attestation_doc_path_token()) + } + fn proxy_re_encrypt_share() -> Parser { Parser::new() .token(Self::yubikey_token()) @@ -691,6 +807,24 @@ impl Command { .token(Self::unsafe_auto_confirm_token()) } + fn reshard_re_encrypt_share() -> Parser { + Parser::new() + .token(Self::yubikey_token()) + .token(Self::secret_path_token()) + .token(Self::attestation_doc_path_token()) + .token(Self::provision_input_path_token()) + .token(Self::quorum_share_dir_multiple_token()) + .token(Self::reshard_input_path_token()) + .token(Self::qos_release_dir_token()) + .token(Self::pcr3_preimage_path_token()) + .token(Self::new_share_set_dir_token()) + .token(Self::old_share_set_dir_token()) + .token(Self::alias_token()) + .token(Self::unsafe_skip_attestation_token()) + .token(Self::unsafe_eph_path_override_token()) + .token(Self::unsafe_auto_confirm_token()) + } + fn post_share() -> Parser { Self::base() .token(Self::approval_path_token()) @@ -803,6 +937,22 @@ impl Command { .token(Self::master_seed_path_token()) .token(Self::output_hex_token()) } + + fn get_reshard_output() -> Parser { + Self::base().token(Self::reshard_output_path_token()) + } + + fn verify_reshard_output() -> Parser { + Parser::new() + .token(Self::yubikey_token()) + .token(Self::secret_path_token()) + .token(Self::reshard_output_path_token()) + .token(Self::share_dir_token()) + } + + fn reshard_post_share() -> Parser { + Self::base().token(Self::provision_input_path_token()) + } } impl GetParserForCommand for Command { @@ -842,6 +992,15 @@ impl GetParserForCommand for Command { Self::P256Sign => Self::p256_sign(), Self::P256AsymmetricEncrypt => Self::p256_asymmetric_encrypt(), Self::P256AsymmetricDecrypt => Self::p256_asymmetric_decrypt(), + Self::GenerateReshardInput => Self::generate_reshard_input(), + Self::BootReshard => Self::boot_reshard(), + Self::GetReshardAttestationDoc => { + Self::get_reshard_attestation_doc() + } + Self::ReshardReEncryptShare => Self::reshard_re_encrypt_share(), + Self::ReshardPostShare => Self::reshard_post_share(), + Self::GetReshardOutput => Self::get_reshard_output(), + Self::VerifyReshardOutput => Self::verify_reshard_output(), } } } @@ -1049,6 +1208,27 @@ impl ClientOpts { .to_string() } + fn reshard_input_path(&self) -> String { + self.parsed + .single(RESHARD_INPUT_PATH) + .expect("Missing `--reshard-input-path`") + .to_string() + } + + fn new_share_set_dir(&self) -> String { + self.parsed + .single(NEW_SHARE_SET_DIR) + .expect("Missing `--new-share-set-dir`") + .to_string() + } + + fn old_share_set_dir(&self) -> String { + self.parsed + .single(OLD_SHARE_SET_DIR) + .expect("Missing `--old-share-set-dir`") + .to_string() + } + fn output_dir(&self) -> String { self.parsed .single(OUTPUT_DIR) @@ -1144,6 +1324,13 @@ impl ClientOpts { .to_string() } + fn reshard_output_path(&self) -> String { + self.parsed + .single(RESHARD_OUTPUT_PATH) + .expect("Missing `--reshard-output-path`") + .to_string() + } + fn ciphertext_path(&self) -> String { self.parsed .single(CIPHERTEXT_PATH) @@ -1151,6 +1338,20 @@ impl ClientOpts { .to_string() } + fn provision_input_path(&self) -> String { + self.parsed + .single(PROVISION_INPUT_PATH) + .expect("Missing `--provision-input-path`") + .to_string() + } + + fn share_dir(&self) -> String { + self.parsed + .single(SHARE_DIR) + .expect("Missing `--share-dir`") + .to_string() + } + fn yubikey(&self) -> bool { self.parsed.flag(YUBIKEY).unwrap_or(false) } @@ -1174,6 +1375,20 @@ impl ClientOpts { fn json(&self) -> bool { self.parsed.flag(JSON).unwrap_or(false) } + + fn quorum_key_path_multiple(&self) -> Vec { + self.parsed + .multiple(QUORUM_KEY_PATH_MULTIPLE) + .expect("Missing `--quorum-key-path-multiple`") + .to_vec() + } + + fn quorum_share_dir_multiple(&self) -> Vec { + self.parsed + .multiple(QUORUM_SHARE_DIR_MULTIPLE) + .expect("Missing `--quorum-share-dir-multiple`") + .to_vec() + } } #[derive(Clone, PartialEq, Debug)] @@ -1264,6 +1479,25 @@ impl ClientRunner { Command::P256AsymmetricDecrypt => { handlers::p256_asymmetric_decrypt(&self.opts); } + Command::GenerateReshardInput => { + handlers::generate_reshard_input(&self.opts); + } + Command::BootReshard => handlers::boot_reshard(&self.opts), + Command::GetReshardAttestationDoc => { + handlers::get_reshard_attestation_doc(&self.opts); + } + Command::ReshardReEncryptShare => { + handlers::reshard_re_encrypt_share(&self.opts); + } + Command::ReshardPostShare => { + handlers::reshard_post_share(&self.opts); + } + Command::GetReshardOutput => { + handlers::get_reshard_output(&self.opts); + } + Command::VerifyReshardOutput => { + handlers::verify_reshard_output(&self.opts); + } } } } @@ -1283,10 +1517,15 @@ impl CLI { } mod handlers { - use super::services::{ApproveManifestArgs, ProxyReEncryptShareArgs}; + use super::services::{ + ApproveManifestArgs, ProxyReEncryptShareArgs, ReshardReEncryptShareArgs, + }; use crate::{ cli::{ - services::{self, GenerateManifestArgs, PairOrYubi}, + services::{ + self, GenerateManifestArgs, GenerateReshardInputArgs, + PairOrYubi, + }, ClientOpts, ProtocolMsg, }, request, @@ -1495,6 +1734,21 @@ mod handlers { } } + pub(super) fn generate_reshard_input(opts: &ClientOpts) { + if let Err(e) = + services::generate_reshard_input(GenerateReshardInputArgs { + qos_release_dir: opts.qos_release_dir(), + quorum_key_paths: opts.quorum_key_path_multiple(), + pcr3_preimage_path: opts.pcr3_preimage_path(), + new_share_set_dir: opts.new_share_set_dir(), + old_share_set_dir: opts.old_share_set_dir(), + reshard_input_path: opts.reshard_input_path(), + }) { + println!("Error: {e:?}"); + std::process::exit(1); + } + } + pub(super) fn approve_manifest(opts: &ClientOpts) { let pair = get_pair_or_yubi(opts); @@ -1530,6 +1784,16 @@ mod handlers { } } + pub(super) fn boot_reshard(opts: &ClientOpts) { + if let Err(e) = services::boot_reshard( + &opts.path_message(), + opts.reshard_input_path(), + ) { + println!("Error: {e:?}"); + std::process::exit(1); + } + } + pub(super) fn get_attestation_doc(opts: &ClientOpts) { services::get_attestation_doc( &opts.path_message(), @@ -1538,6 +1802,13 @@ mod handlers { ); } + pub(super) fn get_reshard_attestation_doc(opts: &ClientOpts) { + services::get_reshard_attestation_doc( + &opts.path_message(), + opts.attestation_doc_path(), + ); + } + pub(super) fn proxy_re_encrypt_share(opts: &ClientOpts) { let pair = get_pair_or_yubi(opts); @@ -1561,6 +1832,32 @@ mod handlers { } } + pub(super) fn reshard_re_encrypt_share(opts: &ClientOpts) { + let pair = get_pair_or_yubi(opts); + + if let Err(e) = + services::reshard_re_encrypt_share(ReshardReEncryptShareArgs { + pair, + quorum_share_dirs: opts.quorum_share_dir_multiple(), + attestation_doc_path: opts.attestation_doc_path(), + provision_input_path: opts.provision_input_path(), + + reshard_input_path: opts.reshard_input_path(), + qos_release_dir: opts.qos_release_dir(), + pcr3_preimage_path: opts.pcr3_preimage_path(), + new_share_set_dir: opts.new_share_set_dir(), + old_share_set_dir: opts.old_share_set_dir(), + + alias: opts.alias(), + unsafe_skip_attestation: opts.unsafe_skip_attestation(), + unsafe_eph_path_override: opts.unsafe_eph_path_override(), + unsafe_auto_confirm: opts.unsafe_auto_confirm(), + }) { + eprintln!("Error: {e:?}"); + std::process::exit(1); + } + } + pub(super) fn post_share(opts: &ClientOpts) { if let Err(e) = services::post_share( &opts.path_message(), @@ -1572,6 +1869,36 @@ mod handlers { } } + pub(super) fn reshard_post_share(opts: &ClientOpts) { + if let Err(e) = services::reshard_post_share( + &opts.path_message(), + opts.provision_input_path(), + ) { + eprintln!("Error: {e:?}"); + std::process::exit(1); + } + } + + pub(super) fn get_reshard_output(opts: &ClientOpts) { + services::get_reshard_output( + &opts.path_message(), + &opts.reshard_output_path(), + ); + } + + pub(super) fn verify_reshard_output(opts: &ClientOpts) { + let pair = get_pair_or_yubi(opts); + + if let Err(e) = services::verify_reshard_output( + opts.reshard_output_path(), + pair, + &opts.share_dir(), + ) { + eprintln!("Error: {e:?}"); + std::process::exit(1); + } + } + pub(super) fn display(opts: &ClientOpts) { if let Err(e) = services::display( &opts.display_type(), diff --git a/src/qos_client/src/cli/services.rs b/src/qos_client/src/cli/services.rs index 95f7525f..0ecfa19f 100644 --- a/src/qos_client/src/cli/services.rs +++ b/src/qos_client/src/cli/services.rs @@ -1,4 +1,5 @@ use std::{ + collections::HashSet, fs, fs::File, io, @@ -20,6 +21,10 @@ use qos_core::protocol::{ }, genesis::{GenesisOutput, GenesisSet}, key::EncryptedQuorumKey, + reshard::{ + ReshardInput, ReshardOutput, ReshardProvisionInput, + ReshardProvisionShare, + }, }, QosHash, }; @@ -47,6 +52,7 @@ const QUORUM_THRESHOLD_FILE: &str = "quorum_threshold"; const DR_WRAPPED_QUORUM_KEY: &str = "dr_wrapped_quorum_key"; const PCRS_PATH: &str = "aws-x86_64.pcrs"; const GENESIS_DR_ARTIFACTS: &str = "genesis_dr_artifacts"; +const SHARE_EXT: &str = "share"; const DANGEROUS_DEV_BOOT_MEMBER: &str = "DANGEROUS_DEV_BOOT_MEMBER"; const DANGEROUS_DEV_BOOT_NAMESPACE: &str = @@ -86,7 +92,7 @@ pub enum Error { #[cfg(feature = "smartcard")] PinEntryError(std::io::Error), /// Failed to read share - ReadShare, + ReadShare(String), /// Error while try to read the quorum public key. FailedToReadQuorumPublicKey(qos_p256::P256Error), /// Error trying to the read a file that is supposed to have a manifest. @@ -109,10 +115,16 @@ pub enum Error { /// Failed to read file that was supposed to contain Ephemeral Key wrapped /// share. FailedToReadEphWrappedShare(std::io::Error), + /// Failed to read [Self::path]. FailedToRead { path: String, error: String, }, + /// Failed to write to [Self::path]. + FailedToWrite { + path: String, + error: String, + }, /// Failed to decode some hex CouldNotDecodeHex(qos_hex::HexError), /// Failed to deserialize something from borsh. @@ -139,6 +151,19 @@ pub enum Error { /// Given quorum key seed does not match the hash of the expected quorum /// key seed. SecretDoesNotMatch, + FileDidNotHaveValidReshardInput(String), + /// Could not read the file with the reshard output + FailedToReadReshardOutput(std::io::Error), + /// Could not serialize the reshard output in the file + FileDidNotHaveReshardOutput(String), + /// Could not find key in new genesis member outputs + KeyNotInNewShareSet, + /// Failed to write to the file specified for the encrypted share. + FailedToWriteEncryptedShare(std::io::Error), + /// Expected only 1 share in the directory. + ExpectedExactlyOneShare, + /// Expected only 1 quorum key in the directory. + ExpectedExactlyOneQuorumKey, } impl From for Error { @@ -754,6 +779,59 @@ pub(crate) fn generate_manifest>( Ok(()) } +pub struct GenerateReshardInputArgs { + pub qos_release_dir: String, + pub quorum_key_paths: Vec, + pub new_share_set_dir: String, + pub old_share_set_dir: String, + pub reshard_input_path: String, + pub pcr3_preimage_path: String, +} + +pub fn generate_reshard_input( + GenerateReshardInputArgs { + qos_release_dir, + quorum_key_paths, + new_share_set_dir, + old_share_set_dir, + reshard_input_path, + pcr3_preimage_path, + }: GenerateReshardInputArgs, +) -> Result<(), Error> { + let nitro_config = + extract_nitro_config(qos_release_dir, pcr3_preimage_path); + + let quorum_keys = quorum_key_paths + .iter() + .map(|path| { + P256Public::from_hex_file(&path) + .map_err(|_| Error::FailedToRead { + path: path.clone(), + error: "p256 error".to_string(), + }) + .map(|pair| pair.to_bytes()) + }) + .collect::, _>>()?; + + let old_share_set = get_share_set(old_share_set_dir); + let new_share_set = get_share_set(new_share_set_dir); + + let reshard_input = ReshardInput { + quorum_keys, + new_share_set, + old_share_set, + enclave: nitro_config, + }; + + write_json_with_msg( + reshard_input_path.as_ref(), + &reshard_input, + "ReshardInput", + ); + + Ok(()) +} + fn extract_nitro_config>( qos_release_dir_path: P, pcr3_preimage_path: P, @@ -1153,6 +1231,27 @@ pub(crate) fn boot_standard>( Ok(()) } +pub(crate) fn boot_reshard( + uri: &str, + reshard_input_path: String, +) -> Result<(), Error> { + // Create manifest envelope + let reshard_input = read_reshard_input(reshard_input_path)?; + + let req = ProtocolMsg::BootReshardRequest { reshard_input }; + + // Broadcast boot reshard instruction and make sure it has the attestation + // doc as the response + let _cose_sign1 = match request::post(uri, &req).unwrap() { + ProtocolMsg::BootReshardResponse { + nsm_response: NsmResponse::Attestation { document }, + } => document, + r => panic!("Unexpected response: {r:?}"), + }; + + Ok(()) +} + pub(crate) fn get_attestation_doc>( uri: &str, attestation_doc_path: P, @@ -1185,6 +1284,26 @@ pub(crate) fn get_attestation_doc>( ); } +pub(crate) fn get_reshard_attestation_doc>( + uri: &str, + attestation_doc_path: P, +) { + let (cose_sign1, _) = + match request::post(uri, &ProtocolMsg::ReshardAttestationDocRequest) { + Ok(ProtocolMsg::ReshardAttestationDocResponse { + nsm_response: NsmResponse::Attestation { document }, + reshard_input, + }) => (document, reshard_input), + r => panic!("Unexpected response: {r:?}"), + }; + + write_with_msg( + attestation_doc_path.as_ref(), + &cose_sign1, + "COSE Sign1 Attestation Doc", + ); +} + pub(crate) struct ProxyReEncryptShareArgs> { pub pair: PairOrYubi, pub share_path: P, @@ -1224,10 +1343,8 @@ pub(crate) fn proxy_re_encrypt_share>( let manifest_envelope = read_manifest_envelope(&manifest_envelope_path)?; let attestation_doc = read_attestation_doc(&attestation_doc_path, unsafe_skip_attestation)?; - let encrypted_share = std::fs::read(share_path).map_err(|e| { - eprintln!("{e:?}"); - Error::ReadShare - })?; + let encrypted_share = std::fs::read(share_path) + .map_err(|e| Error::ReadShare(e.to_string()))?; let pcr3_preimage = find_pcr3(&pcr3_preimage_path); @@ -1402,6 +1519,227 @@ where true } +pub struct ReshardReEncryptShareArgs { + pub pair: PairOrYubi, + pub quorum_share_dirs: Vec, + pub attestation_doc_path: String, + pub provision_input_path: String, + pub reshard_input_path: String, + pub qos_release_dir: String, + pub pcr3_preimage_path: String, + pub new_share_set_dir: String, + pub old_share_set_dir: String, + pub alias: String, + pub unsafe_skip_attestation: bool, + pub unsafe_eph_path_override: Option, + pub unsafe_auto_confirm: bool, +} + +pub(crate) fn reshard_re_encrypt_share( + ReshardReEncryptShareArgs { + mut pair, + quorum_share_dirs, + attestation_doc_path, + provision_input_path, + reshard_input_path, + qos_release_dir, + pcr3_preimage_path, + new_share_set_dir, + old_share_set_dir, + alias, + unsafe_skip_attestation, + unsafe_eph_path_override, + unsafe_auto_confirm, + }: ReshardReEncryptShareArgs, +) -> Result<(), Error> { + let reshard_input = read_reshard_input(reshard_input_path)?; + let attestation_doc = + read_attestation_doc(&attestation_doc_path, unsafe_skip_attestation)?; + let mut new_share_set = get_share_set(&new_share_set_dir); + let member = QuorumMember { pub_key: pair.public_key_bytes()?, alias }; + let pcr3_preimage = find_pcr3(&pcr3_preimage_path); + + // Verify the attestation doc matches up with the pcrs in the manifest + if unsafe_skip_attestation { + println!("**WARNING:** Skipping attestation document verification."); + } else { + verify_attestation_doc_against_user_input( + &attestation_doc, + &reshard_input.qos_hash(), + &reshard_input.enclave.pcr0, + &reshard_input.enclave.pcr1, + &reshard_input.enclave.pcr2, + &extract_pcr3(pcr3_preimage_path.clone()), + )?; + } + + // Pull out the ephemeral key or use the override + let eph_pub: P256Public = if let Some(eph_path) = unsafe_eph_path_override { + P256Pair::from_hex_file(eph_path) + .expect("Could not read ephemeral key override") + .public_key() + } else { + P256Public::from_bytes( + &attestation_doc + .public_key + .expect("No ephemeral key in the attestation doc"), + ) + .expect("Ephemeral key not valid public key") + }; + + // Programmatic verification + reshard_re_encrypt_share_programmatic_verification( + &member, + old_share_set_dir, + &new_share_set, + &reshard_input, + qos_release_dir, + pcr3_preimage_path, + ); + + // Human verification + if !unsafe_auto_confirm { + reshard_re_encrypt_share_human_verification( + &pcr3_preimage, + &mut new_share_set, + &reshard_input, + ); + } + + let provision_shares = quorum_share_dirs + .iter() + .map(find_quorum_key_and_share) + .collect::, _>>()?; + let found_quorum_keys: HashSet<_> = + provision_shares.iter().map(|p| p.pub_key.clone()).collect(); + if found_quorum_keys != reshard_input.quorum_keys() { + let keys = found_quorum_keys + .symmetric_difference(&reshard_input.quorum_keys()) + .map(|b| qos_hex::encode(b)) + .collect::>() + .join(", "); + panic!("quorum keys given did not match keys in reshard input: difference: {}", keys); + } + + let shares = provision_shares + .into_iter() + .map(|p| { + let plaintext_share = &pair + .decrypt(&p.share) + .expect("Failed to decrypt share with personal key."); + + ReshardProvisionShare { + share: eph_pub + .encrypt(plaintext_share) + .expect("Envelope encryption error"), + pub_key: p.pub_key, + } + }) + .collect(); + + let approval = Approval { + signature: pair + .sign(&reshard_input.qos_hash()) + .expect("Failed to sign"), + member, + }; + + let provision_input = ReshardProvisionInput { approval, shares }; + + write_json_with_msg( + provision_input_path.as_ref(), + &provision_input, + "Reshard provision input", + ); + + Ok(()) +} + +fn reshard_re_encrypt_share_programmatic_verification( + member: &QuorumMember, + old_share_set_dir: String, + new_share_set: &ShareSet, + reshard_input: &ReshardInput, + qos_release_dir: String, + pcr3_preimage_path: String, +) { + let old_share_set = get_share_set(old_share_set_dir); + let nitro_config: NitroConfig = + extract_nitro_config(qos_release_dir, pcr3_preimage_path); + + // Verify that member is part of new share set + assert!( + reshard_input.old_share_set.members.contains(member), + "Member does not belong to old share set." + ); + // Verify old share set matches reshard input + assert_eq!(old_share_set, reshard_input.old_share_set, "Specified old share set does not match old share set in reshard input."); + // Verify new share set matches reshard input + assert_eq!(*new_share_set, reshard_input.new_share_set, "Specified new share set does not match new share set in reshard input."); + // Verify that pcrs 0, 1, 2 & 3 match reshard input + assert_eq!( + nitro_config, reshard_input.enclave, + "Enclave configuration in reshard input does not match given qos dist (PCRs, qos version, etc)" + ); +} + +fn reshard_re_encrypt_share_human_verification( + pcr3_preimage: &str, + new_share_set: &mut ShareSet, + reshard_input: &ReshardInput, +) { + let stdin = io::stdin(); + let stdin_locked = stdin.lock(); + let mut prompter = Prompter { reader: stdin_locked, writer: io::stdout() }; + { + // Last chance to verify that this enclave belongs to turnkey and + // not an attacker + let prompt = format!( + "Does this AWS IAM role belong to the intended organization: {pcr3_preimage}? (yes/no)" + ); + assert!(prompter.prompt_is_yes(&prompt), "You indicated that this IAM role does not belong to your organization."); + } + { + new_share_set.members.sort(); + let public_keys = new_share_set + .members + .iter() + .map(|m| qos_hex::encode(&m.pub_key)) + .collect::>() + .join("\n"); + let prompt = format!( + "Does this new share set look correct? (yes/no)\n{public_keys}" + ); + assert!( + prompter.prompt_is_yes(&prompt), + "You indicated that this is not the correct share set." + ); + } + { + let prompt = format!( + "Is this the correct reconstruction threshold for the new share set: {}? (yes/no)", + new_share_set.threshold + ); + assert!(prompter.prompt_is_yes(&prompt), "You indicated that this is not the correct reconstruction threshold."); + } + { + let quorum_public_keys = reshard_input + .quorum_keys + .iter() + .map(|p| qos_hex::encode(p)) + .collect::>() + .join("\n"); + + let prompt = format!( + "Are these the correct quorum keys to reshard? (yes/no)\n{quorum_public_keys}" + ); + assert!( + prompter.prompt_is_yes(&prompt), + "You indicated that this was not the correct set of quorum keys to reshard" + ); + } +} + pub(crate) fn post_share>( uri: &str, eph_wrapped_share_path: P, @@ -1427,6 +1765,112 @@ pub(crate) fn post_share>( Ok(()) } +pub(crate) fn reshard_post_share( + uri: &str, + provision_input_path: String, +) -> Result<(), Error> { + // Get the ephemeral key wrapped share + let input: ReshardProvisionInput = { + let buf = fs::read(&provision_input_path).map_err(|e| { + Error::FailedToRead { + path: provision_input_path, + error: e.to_string(), + } + })?; + + serde_json::from_slice(&buf) + .expect("failed to deserialize provision input") + }; + + let req = ProtocolMsg::ReshardProvisionRequest { input }; + let is_reconstructed = match request::post(uri, &req).unwrap() { + ProtocolMsg::ReshardProvisionResponse { reconstructed } => { + reconstructed + } + r => panic!("Unexpected response: {r:?}"), + }; + + if is_reconstructed { + println!("The quorum key has been reconstructed."); + } else { + println!("The quorum key has *not* been reconstructed."); + }; + + Ok(()) +} + +pub(crate) fn get_reshard_output(uri: &str, reshard_output_path: &str) { + let req = ProtocolMsg::ReshardOutputRequest; + let reshard_output = match request::post(uri, &req).unwrap() { + ProtocolMsg::ReshardOutputResponse { reshard_output } => reshard_output, + r => panic!("Unexpected response: {r:?}"), + }; + + write_json_with_msg( + reshard_output_path.as_ref(), + &reshard_output, + "ReshardOutput", + ); +} + +pub(crate) fn verify_reshard_output( + reshard_output_path: String, + mut pair: PairOrYubi, + share_dir: &str, +) -> Result<(), Error> { + let reshard_output = read_reshard_output(reshard_output_path)?; + let pub_key = pair.public_key_bytes()?; + + let mut alias: Option = None; + let member_shares = reshard_output + .outputs + .into_iter() + .map(|(quorum_pub, member_outputs)| { + let member_output = member_outputs + .iter() + .find(|m| m.share_set_member.pub_key == pub_key) + .ok_or(Error::KeyNotInNewShareSet)?; + let decrypted_share_hash = sha_512( + &pair.decrypt(&member_output.encrypted_quorum_key_share)?, + ); + assert_eq!( + member_output.share_hash, decrypted_share_hash, + "decrypted share did not match expected hash" + ); + alias = Some(member_output.share_set_member.alias.clone()); + + Ok((quorum_pub, member_output.encrypted_quorum_key_share.clone())) + }) + .collect::, Vec)>, Error>>()?; + + let alias = alias.expect( + "we exit early above if the person is not a member of the share set", + ); + for (quorum_key, share) in member_shares { + let dir_name = qos_hex::encode(&quorum_key[0..4]); + let dir_path = Path::new(&share_dir).join(dir_name); + fs::create_dir(&dir_path) + .map_err(|e: io::Error| { + panic!("failed to create dir {:?}: {}", dir_path, e) + }) + .unwrap(); + let quorum_key_path = dir_path.clone().join("quorum_key.pub"); + let share_path = dir_path.join(format!("{}.share", alias)); + + fs::write(&quorum_key_path, qos_hex::encode(&quorum_key)).map_err( + |e| Error::FailedToWrite { + path: quorum_key_path.into_os_string().into_string().unwrap(), + error: e.to_string(), + }, + )?; + fs::write(&share_path, share).map_err(|e| Error::FailedToWrite { + path: share_path.into_os_string().into_string().unwrap(), + error: e.to_string(), + })?; + } + Ok(()) +} + #[cfg(feature = "smartcard")] pub(crate) fn yubikey_sign(hex_payload: &str) -> Result<(), Error> { let bytes = qos_hex::decode(hex_payload)?; @@ -1533,7 +1977,8 @@ pub(crate) fn display>( file_path: P, json: bool, ) -> Result<(), Error> { - let bytes = fs::read(file_path).map_err(|_| Error::ReadShare)?; + let bytes = + fs::read(file_path).map_err(|e| Error::ReadShare(e.to_string()))?; match *display_type { DisplayType::Manifest => { let decoded = Manifest::try_from_slice(&bytes)?; @@ -1885,6 +2330,58 @@ fn get_genesis_set>(dir: P) -> GenesisSet { GenesisSet { members, threshold: find_threshold(dir) } } +fn find_quorum_key_and_share>( + dir: P, +) -> Result { + let dir_contents = find_file_paths(&dir); + let mut share = dir_contents + .iter() + .filter(|path| { + let file_name = split_file_name(path); + file_name.last().map_or(false, |s| s.as_str() == SHARE_EXT) + }) + .map(|p| { + fs::read(p).map_err(|e| Error::FailedToRead { + path: p.to_string_lossy().to_string(), + error: e.to_string(), + }) + }) + .collect::, _>>()?; + + if share.len() != 1 { + return Err(Error::ExpectedExactlyOneShare); + } + + let mut quorum_key: Vec<_> = dir_contents + .iter() + .filter_map(|path| { + let file_name = split_file_name(path); + if file_name.last().map_or(true, |s| s.as_str() != PUB_EXT) { + return None; + }; + if file_name.first().map_or(true, |s| s.as_str() != "quorum_key") { + return None; + }; + + let quorum_key = P256Public::from_hex_file(path) + .map_err(|e| { + panic!("Could not read hex from quorum_key.pub: {path:?}: {e:?}") + }) + .unwrap() + .to_bytes(); + Some(quorum_key) + }) + .collect(); + if quorum_key.len() != 1 { + return Err(Error::ExpectedExactlyOneQuorumKey); + } + + Ok(ReshardProvisionShare { + pub_key: mem::take(&mut quorum_key[0]), + share: mem::take(&mut share[0]), + }) +} + fn find_approvals>( boot_dir: P, manifest: &Manifest, @@ -1957,6 +2454,28 @@ fn read_manifest_envelope>( .map_err(|_| Error::FileDidNotHaveValidManifestEnvelope) } +fn read_reshard_input(file: String) -> Result { + let buf = fs::read(&file).map_err(|e| Error::FailedToRead { + path: file, + error: e.to_string(), + })?; + + let reshard_input: ReshardInput = serde_json::from_slice(&buf) + .map_err(|e| Error::FileDidNotHaveValidReshardInput(e.to_string()))?; + + Ok(reshard_input) +} + +fn read_reshard_output(file: String) -> Result { + let buf = fs::read(&file).map_err(|e| Error::FailedToRead { + path: file, + error: e.to_string(), + })?; + + serde_json::from_slice(&buf) + .map_err(|e| Error::FileDidNotHaveReshardOutput(e.to_string())) +} + fn read_attestation_approval>( path: P, ) -> Result { @@ -2098,6 +2617,25 @@ fn write_with_msg(path: &Path, buf: &[u8], item_name: &str) { println!("{item_name} written to: {path_str}"); } +fn write_json_with_msg( + path: &Path, + item: &T, + item_name: &str, +) { + let path_str = path.as_os_str().to_string_lossy(); + + let buf = serde_json::to_vec_pretty(item).unwrap_or_else(|_| { + panic!( + "Failed serializing to json when writing {} to file", + path_str.clone() + ) + }); + fs::write(path, buf).unwrap_or_else(|_| { + panic!("Failed writing {} to file", path_str.clone()) + }); + println!("{item_name} written to: {path_str}"); +} + struct Prompter { reader: R, writer: W, diff --git a/src/qos_core/src/protocol/error.rs b/src/qos_core/src/protocol/error.rs index 74e5f4bb..7eda6cd8 100644 --- a/src/qos_core/src/protocol/error.rs +++ b/src/qos_core/src/protocol/error.rs @@ -141,6 +141,29 @@ pub enum ProtocolError { /// The new manifest was different from the old manifest when we expected /// them to be the same because they have the same nonce DifferentManifest, + /// Expected to have [crate::protocol::services::reshard::ReshardInput] in enclave state, but it was not found. + MissingReshardInput, + /// Expected to have [crate::protocol::services::reshard::ReshardOutput] in enclave state, but it was not + /// found. + MissingReshardOutput, + /// The same member was in the share set multiple times. + DuplicateNewShareSetMember, + /// The share set member posted more or less shares then the number of + /// quorum keys targeted for reconstruction. + ShareCountDoesNotMatchExpectedQuorumKeyCount, + /// Could not decrypt the share with the ephemeral key. + ShareDecryptionFailed, + /// Internal error indicating that the count of shares in each secret + /// builder do not match. We expect each secret builder to have the same + /// count of shares because we enforce that the count of shares post by + /// each member equals the count of quorum keys specified for + /// reconstruction in the reshard input. + InternalDiffCountsForQuorumKeyShares, + /// The same quorum key was specified multiple times in the reshard input. + DuplicateQuorumKeys, + /// Internal error indicating that reshard provisioner was not initialized + /// on boot. + MissingReshardProvisioner, } impl From for ProtocolError { diff --git a/src/qos_core/src/protocol/msg.rs b/src/qos_core/src/protocol/msg.rs index 718164c1..1df262ff 100644 --- a/src/qos_core/src/protocol/msg.rs +++ b/src/qos_core/src/protocol/msg.rs @@ -2,10 +2,12 @@ use qos_nsm::types::NsmResponse; +use super::services::reshard::ReshardProvisionInput; use crate::protocol::{ services::{ boot::{Approval, ManifestEnvelope}, genesis::{GenesisOutput, GenesisSet}, + reshard::{ReshardInput, ReshardOutput}, }, ProtocolError, }; @@ -138,6 +140,49 @@ pub enum ProtocolMsg { /// if the manifest envelope does not exist. manifest_envelope: Box>, }, + + /// Reshard a quorum key to the `new_share_set` in the [`ReshardInput`] + BootReshardRequest { + /// The parameters for resharding + reshard_input: ReshardInput, + }, + /// Response to [`Self::BootReshardRequest`]. + BootReshardResponse { + /// Should be `[NsmResponse::Attestation`]. `user_data` is the the + /// reshard_input + nsm_response: NsmResponse, + }, + + /// Request an attestation doc with the `ReshardInput` as the user data/ + ReshardAttestationDocRequest, + /// Response to [`Self::ReshardAttestationDocRequest`] + ReshardAttestationDocResponse { + /// Should be `[NsmResponse::Attestation`]. `user_data` is the the + /// reshard_input + nsm_response: NsmResponse, + /// The reshard parameters this enclave is setup for. + reshard_input: ReshardInput, + }, + + /// Post a quorum key shard so it can be provisioned and resharded. + ReshardProvisionRequest { + /// Approval and shares for each quorum key + input: ReshardProvisionInput, + }, + /// Response to a `Self::ReshardProvisionRequest` + ReshardProvisionResponse { + /// If the Quorum key was reconstructed. False indicates still waiting + /// for the Kth share. + reconstructed: bool, + }, + + /// Request the reshard service's output. + ReshardOutputRequest, + /// Response to [Self::ReshardOutputRequest]. + ReshardOutputResponse { + /// The output of the reshard services. + reshard_output: ReshardOutput, + }, } #[cfg(test)] diff --git a/src/qos_core/src/protocol/services/attestation.rs b/src/qos_core/src/protocol/services/attestation.rs index 3f206de6..d3a402d4 100644 --- a/src/qos_core/src/protocol/services/attestation.rs +++ b/src/qos_core/src/protocol/services/attestation.rs @@ -5,6 +5,7 @@ use qos_nsm::{ use crate::protocol::{ProtocolError, ProtocolState, QosHash}; +/// manifest hash in user data pub(in crate::protocol) fn live_attestation_doc( state: &mut ProtocolState, ) -> Result { @@ -20,13 +21,31 @@ pub(in crate::protocol) fn live_attestation_doc( )) } +/// reshard input hash in user data +pub(in crate::protocol) fn reshard_attestation_doc( + state: &mut ProtocolState, +) -> Result { + let ephemeral_public_key = + state.handles.get_ephemeral_key()?.public_key().to_bytes(); + let reshard_input = state + .reshard_input + .clone() + .ok_or(ProtocolError::MissingReshardInput)?; + + Ok(get_post_boot_attestation_doc( + &*state.attestor, + ephemeral_public_key, + reshard_input.qos_hash().to_vec(), + )) +} + pub(super) fn get_post_boot_attestation_doc( attestor: &dyn NsmProvider, ephemeral_public_key: Vec, - manifest_hash: Vec, + user_data: Vec, ) -> NsmResponse { let request = NsmRequest::Attestation { - user_data: Some(manifest_hash), + user_data: Some(user_data), nonce: None, public_key: Some(ephemeral_public_key), }; diff --git a/src/qos_core/src/protocol/services/genesis.rs b/src/qos_core/src/protocol/services/genesis.rs index 9a86634f..f3afd115 100644 --- a/src/qos_core/src/protocol/services/genesis.rs +++ b/src/qos_core/src/protocol/services/genesis.rs @@ -51,15 +51,24 @@ pub struct RecoveredPermutation(Vec); /// Genesis output per Setup Member. #[derive( - PartialEq, Eq, Clone, borsh::BorshSerialize, borsh::BorshDeserialize, + PartialEq, + Eq, + Clone, + borsh::BorshSerialize, + borsh::BorshDeserialize, + serde::Serialize, + serde::Deserialize, )] +#[serde(rename_all = "camelCase")] pub struct GenesisMemberOutput { /// The Quorum Member whom's Setup Key was used. pub share_set_member: QuorumMember, /// Quorum Key Share encrypted to the `setup_member`'s Personal Key. + #[serde(with = "qos_hex::serde")] pub encrypted_quorum_key_share: Vec, /// Sha512 hash of the plaintext quorum key share. Used by the share set /// member to verify they correctly decrypted the share. + #[serde(with = "qos_hex::serde")] pub share_hash: [u8; 64], } diff --git a/src/qos_core/src/protocol/services/mod.rs b/src/qos_core/src/protocol/services/mod.rs index 5a18d19b..f6cbf34d 100644 --- a/src/qos_core/src/protocol/services/mod.rs +++ b/src/qos_core/src/protocol/services/mod.rs @@ -5,3 +5,4 @@ pub mod boot; pub mod genesis; pub mod key; pub mod provision; +pub mod reshard; diff --git a/src/qos_core/src/protocol/services/provision.rs b/src/qos_core/src/protocol/services/provision.rs index 799af628..e1efc4da 100644 --- a/src/qos_core/src/protocol/services/provision.rs +++ b/src/qos_core/src/protocol/services/provision.rs @@ -47,7 +47,7 @@ impl SecretBuilder { self.shares.len() } - fn clear(&mut self) { + pub(crate) fn clear(&mut self) { self.shares = vec![]; } } diff --git a/src/qos_core/src/protocol/services/reshard.rs b/src/qos_core/src/protocol/services/reshard.rs new file mode 100644 index 00000000..fce811ca --- /dev/null +++ b/src/qos_core/src/protocol/services/reshard.rs @@ -0,0 +1,657 @@ +//! Quorum Key Resharding logic and types. + +use core::iter::zip; +use std::collections::{HashMap, HashSet}; + +use qos_crypto::sha_512; +use qos_nsm::types::NsmResponse; +use qos_p256::{P256Pair, P256Public}; + +use super::provision::SecretBuilder; +use crate::protocol::{ + services::{ + attestation, + boot::{Approval, NitroConfig, ShareSet}, + genesis::GenesisMemberOutput, + }, + ProtocolError, ProtocolState, QosHash, +}; + +/// A share and the quorum key it is for. +#[derive( + Debug, + PartialEq, + Eq, + Clone, + PartialOrd, + Hash, + Ord, + borsh::BorshSerialize, + borsh::BorshDeserialize, + serde::Serialize, + serde::Deserialize, +)] +pub struct ReshardProvisionShare { + /// Share, encrypted to the ephemeral key + #[serde(with = "qos_hex::serde")] + pub share: Vec, + /// Public key the share targets + #[serde(with = "qos_hex::serde")] + pub pub_key: Vec, +} + +/// A single members input +#[derive( + Debug, + PartialEq, + Eq, + Clone, + borsh::BorshSerialize, + borsh::BorshDeserialize, + serde::Serialize, + serde::Deserialize, +)] +pub struct ReshardProvisionInput { + /// Approval over reshard input + pub approval: Approval, + /// Shares and the associated quorum keys + pub shares: Vec, +} + +/// The parameters for setting up the reshard service. +#[derive( + Debug, + PartialEq, + Eq, + Clone, + borsh::BorshSerialize, + borsh::BorshDeserialize, + serde::Serialize, + serde::Deserialize, +)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(any(feature = "mock", test), derive(Default))] +pub struct ReshardInput { + /// List of quorum public keys + pub quorum_keys: Vec>, + /// The set and threshold to shard the key. + pub new_share_set: ShareSet, + /// The set the key is currently sharded too. + pub old_share_set: ShareSet, + /// The expected configuration of the enclave. Useful to verify the + /// attestation document against. We also want those posting shares to + /// explicitly approve the version of QOS used. + pub enclave: NitroConfig, +} + +impl ReshardInput { + fn deterministic(&mut self) { + self.quorum_keys.sort(); + } + + fn validate(&mut self) -> Result<(), ProtocolError> { + self.deterministic(); + + let new_share_set_members: HashSet<_> = self + .new_share_set + .members + .iter() + .map(|m| m.pub_key.clone()) + .collect(); + + if new_share_set_members.len() != self.new_share_set.members.len() { + return Err(ProtocolError::DuplicateNewShareSetMember); + } + + let quorum_pub_keys: HashSet<_> = self.quorum_keys.iter().collect(); + if quorum_pub_keys.len() != self.quorum_keys.len() { + return Err(ProtocolError::DuplicateQuorumKeys); + } + + Ok(()) + } + + /// Get a set of the quorum keys. + #[must_use] + pub fn quorum_keys(&self) -> HashSet> { + self.quorum_keys.iter().cloned().collect() + } +} + +pub(crate) struct ReshardProvisioner { + secret_builders: HashMap, SecretBuilder>, + quorum_key_count: usize, +} + +impl ReshardProvisioner { + pub(in crate::protocol) fn new(quorum_key_count: usize) -> Self { + Self { secret_builders: HashMap::new(), quorum_key_count } + } + + pub(in crate::protocol) fn add_shares( + &mut self, + shares: Vec, + eph_key: &P256Pair, + ) -> Result<(), ProtocolError> { + if shares.len() != self.quorum_key_count { + return Err( + ProtocolError::ShareCountDoesNotMatchExpectedQuorumKeyCount, + ); + } + + for ReshardProvisionShare { share, pub_key } in shares { + let decrypted_share = eph_key + .decrypt(&share) + .map_err(|_| ProtocolError::ShareDecryptionFailed)?; + + let builder = self + .secret_builders + .entry(pub_key) + .or_insert_with(SecretBuilder::new); + builder.add_share(decrypted_share)?; + } + + Ok(()) + } + + pub(in crate::protocol) fn share_count( + &self, + ) -> Result { + Ok(self + .secret_builders + .values() + .try_fold(None, |count, builder| { + if let Some(current_count) = count { + if current_count != builder.count() { + return Err( + ProtocolError::InternalDiffCountsForQuorumKeyShares, + ); + } + Ok(count) + } else { + Ok(Some(builder.count())) + } + })? + .unwrap_or(0)) + } + + pub(in crate::protocol) fn build( + &mut self, + ) -> Result, ProtocolError> { + self.secret_builders + .drain() + .map(|(public, builder)| { + let master_seed: [u8; 32] = builder + .build()? + .try_into() + .map_err(|_| ProtocolError::IncorrectSecretLen)?; + + let pair = P256Pair::from_master_seed(&master_seed)?; + let public_key_bytes = pair.public_key().to_bytes(); + + if public_key_bytes != public { + return Err( + ProtocolError::ReconstructionErrorIncorrectPubKey, + ); + } + + Ok(pair) + }) + .collect::, ProtocolError>>() + } +} + +/// The output of performing a quorum key reshard. +#[derive( + Debug, + PartialEq, + Eq, + Clone, + borsh::BorshSerialize, + borsh::BorshDeserialize, + serde::Serialize, + serde::Deserialize, +)] +#[serde(rename_all = "camelCase")] +#[cfg_attr(any(feature = "mock", test), derive(Default))] +pub struct ReshardOutput { + /// The new encrypted shards along with metadata about the share set member + /// they where encrypted to. + pub outputs: Vec<(Vec, Vec)>, + /// The set the keys where sharded too. + pub new_share_set: ShareSet, +} + +pub(in crate::protocol) fn boot_reshard( + state: &mut ProtocolState, + mut reshard_input: ReshardInput, +) -> Result { + reshard_input.validate()?; + + state.reshard_provisioner = + Some(ReshardProvisioner::new(reshard_input.quorum_keys.len())); + + state.reshard_input = Some(reshard_input); + + let ephemeral_key = P256Pair::generate()?; + state.handles.put_ephemeral_key(&ephemeral_key)?; + + attestation::reshard_attestation_doc(state) +} + +pub(in crate::protocol) fn reshard_output( + state: &mut ProtocolState, +) -> Result { + state.reshard_output.clone().ok_or(ProtocolError::MissingReshardOutput) +} + +pub(in crate::protocol) fn reshard_provision( + input: ReshardProvisionInput, + state: &mut ProtocolState, +) -> Result { + let reshard_input = state + .reshard_input + .as_ref() + .ok_or(ProtocolError::MissingReshardInput)? + .clone(); + + input.approval.verify(&reshard_input.qos_hash())?; + + if !reshard_input.old_share_set.members.contains(&input.approval.member) { + return Err(ProtocolError::NotShareSetMember); + } + + let ephemeral_key = state.handles.get_ephemeral_key()?; + state + .get_mut_reshard_provisioner()? + .add_shares(input.shares, &ephemeral_key)?; + + let quorum_threshold = reshard_input.old_share_set.threshold as usize; + if state.get_mut_reshard_provisioner()?.share_count()? < quorum_threshold { + // Nothing else to do if we don't have the threshold to reconstruct + return Ok(false); + } + + let quorum_key_pairs = state.get_mut_reshard_provisioner()?.build()?; + let outputs = quorum_key_pairs + .iter() + .map(|pair| { + let master_seed = pair.to_master_seed(); + let pub_key = pair.public_key().to_bytes(); + + let shares = qos_crypto::shamir::shares_generate( + &master_seed[..], + reshard_input.new_share_set.members.len(), + reshard_input.new_share_set.threshold as usize, + ); + + // Now, lets create the new shards + let member_outputs = + zip(shares, reshard_input.new_share_set.members.iter().cloned()) + // .map(|(share, share_set_member)| -> Result { + .map(|(share, share_set_member)| -> Result { + let personal_pub = P256Public::from_bytes(&share_set_member.pub_key)?; + let encrypted_quorum_key_share = personal_pub.encrypt(&share)?; + + Ok(GenesisMemberOutput { + share_set_member, + encrypted_quorum_key_share, + share_hash: sha_512(&share), + }) + }) + .collect::, _>>()?; + + Ok((pub_key, member_outputs)) + }) + .collect::, ProtocolError>>()?; + + state.reshard_output = Some(ReshardOutput { + outputs, + new_share_set: reshard_input.new_share_set, + }); + + Ok(true) +} + +#[cfg(test)] +mod tests { + use qos_crypto::{n_choose_k, shamir::shares_generate}; + use qos_nsm::mock::MockNsm; + use qos_test_primitives::PathWrapper; + + use super::*; + use crate::{ + handles::Handles, + io::SocketAddress, + protocol::{services::boot::QuorumMember, ProtocolPhase, QosHash}, + }; + + struct ReshardSetup { + state: ProtocolState, + new_members: Vec<(QuorumMember, P256Pair)>, + eph_pair: P256Pair, + /// Quorum pairs and associated shares, encrypted to the ephemeral key. + provision_inputs: Vec, + } + + fn reshard_setup(eph_file: &str) -> ReshardSetup { + let handles = Handles::new( + eph_file.to_string(), + "/tmp/qos-quorum".to_string(), + "/tmp/qos-manifest".to_string(), + "/tmp/qos-pivot".to_string(), + ); + let eph_pair = P256Pair::generate().unwrap(); + handles.put_ephemeral_key(&eph_pair).unwrap(); + + let quorum_pairs: Vec<_> = (0..4) + .map(|_| { + let pair = P256Pair::generate().unwrap(); + + let encrypted_shares: Vec<_> = shares_generate( + &pair.to_master_seed()[..], + 4, + 3, // old share set threshold + ) + .iter() + .map(|s| eph_pair.public_key().encrypt(s).unwrap()) + .collect(); + + (pair, encrypted_shares) + }) + .collect(); + let quorum_keys: Vec<_> = + quorum_pairs.iter().map(|p| p.0.public_key().to_bytes()).collect(); + + let old_members: Vec<_> = (0..4) + .map(|_| P256Pair::generate().unwrap()) + .enumerate() + .map(|(i, pair)| { + let member = QuorumMember { + alias: i.to_string(), + pub_key: pair.public_key().to_bytes(), + }; + + (member, pair) + }) + .collect(); + + let new_members: Vec<_> = (0..4) + .map(|_| P256Pair::generate().unwrap()) + .enumerate() + .map(|(i, pair)| { + let member = QuorumMember { + alias: i.to_string(), + pub_key: pair.public_key().to_bytes(), + }; + + (member, pair) + }) + .collect(); + + let reshard_input = ReshardInput { + quorum_keys, + new_share_set: ShareSet { + threshold: 2, + members: new_members.iter().map(|(qm, _)| qm.clone()).collect(), + }, + old_share_set: ShareSet { + threshold: 3, + members: old_members.iter().map(|(qm, _)| qm.clone()).collect(), + }, + enclave: NitroConfig { + pcr0: vec![4; 32], + pcr1: vec![3; 32], + pcr2: vec![2; 32], + pcr3: vec![1; 32], + aws_root_certificate: b"bezo's son, a dad of certs".to_vec(), + qos_commit: "super chill commit ref you can bro down with" + .to_string(), + }, + }; + + let provision_inputs: Vec<_> = old_members + .into_iter() + .enumerate() + .map(|(idx, (member, pair))| { + let shares = quorum_pairs + .iter() + .map(|(pair, shares)| ReshardProvisionShare { + share: shares[idx].clone(), + pub_key: pair.public_key().to_bytes(), + }) + .collect(); + let approval = Approval { + member, + signature: pair.sign(&reshard_input.qos_hash()).unwrap(), + }; + + ReshardProvisionInput { approval, shares } + }) + .collect(); + + let mut state = ProtocolState::new( + Box::new(MockNsm), + handles, + SocketAddress::new_unix("./never.sock"), + None, + ); + state.reshard_input = Some(reshard_input); + state.reshard_provisioner = + Some(ReshardProvisioner::new(quorum_pairs.len())); + state.transition(ProtocolPhase::ReshardWaitingForQuorumShards).unwrap(); + + ReshardSetup { state, new_members, eph_pair, provision_inputs } + } + + #[test] + fn reshard_provision_works() { + let eph_file: PathWrapper = + "/tmp/reshard_provision_works.eph.key".into(); + + let ReshardSetup { mut state, new_members, provision_inputs, .. } = + reshard_setup(&eph_file); + + // We expect reshard_provision to return Ok(false) for the first + // 2 + for i in provision_inputs.iter().take(2) { + assert_eq!(reshard_provision(i.clone(), &mut state), Ok(false)); + } + + // And then return Ok(true) for the 3rd share to signal it has been + // reconstructed + assert_eq!( + reshard_provision(provision_inputs[2].clone(), &mut state), + Ok(true) + ); + + let reshard_output = state.reshard_output.clone().unwrap(); + let reshard_input = state.reshard_input.clone().unwrap(); + assert_eq!(reshard_output.new_share_set, reshard_input.new_share_set); + + for (quorum_pub, member_outputs) in reshard_output.outputs { + // Check that decrypted shares match hash + let mut decrypted_shares = vec![]; + for (member_out, (member, pair)) in + zip(member_outputs, new_members.clone()) + { + let share = pair + .decrypt(&member_out.encrypted_quorum_key_share) + .unwrap(); + assert_eq!( + &member_out.share_hash, + &qos_crypto::sha_512(&share), + ); + assert_eq!(member_out.share_set_member, member); + + decrypted_shares.push(share); + } + + // Now make sure all combos of shares work + for combo in n_choose_k::combinations( + &decrypted_shares, + reshard_output.new_share_set.threshold as usize, + ) { + let secret: [u8; 32] = + qos_crypto::shamir::shares_reconstruct(&combo) + .try_into() + .unwrap(); + let quorum_key = P256Pair::from_master_seed(&secret).unwrap(); + assert_eq!(quorum_pub, quorum_key.public_key().to_bytes()); + } + } + } + + #[test] + fn reshard_provision_rejects_wrong_reconstructed_key() { + let eph_file: PathWrapper = + "/tmp/reshard_provision_rejects_wrong_reconstructed_key.eph.key" + .into(); + + let ReshardSetup { eph_pair, mut state, mut provision_inputs, .. } = + reshard_setup(&eph_file); + + let reshard_input = state.reshard_input.clone().unwrap(); + let random_pair = P256Pair::generate().unwrap(); + let encrypted_shares: Vec<_> = shares_generate( + random_pair.to_master_seed(), + 4, + reshard_input.new_share_set.threshold as usize, + ) + .iter() + .map(|shard| eph_pair.public_key().encrypt(shard).unwrap()) + .collect(); + + for (i, user_input) in provision_inputs.iter_mut().enumerate() { + // loop through each of the users input and modify their first share + // to be for a different public key + user_input.shares[0].share = encrypted_shares[i].clone(); + } + + // We expect reshard_provision to return Ok(false) for the first + // 2 + for i in provision_inputs.iter().take(2) { + assert_eq!(reshard_provision(i.clone(), &mut state), Ok(false)); + } + + // And then return an error for the 3rd share to signal it has been + // reconstructed + assert_eq!( + reshard_provision(provision_inputs[2].clone(), &mut state), + Err(ProtocolError::ReconstructionErrorIncorrectPubKey) + ); + } + + #[test] + fn reshard_provision_rejects_bad_approval_signature() { + let eph_file: PathWrapper = + "/tmp/reshard_provision_rejects_bad_approval_signature.eph.key" + .into(); + + let ReshardSetup { + mut state, mut provision_inputs, new_members, .. + } = reshard_setup(&eph_file); + + // give the third approval a random signature + provision_inputs[2].approval.signature = + new_members[2].1.sign(&[42; 32]).unwrap(); + + // We expect reshard_provision to return Ok(false) for the first + // 2 + for i in provision_inputs.iter().take(2) { + assert_eq!(reshard_provision(i.clone(), &mut state), Ok(false)); + } + + // And then return an error for the 3rd share to signal it has been + // reconstructed + assert_eq!( + reshard_provision(provision_inputs[2].clone(), &mut state), + Err(ProtocolError::CouldNotVerifyApproval) + ); + } + + #[test] + fn reshard_provision_rejects_approval_not_from_member() { + let eph_file: PathWrapper = + "/tmp/reshard_provision_rejects_approval_not_from_member.eph.key" + .into(); + + let ReshardSetup { + eph_pair, + mut state, + mut provision_inputs, + new_members, + .. + } = reshard_setup(&eph_file); + + let reshard_input = state.reshard_input.clone().unwrap(); + let random_pair = P256Pair::generate().unwrap(); + let _encrypted_shares: Vec<_> = shares_generate( + random_pair.to_master_seed(), + 4, + reshard_input.new_share_set.threshold as usize, + ) + .iter() + .map(|shard| eph_pair.public_key().encrypt(shard).unwrap()) + .collect(); + + // the old and new members are unique. We only expect approvals from the + // old members. So if a new members approval comes in, we don't accept + // it. + provision_inputs[2].approval.signature = + new_members[0].1.sign(&reshard_input.qos_hash()).unwrap(); + provision_inputs[2].approval.member = new_members[0].0.clone(); + + // We expect reshard_provision to return Ok(false) for the first + // 2 + for i in provision_inputs.iter().take(2) { + assert_eq!(reshard_provision(i.clone(), &mut state), Ok(false)); + } + + // And then return an error for the 3rd share to signal it has been + // reconstructed + assert_eq!( + reshard_provision(provision_inputs[2].clone(), &mut state), + Err(ProtocolError::NotShareSetMember) + ); + } + + #[test] + fn boot_reshard_works() { + let eph_file: PathWrapper = "/tmp/boot_reshard_works.eph.key".into(); + + let handles = Handles::new( + eph_file.to_string(), + "/tmp/qos-quorum".to_string(), + "/tmp/qos-manifest".to_string(), + "/tmp/qos-pivot".to_string(), + ); + let mut state = ProtocolState::new( + Box::new(MockNsm), + handles, + SocketAddress::new_unix("./never.sock"), + None, + ); + + let reshard_input = ReshardInput { + quorum_keys: vec![vec![1; 65], vec![2; 65]], + new_share_set: ShareSet { threshold: 2, members: vec![] }, + old_share_set: ShareSet { threshold: 3, members: vec![] }, + enclave: NitroConfig { + pcr0: vec![4; 32], + pcr1: vec![3; 32], + pcr2: vec![2; 32], + pcr3: vec![1; 32], + aws_root_certificate: + b"super swag root cert your friends told you about".to_vec(), + qos_commit: "a commit ref".to_string(), + }, + }; + + assert!(boot_reshard(&mut state, reshard_input.clone(),).is_ok()); + + assert_eq!(state.reshard_input, Some(reshard_input)); + assert_eq!(state.reshard_output, None); + assert!(state.handles.get_ephemeral_key().is_ok()); + } +} diff --git a/src/qos_core/src/protocol/state.rs b/src/qos_core/src/protocol/state.rs index 4f36d39b..dd00249e 100644 --- a/src/qos_core/src/protocol/state.rs +++ b/src/qos_core/src/protocol/state.rs @@ -4,9 +4,17 @@ use nix::sys::time::{TimeVal, TimeValLike}; use qos_nsm::NsmProvider; use super::{ - error::ProtocolError, msg::ProtocolMsg, services::provision::SecretBuilder, + error::ProtocolError, + msg::ProtocolMsg, + services::{ + provision::SecretBuilder, + reshard::{ReshardInput, ReshardProvisioner}, + }, +}; +use crate::{ + client::Client, handles::Handles, io::SocketAddress, + protocol::services::reshard::ReshardOutput, }; -use crate::{client::Client, handles::Handles, io::SocketAddress}; /// The timeout for the qos core when making requests to an enclave app. pub const ENCLAVE_APP_SOCKET_CLIENT_TIMEOUT_SECS: i64 = 5; @@ -36,6 +44,13 @@ pub enum ProtocolPhase { QuorumKeyProvisioned, /// Waiting for a forwarded key to be injected WaitingForForwardedKey, + /// Waiting for quorum key shards to be posted so the reshard service can + /// be executed. + ReshardWaitingForQuorumShards, + /// Reshard service has completed + ReshardBooted, + /// Reshard failed to reconstruct the quorum key + UnrecoverableReshardFailedBadShares, } /// Enclave routes @@ -58,11 +73,12 @@ impl ProtocolRoute { let resp = (self.handler)(msg, state); // ignore transitions in special cases - if let Some(Ok(ProtocolMsg::ProvisionResponse { reconstructed })) = resp - { - if !reconstructed { - return resp; - } + match resp { + Some(Ok( + ProtocolMsg::ProvisionResponse { reconstructed } + | ProtocolMsg::ReshardProvisionResponse { reconstructed }, + )) if !reconstructed => return resp, + _ => { /* This isn't a special case, keep going */ } } // handle state transitions @@ -107,6 +123,14 @@ impl ProtocolRoute { ) } + pub fn reshard_attestation_doc(current_phase: ProtocolPhase) -> Self { + ProtocolRoute::new( + Box::new(handlers::reshard_attestation_doc), + current_phase, + current_phase, + ) + } + pub fn boot_genesis(_current_phase: ProtocolPhase) -> Self { ProtocolRoute::new( Box::new(handlers::boot_genesis), @@ -123,6 +147,14 @@ impl ProtocolRoute { ) } + pub fn boot_reshard(_current_phase: ProtocolPhase) -> Self { + ProtocolRoute::new( + Box::new(handlers::boot_reshard), + ProtocolPhase::ReshardWaitingForQuorumShards, + ProtocolPhase::UnrecoverableError, + ) + } + pub fn boot_key_forward(_current_phase: ProtocolPhase) -> Self { ProtocolRoute::new( Box::new(handlers::boot_key_forward), @@ -139,6 +171,22 @@ impl ProtocolRoute { ) } + pub fn reshard_provision(_current_phase: ProtocolPhase) -> Self { + ProtocolRoute::new( + Box::new(handlers::reshard_provision), + ProtocolPhase::ReshardBooted, + ProtocolPhase::UnrecoverableReshardFailedBadShares, + ) + } + + pub fn reshard_output(current_phase: ProtocolPhase) -> Self { + ProtocolRoute::new( + Box::new(handlers::reshard_output), + current_phase, + current_phase, + ) + } + pub fn proxy(current_phase: ProtocolPhase) -> Self { ProtocolRoute::new( Box::new(handlers::proxy), @@ -179,9 +227,13 @@ pub(crate) struct ProtocolState { pub app_client: Client, pub handles: Handles, phase: ProtocolPhase, + pub reshard_input: Option, + pub reshard_output: Option, + pub(crate) reshard_provisioner: Option, } impl ProtocolState { + #[allow(unused_variables)] pub fn new( attestor: Box, handles: Handles, @@ -208,6 +260,9 @@ impl ProtocolState { app_addr, TimeVal::seconds(ENCLAVE_APP_SOCKET_CLIENT_TIMEOUT_SECS), ), + reshard_input: None, + reshard_output: None, + reshard_provisioner: None, } } @@ -215,6 +270,14 @@ impl ProtocolState { self.phase } + pub(crate) fn get_mut_reshard_provisioner( + &mut self, + ) -> Result<&mut ReshardProvisioner, ProtocolError> { + self.reshard_provisioner + .as_mut() + .ok_or(ProtocolError::MissingReshardProvisioner) + } + pub fn handle_msg(&mut self, msg_req: &ProtocolMsg) -> Vec { for route in &self.routes() { match route.try_msg(msg_req, self) { @@ -257,6 +320,7 @@ impl ProtocolState { ProtocolRoute::boot_genesis(self.phase), ProtocolRoute::boot_standard(self.phase), ProtocolRoute::boot_key_forward(self.phase), + ProtocolRoute::boot_reshard(self.phase), ], ProtocolPhase::WaitingForQuorumShards => { vec![ @@ -289,6 +353,29 @@ impl ProtocolState { ProtocolRoute::inject_key(self.phase), ] } + ProtocolPhase::ReshardWaitingForQuorumShards => { + vec![ + // baseline routes + ProtocolRoute::status(self.phase), + ProtocolRoute::reshard_attestation_doc(self.phase), + // phase specific routes + ProtocolRoute::reshard_provision(self.phase), + ] + } + ProtocolPhase::UnrecoverableReshardFailedBadShares => { + vec![ + // baseline routes + ProtocolRoute::status(self.phase), + ] + } + ProtocolPhase::ReshardBooted => { + vec![ + // baseline routes + ProtocolRoute::status(self.phase), + // phase specific routes + ProtocolRoute::reshard_output(self.phase), + ] + } } } @@ -307,6 +394,7 @@ impl ProtocolState { ProtocolPhase::UnrecoverableError, ProtocolPhase::GenesisBooted, ProtocolPhase::WaitingForQuorumShards, + ProtocolPhase::ReshardWaitingForQuorumShards, ProtocolPhase::WaitingForForwardedKey, ], ProtocolPhase::GenesisBooted => { @@ -327,6 +415,18 @@ impl ProtocolState { ProtocolPhase::QuorumKeyProvisioned, ] } + ProtocolPhase::ReshardWaitingForQuorumShards => { + vec![ + ProtocolPhase::UnrecoverableReshardFailedBadShares, + ProtocolPhase::ReshardBooted, + ] + } + ProtocolPhase::ReshardBooted => { + vec![ProtocolPhase::UnrecoverableError] + } + ProtocolPhase::UnrecoverableReshardFailedBadShares => { + vec![ProtocolPhase::UnrecoverableError] + } }; if !transitions.contains(&next) { @@ -343,9 +443,11 @@ impl ProtocolState { mod handlers { use super::ProtocolRouteResponse; use crate::protocol::{ + error::ProtocolError, msg::ProtocolMsg, services::{ - attestation, boot, genesis, key, key::EncryptedQuorumKey, provision, + attestation, boot, genesis, key, key::EncryptedQuorumKey, + provision, reshard, }, ProtocolState, }; @@ -412,6 +514,40 @@ mod handlers { } } + pub(super) fn reshard_provision( + req: &ProtocolMsg, + state: &mut ProtocolState, + ) -> ProtocolRouteResponse { + if let ProtocolMsg::ReshardProvisionRequest { input } = req { + let result = reshard::reshard_provision(input.clone(), state) + .map(|reconstructed| ProtocolMsg::ReshardProvisionResponse { + reconstructed, + }) + .map_err(ProtocolMsg::ProtocolErrorResponse); + + Some(result) + } else { + None + } + } + + pub(super) fn reshard_output( + req: &ProtocolMsg, + state: &mut ProtocolState, + ) -> ProtocolRouteResponse { + if let ProtocolMsg::ReshardOutputRequest = req { + let result = reshard::reshard_output(state) + .map(|reshard_output| ProtocolMsg::ReshardOutputResponse { + reshard_output, + }) + .map_err(ProtocolMsg::ProtocolErrorResponse); + + Some(result) + } else { + None + } + } + /// Handle `ProtocolMsg::BootStandardRequest`. pub(super) fn boot_standard( req: &ProtocolMsg, @@ -452,6 +588,23 @@ mod handlers { } } + pub(super) fn boot_reshard( + req: &ProtocolMsg, + state: &mut ProtocolState, + ) -> ProtocolRouteResponse { + if let ProtocolMsg::BootReshardRequest { reshard_input } = req { + let result = reshard::boot_reshard(state, reshard_input.clone()) + .map(|nsm_response| ProtocolMsg::BootReshardResponse { + nsm_response, + }) + .map_err(ProtocolMsg::ProtocolErrorResponse); + + Some(result) + } else { + None + } + } + pub(super) fn live_attestation_doc( req: &ProtocolMsg, state: &mut ProtocolState, @@ -474,6 +627,37 @@ mod handlers { } } + pub(super) fn reshard_attestation_doc( + req: &ProtocolMsg, + state: &mut ProtocolState, + ) -> ProtocolRouteResponse { + if let ProtocolMsg::ReshardAttestationDocRequest = req { + let reshard_input = match state + .reshard_input + .as_ref() + .ok_or(ProtocolError::MissingReshardInput) + { + Ok(r) => r.clone(), + Err(e) => { + return Some(Ok(ProtocolMsg::ProtocolErrorResponse(e))) + } + }; + + let result = attestation::reshard_attestation_doc(state) + .map(|nsm_response| { + ProtocolMsg::ReshardAttestationDocResponse { + nsm_response, + reshard_input, + } + }) + .map_err(ProtocolMsg::ProtocolErrorResponse); + + Some(result) + } else { + None + } + } + pub(super) fn boot_key_forward( req: &ProtocolMsg, state: &mut ProtocolState, diff --git a/src/qos_crypto/mock/rsa_private.mock.pem b/src/qos_crypto/mock/rsa_private.mock.pem deleted file mode 100644 index f67905f3..00000000 --- a/src/qos_crypto/mock/rsa_private.mock.pem +++ /dev/null @@ -1,51 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -MIIJKQIBAAKCAgEAzjT6oPhEHDs87REJ5DQEbzr81/0NY0hpI7CkkDy2H57oQX0k -hD+LW7qFu16vbIKmxSaJX1SaMHsWqbHIBCTTVZlsxXOrSwuO3YP78inrvNZrxafJ -+LkIaV8UZeXY1h3ACqj3kDJLV1/5EZfTyfMaNUwIUZdiqoa4bwFWDFhNXrlCgIfM -SGyErfMRR7qQCGhEtRUnZPW/+hbuZU1OfbkUAXvsb2IwgJHwqAdYcb6F0bkXZIta -aXEoUG2qIgspYJgOF81rvzdvdEh8ezBEJZIaExovIa0oc5D3yI7ZHbgGMRRT3Mah -oS+UoSBHTjR2yBCMmPMwDuhUCZtqf6/Mz5EouRWst5ezAG2vAqrPRFD0jI8tUDY7 -wSQ0nC8NLUO66CdvHkVZQhHedePEfnk1tJcGvvKWCIDf5AtHARZ89DS5uooH0DyD -ACLJevVE+UAUHxjgOz0Y6SQ4R2sQnROm7b5tWZD24puZ2bsyLz2FMd+Hb/UtDF5C -ZPc5kXqVpQ6W9TaHtQPxS9QrUsyLP2JjQvYZokrBQPwLrdXKdutGO76LjbyTF6ep -lGz87T86zuHW70rEjLXUCo2Xz32gbrNgNRLugNSwrMuIihhGaoFWOZLPXEV30c2r -z0uEDdqt5jFt0fDbNJPSMZnStaIrZjlv/g5CKA1kgPFbYytGcJ1/6cUGjSUCAwEA -AQKCAgAOZRZl7E7c5anAJuNY4eS5WxXRgiHQH3rveeJTC1nvZSlzgOfur3zr/15c -kSSP36MCukj2tbI51i3j1LxQxb1XCWnVctivWXQ0tIT/B7wkJ5fIaYko6snSiUek -QWJcuCDy3Y7CqzAlPlblyoKHY4gd1lvyTi4eF1+CqEY4gGWOSkKBNHmnSKQOfJxl -NHnfjF/XgE2Kt7kaHUWxHq9vCV+DJwJ/WAxovUdeg3zCG/m9hT1D0JKUL8kPrEgn -Lj5KNvMs6DMwWw9Vv8Wo19q6ALP/R2Go6SycvK7/ejFf6LvN60xbaiZYWTj38ofn -xrPQDY+zEa72K3PKY+YQWdZjWw/Lk7zwhQfmFFY9Hfn2ff9IpaPOTyeman7msGSO -fya0WRG4ZE/SoekoGrQ/A+PkuYxeOBfRu1WL349+hKs3zI0y/cOMsL/BMAX1mdrz -N9KLFz4l4oYhABgMTwnR+EjSn4sVzhPVsM9YbfMmHlb6L3hgxMvm/Fg8hz8FR95w -OkkOK0lXT5jlfwrV3rwKojLv2T8lXnQtlpUa5kl+H5Md8OZYacvsyTRZE2BTwoY7 -g+w3n3uBMXL0Q0c9K6lxoyIn6LKRMADk6/L0IpC40POWRHoiaZYbOBVAQDHPiSuL -c/pQdq5ClXVEkbPSFRP4xKwQvCxnEiZqCk5CI9+zHmOCU6AWoQKCAQEA83NGnmQy -/jZqLceYsHETdzQZ7F6nN/OWs6sGd94F91QbVOLFV5XayaVMR8GOfOl7T8jVzQly -s+Hc+4tHItbMT9Umje682MS5K0RPtnJFTIgyL1SurIV97ni8YCVG8iGKUOkTMQ+U -bVoRhbb8bXYB9q0QNhYboUKsc9OoyUnR1jBy/zqJGkP1daEQWZozUSn6pWJBIXb0 -UpheCVGa80a6ccYdi92OGoTOYVF5t4ZA0LRxOKMjfN/0A1Fysz/PsmRfmrgnSFOl -HuDf0YO6vSwlgXX9LQ2//gkTsibc1/GHazmvTk57sjYyjCWGRKJCJ/rpjECNNjDK -X3F7v9jM+14waQKCAQEA2NY3ZYD9NXuJM0ZtWZLTMAJkSWsWHBQwqezflJEXP0QO -vomu73Qx6Xnqurg2lZrJg3bBYebwMDSCt9w391z/LyTtmJ6P9jAWkr4unUlG3nXa -WUAyu668VQZkFKh9iasjvC9ln2qyrp2/9sA/C4ZPGLS1PxpGkSKtzcEHXZINB9ht -caYxPkccV6WsMe/ytGFhKdhPb+O1POSE0YwEVt0e7+iki09zSNJ7e5jA31lEriat -CK6KY4FjAmpxMlS2d0mBuBd4NCW5vzXKKNVH8BUXVXbm6h4gAjc6rU6nV0KhpW34 -VVvBmewYDB2b4JzcgjiDlFy8r5EVdZy4ij4nhhhfXQKCAQEAkdyFeS0LqGgt4dPu -1fhJ82fSCF8FzW4y4t8bdwIdjPxli8x69GkityJEu9Fqb8jsSvdHshtxD/nJjyT5 -sBQGQeaxvORHXZEwaI37PJLmll4bw2P3bAJnW1QXeXucMEKMPsIG76QoCASo7vad -82966bLzPZStZUcvUA6G2GNUSAKrQ+RsdMI29Q1VYHoVORHvzNs7rrM426vS2757 -GjtMRhKvbTeHhrf/dyt7w8u6VdFm7MpB3vXHm51XHbKj3HxrE6Y2Uw1ap0+QilVk -sycaKaDp2e6dE7WYiWrjcraRrlrXgBFh53q1emaZNdIJ1S5uc8vRT6CX/+tce6uH -1Suv0QKCAQAFq8STIQZ+SZbTAnqFpzNixA0/Zk+TuGt1Zj6Ksii7fNot3Yf3t0A+ -7PNYosy6qOuwRoDUQKfzeswYZugHziTWZM7Z+Pum4qcUe2jYsDvsQYTOZMFu6yj9 -yEcBy05NNW6f01WDD9VQf8uvdmOvt3mGGePLnLJPxWpqQSwiJFm25NAn8sLC8DUr -jaetPqtIUGusHn4lXP02dHuMx26tnubaO2liQ1euheK43svci4ciTtyjp3zzEUU3 -oPUI7fI/uGpuGB8KrhnniE6bNsjE3KhZkdyELvmDVVJxiecSfymfG/sssFOl5OjU -GEolW7Tgqv21+Z7tsIuxIcIpy2pZNXX5AoIBAQCgTSzO6goiQrJTD/OU338IDTYx -nnJ0+vecyfPOBymvn5Q0QBIzjs+Mxy0X1IAwBAUlC8kcDJyoaFzyt67K+OKB04vc -+bezx2efDzRCaHqYlP6NeFP+C/LdcZvjsQb5UYMebk7XYqDHhKoy39n838ahYsjZ -NoeRk9bEBZZhhEa715Vmugkk+SL7lbFvSXvtJfJltW0oqjnjY1lVVWrzrHJPq+J1 -D0RqOc/pJ/b6bAo3PAODqGcMV5XQE0zC9ALTH/K20g0PpfCe9xuEYW6qeycyVIAa -0JY2WbC6v4PTjpO1X3cP1UF0GCMrkmc0fSbUxd5+sl4ThVPdtPxCpmBzORYB ------END RSA PRIVATE KEY----- diff --git a/src/qos_crypto/mock/rsa_public.mock.pem b/src/qos_crypto/mock/rsa_public.mock.pem deleted file mode 100644 index 87226850..00000000 --- a/src/qos_crypto/mock/rsa_public.mock.pem +++ /dev/null @@ -1,14 +0,0 @@ ------BEGIN PUBLIC KEY----- -MIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAzjT6oPhEHDs87REJ5DQE -bzr81/0NY0hpI7CkkDy2H57oQX0khD+LW7qFu16vbIKmxSaJX1SaMHsWqbHIBCTT -VZlsxXOrSwuO3YP78inrvNZrxafJ+LkIaV8UZeXY1h3ACqj3kDJLV1/5EZfTyfMa -NUwIUZdiqoa4bwFWDFhNXrlCgIfMSGyErfMRR7qQCGhEtRUnZPW/+hbuZU1OfbkU -AXvsb2IwgJHwqAdYcb6F0bkXZItaaXEoUG2qIgspYJgOF81rvzdvdEh8ezBEJZIa -ExovIa0oc5D3yI7ZHbgGMRRT3MahoS+UoSBHTjR2yBCMmPMwDuhUCZtqf6/Mz5Eo -uRWst5ezAG2vAqrPRFD0jI8tUDY7wSQ0nC8NLUO66CdvHkVZQhHedePEfnk1tJcG -vvKWCIDf5AtHARZ89DS5uooH0DyDACLJevVE+UAUHxjgOz0Y6SQ4R2sQnROm7b5t -WZD24puZ2bsyLz2FMd+Hb/UtDF5CZPc5kXqVpQ6W9TaHtQPxS9QrUsyLP2JjQvYZ -okrBQPwLrdXKdutGO76LjbyTF6eplGz87T86zuHW70rEjLXUCo2Xz32gbrNgNRLu -gNSwrMuIihhGaoFWOZLPXEV30c2rz0uEDdqt5jFt0fDbNJPSMZnStaIrZjlv/g5C -KA1kgPFbYytGcJ1/6cUGjSUCAwEAAQ== ------END PUBLIC KEY----- diff --git a/src/qos_crypto/src/lib.rs b/src/qos_crypto/src/lib.rs index 3bfac9a2..f94fdd3a 100644 --- a/src/qos_crypto/src/lib.rs +++ b/src/qos_crypto/src/lib.rs @@ -7,6 +7,7 @@ use sha2::Digest; +pub mod n_choose_k; pub mod shamir; /// Create a SHA256 hash digest of `buf`. diff --git a/src/qos_crypto/src/n_choose_k.rs b/src/qos_crypto/src/n_choose_k.rs new file mode 100644 index 00000000..add5b706 --- /dev/null +++ b/src/qos_crypto/src/n_choose_k.rs @@ -0,0 +1,140 @@ +//! n choose k helper + +/// Computes n choose k combinations over a vector of elements of type T. +/// +/// # Arguments +/// +/// * `input` - A reference to a vector of elements of type T. +/// * `k` - The number of elements to choose in each combination. +/// +/// # Examples +/// +/// ``` +/// use qos_crypto::n_choose_k::combinations; +/// +/// let input = vec![1, 2, 3, 4]; +/// let k = 2; +/// let combinations = combinations(&input, k); +/// +/// // Verify that the computed combinations match the expected result +/// assert_eq!(combinations, vec![ +/// vec![1, 2], +/// vec![1, 3], +/// vec![1, 4], +/// vec![2, 3], +/// vec![2, 4], +/// vec![3, 4], +/// ]); +/// ``` +#[must_use] +pub fn combinations(input: &[T], k: usize) -> Vec> { + let n = input.len(); + + if k > n || k == 0 { + return Vec::new(); + } + + let mut combos = + Vec::with_capacity(expected_combinations_count(input.len(), k)); + let mut indices: Vec<_> = (0..k).collect(); + + // Generate combinations + while indices[0] <= n - k { + // Create a combination by mapping indices to corresponding elements in + // the input + let combination: Vec<_> = + indices.iter().map(|&i| input[i].clone()).collect(); + combos.push(combination.clone()); + + let mut i = k; + while i > 1 && indices[i - 1] == n - k + i - 1 { + i -= 1; + } + + indices[i - 1] += 1; + + for j in i..k { + indices[j] = indices[j - 1] + 1; + } + } + + combos +} + +fn expected_combinations_count(n: usize, k: usize) -> usize { + factorial(n) / (factorial(k) * factorial(n - k)) +} + +fn factorial(n: usize) -> usize { + if n == 0 || n == 1 { + 1 + } else { + (2..=n).product() + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn normal_cases() { + // n = 4, k = 2 + let byte_input = vec![b'a', b'b', b'c', b'd']; + let k1 = 2; + let byte_result = combinations(&byte_input, k1); + assert_eq!( + byte_result.len(), + expected_combinations_count(byte_input.len(), k1) + ); + assert!(byte_result.contains(&vec![b'a', b'b'])); + assert!(byte_result.contains(&vec![b'a', b'c'])); + assert!(byte_result.contains(&vec![b'a', b'd'])); + assert!(byte_result.contains(&vec![b'b', b'c'])); + assert!(byte_result.contains(&vec![b'b', b'd'])); + assert!(byte_result.contains(&vec![b'c', b'd'])); + + // n = 3, k = 3 + let char3_input = vec!['x', 'y', 'z']; + let k2 = 3; + let char3_result = combinations(&char3_input, k2); + assert_eq!( + char3_result.len(), + expected_combinations_count(char3_input.len(), k2) + ); + assert_eq!(char3_result, vec![vec!['x', 'y', 'z']]); + + // n = 2, k = 1 + let char2_input = vec!['p', 'q']; + let k3 = 1; + let char2_result = combinations(&char2_input, k3); + assert_eq!( + char2_result.len(), + expected_combinations_count(char2_input.len(), k3) + ); + assert_eq!(char2_result, vec![vec!['p'], vec!['q']]); + } + + #[test] + fn edge_cases() { + // empty input + let empty_input: Vec = Vec::new(); + let empty_result = combinations(&empty_input, 0); + assert_eq!(empty_result.len(), 0); + + // k = 0 + let input = vec![1, 2, 3, 4, 5]; + let k = 0; + let result = combinations(&input, k); + assert_eq!(result.len(), 0); + } + + #[test] + fn factorial_works() { + assert_eq!(factorial(0), 1); + assert_eq!(factorial(1), 1); + assert_eq!(factorial(5), 120); + assert_eq!(factorial(10), 3_628_800); + assert_eq!(factorial(20), 2_432_902_008_176_640_000); + } +} diff --git a/src/qos_crypto/src/shamir.rs b/src/qos_crypto/src/shamir.rs index f73dd9c4..2c0beca7 100644 --- a/src/qos_crypto/src/shamir.rs +++ b/src/qos_crypto/src/shamir.rs @@ -240,5 +240,10 @@ mod test { shares.shuffle(&mut rand::thread_rng()); let reconstructed = shares_reconstruct(&shares); assert_eq!(secret.to_vec(), reconstructed); + + for combo in crate::n_choose_k::combinations(&all_shares, k) { + let reconstructed = shares_reconstruct(&combo); + assert_eq!(secret.to_vec(), reconstructed); + } } } diff --git a/src/qos_host/src/lib.rs b/src/qos_host/src/lib.rs index 0f0a248d..3461322d 100644 --- a/src/qos_host/src/lib.rs +++ b/src/qos_host/src/lib.rs @@ -202,9 +202,14 @@ impl HostServer { ProtocolPhase::UnrecoverableError | ProtocolPhase::WaitingForBootInstruction | ProtocolPhase::WaitingForQuorumShards - | ProtocolPhase::WaitingForForwardedKey => StatusCode::SERVICE_UNAVAILABLE, + | ProtocolPhase::WaitingForForwardedKey + | ProtocolPhase::ReshardWaitingForQuorumShards + | ProtocolPhase::UnrecoverableReshardFailedBadShares => { + StatusCode::SERVICE_UNAVAILABLE + } ProtocolPhase::QuorumKeyProvisioned - | ProtocolPhase::GenesisBooted => StatusCode::OK, + | ProtocolPhase::GenesisBooted + | ProtocolPhase::ReshardBooted => StatusCode::OK, }; (status, Html(inner)) diff --git a/src/toolchain b/src/toolchain new file mode 160000 index 00000000..00ce00c2 --- /dev/null +++ b/src/toolchain @@ -0,0 +1 @@ +Subproject commit 00ce00c246766b8a2ce09b96cf23b812a04d03a4 From 712d75b27039472f04ea9dd6660c660cb1846682 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Tue, 23 Jan 2024 19:15:39 -0500 Subject: [PATCH 02/31] Update changelog for #428 --- CHANGELOG.MD | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/CHANGELOG.MD b/CHANGELOG.MD index 8c33f23d..6e1cf1cc 100644 --- a/CHANGELOG.MD +++ b/CHANGELOG.MD @@ -14,6 +14,15 @@ Removed: for now removed features. Fixed: for any bug fixes. Security: in case of vulnerabilities. +## Unreleased + +### Added + +- BREAKING CHANGE: qos_core: quorum key resharding service, new state machine transitions, and new `ProtocolMsg` variants (#428) +- qos_client: commands to run quorum key resharding and high level documentation (#428) +- qos_crypto: function to generate n choose k variants (#428) +- qos_hex: support more array sizes for serde deserialize + ## [0.4.0] 2024.4.9 ### Added From ff89a9140311379fc0ed7891abe06b2fa4fd2866 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Wed, 24 Jan 2024 18:44:09 -0500 Subject: [PATCH 03/31] Update gitignore --- .gitignore | 3 +++ .../mock/reshard/user1/qkey1/user1.share | Bin 0 -> 130 bytes .../mock/reshard/user2/qkey1/user2.share | Bin 0 -> 130 bytes .../mock/reshard/user3/qkey1/user3.share | Bin 0 -> 130 bytes 4 files changed, 3 insertions(+) create mode 100644 src/integration/mock/reshard/user1/qkey1/user1.share create mode 100644 src/integration/mock/reshard/user2/qkey1/user2.share create mode 100644 src/integration/mock/reshard/user3/qkey1/user3.share diff --git a/.gitignore b/.gitignore index 198ea39d..ec33ad6a 100644 --- a/.gitignore +++ b/.gitignore @@ -23,6 +23,9 @@ target/ !src/integration/mock/boot-e2e/all-personal-dir/user3-dir/* !src/integration/mock/boot-e2e/genesis-dir/* !src/integration/mock/new-share-set-secrets/* +!src/integration/mock/reshard/user1/qkey1/* +!src/integration/mock/reshard/user2/qkey1/* +!src/integration/mock/reshard/user3/qkey1/* src/integration/mock/pivot-build-fingerprints.txt src/integration/pivot_ok2_works src/integration/pivot_ok_works diff --git a/src/integration/mock/reshard/user1/qkey1/user1.share b/src/integration/mock/reshard/user1/qkey1/user1.share new file mode 100644 index 0000000000000000000000000000000000000000..f42b37d9336f529d9ba53415a38688d213e6b7df GIT binary patch literal 130 zcmV-|0Db=!7uE74GA>mMOEG)|h$uz?x`{Z_fYG*tbYoC@1@^MxSu_ds7Tm^LS3qd94?F#rGn ksh3L-+h5bv=22?hkYrA+(!<);`@(p0Orrv~o)LUHx#^HPeEssm82>S4SF93k!su`rKQXd)-#hHK+lT=NB?17m z-CTCJouYu5nB~PrEU>$hkmxuO#;nQf7_;)=Km9H=F#rGnfl^*bQ3Y0b$c7RV**wF3buxB~zK6m#7e%==dwO)s%ptkilFcvo*oz}rc3`)2Ayjx2m8;()2 zC;!MX3r|P5iX_$osnp^CT)`WmG^y^N{LGnGN6kX Date: Tue, 13 Feb 2024 11:59:14 -0500 Subject: [PATCH 04/31] wip --- src/qos_client/RESHARD_GUIDE.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qos_client/RESHARD_GUIDE.md b/src/qos_client/RESHARD_GUIDE.md index 95fe5771..c7fa4489 100644 --- a/src/qos_client/RESHARD_GUIDE.md +++ b/src/qos_client/RESHARD_GUIDE.md @@ -104,7 +104,7 @@ Verify that we can decrypt our shares. This step should be done on an airgapped The new shares will be written to subdirectories generated within the given share dir. The subdirectories will be named with the first four bytes of the quorum key. Each subdirectory will contain a new share and the quorum key it targets. After running the command against an empty `share-dir` and two sharded quorum, keys, the layout would look like: -``` +```sh - share-dir - 04009fd6 - quorum_key.pub From 64e73c24e75210ca631598bb51bb4798212f4322 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Wed, 17 Apr 2024 22:51:02 -0400 Subject: [PATCH 05/31] wip enclave reshard service --- src/images/enclave_reshard/Containerfile | 12 +++ src/images/enclave_reshard/nitro.pcrs | 3 + .../scripts/production-reshard | 32 +++++++ src/images/enclave_reshard/scripts/qos-utils | 83 +++++++++++++++++++ 4 files changed, 130 insertions(+) create mode 100644 src/images/enclave_reshard/Containerfile create mode 100644 src/images/enclave_reshard/nitro.pcrs create mode 100755 src/images/enclave_reshard/scripts/production-reshard create mode 100755 src/images/enclave_reshard/scripts/qos-utils diff --git a/src/images/enclave_reshard/Containerfile b/src/images/enclave_reshard/Containerfile new file mode 100644 index 00000000..5ea1aae1 --- /dev/null +++ b/src/images/enclave_reshard/Containerfile @@ -0,0 +1,12 @@ +FROM stagex/kubectl@sha256:5d0d1ad1790d9eed1b526134739778b0878b07ffc8120e4743190aea931da762 as kubectl +FROM stagex/bash:sx2024.03.0@sha256:d1cbbb56847e6b1e7b879214aa6926b6fdfa210e9b42a2f612a6aea850ddeefc AS bash +FROM stagex/curl:sx2024.03.0@sha256:925222b88f66aeed382b375343eb49ac1904575ab33d78896fd5285cdc709e7f AS curl + +COPY --from=qos_client /qos_client /usr/local/bin/ + +ADD images/enclave_reshard/scripts/* /usr/local/bin/ + +RUN mkdir -p /qos-dist +ADD images/enclave_boot/nitro.pcrs /qos-dist/aws-x86_64.pcrs + +USER 1000:1000 \ No newline at end of file diff --git a/src/images/enclave_reshard/nitro.pcrs b/src/images/enclave_reshard/nitro.pcrs new file mode 100644 index 00000000..3c5e5ab2 --- /dev/null +++ b/src/images/enclave_reshard/nitro.pcrs @@ -0,0 +1,3 @@ +0282aeaec6d1e83cf32e95bbec8888a2f5133a4c9506dfdb5309cc547b8f21fda4e1325beb9e0bc9a8dfb385f11afcca PCR0 +0282aeaec6d1e83cf32e95bbec8888a2f5133a4c9506dfdb5309cc547b8f21fda4e1325beb9e0bc9a8dfb385f11afcca PCR1 +21b9efbc184807662e966d34f390821309eeac6802309798826296bf3e8bec7c10edb30948c90ba67310f7b964fc500a PCR2 \ No newline at end of file diff --git a/src/images/enclave_reshard/scripts/production-reshard b/src/images/enclave_reshard/scripts/production-reshard new file mode 100755 index 00000000..bae122f3 --- /dev/null +++ b/src/images/enclave_reshard/scripts/production-reshard @@ -0,0 +1,32 @@ +#! /bin/bash + +set -euo pipefail + +source ./qos-utils + + +QOS_RELEASE_DIR=/qos-dist +PCR3_PRE_IMAGE_PATH=/etc/enclave/pcr3_preimage +UMP_QUORUM_KEY_PATH=/etc/quorum-keys/ump +SIGNER_QUORUM_KEY_PATH=/etc/quorum-keys/signer +NOTARIZER_QUORUM_KEY_PATH=/etc/quorum-keys/notarizer +EVM_PARSER_QUORUM_KEY_PATH=/etc/quorum-keys/evm-parser +OLD_SHARE_SET_DIR=/etc/enclave-sets/old-share +NEW_SHARE_SET_DIR=/etc/enclave-sets/new-share +RESHARD_INPUT_PATH=/reshard-input + +qos_client generate-reshard-input \ + --qos-release-dir "${QOS_RELEASE_DIR}" \ + --pcr3-preimage-path "${PCR3_PRE_IMAGE_PATH}" \ + --quorum-key-path-multiple "${UMP_QUORUM_KEY_PATH}" \ + --quorum-key-path-multiple "${SIGNER_QUORUM_KEY_PATH}" \ + --quorum-key-path-multiple "${NOTARIZER_QUORUM_KEY_PATH}" \ + --quorum-key-path-multiple "${EVM_PARSER_QUORUM_KEY_PATH}" \ + --old-share-set-dir "${OLD_SHARE_SET_DIR}" \ + --new-share-set-dir "${NEW_SHARE_SET_DIR}" \ + --reshard-input-path "${RESHARD_INPUT_PATH}" + +qos_client boot-reshard \ + --reshard-input-path "${RESHARD_INPUT_PATH}" \ + --host-port "${QOS_PORT}" \ + --host-ip "${QOS_PORT}" diff --git a/src/images/enclave_reshard/scripts/qos-utils b/src/images/enclave_reshard/scripts/qos-utils new file mode 100755 index 00000000..174197e0 --- /dev/null +++ b/src/images/enclave_reshard/scripts/qos-utils @@ -0,0 +1,83 @@ +#! /bin/bash + +: "${LOOP:=}" + +genAppHash() { + sha256sum "${PIVOT_EXECUTABLE}" | awk '{print $1}' > "${PIVOT_HASH_PATH}" +} + +getAttestationDocument() { + local QOS_HOST="${1:?Missing qos host}" + local QOS_PORT="${2:?Missing qos port}" + qos_client \ + get-attestation-doc \ + --host-port "${QOS_PORT}" \ + --host-ip "${QOS_HOST}" \ + --attestation-doc-path "${ATTESTATION_DOC_PATH}" \ + --manifest-envelope-path "${UNUSED_MANIFEST_ENVELOPE_PATH}" +} + +getEnclaveState() { + local QOS_HOST="${1:?Missing qos host}" + local QOS_PORT="${2:?Missing qos port}" + local ENCLAVE_STATE + ENCLAVE_STATE=$(qos_client \ + enclave-status \ + --host-ip "${QOS_HOST}" \ + --host-port "${QOS_PORT}") + local STATUS_REGEX="Enclave phase: ([a-zA-Z]+)" + [[ $ENCLAVE_STATE =~ $STATUS_REGEX ]] && echo "${BASH_REMATCH[1]}" +} + +waitForEnclaveReady() { + while true; do + set -e # allow curl to fail + STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://${QOS_HOST}:8080") || echo "0" + set +e + if [ $STATUS_CODE -eq 200 ]; then + break; + fi + + sleep 1 + done + + while true; do + STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://${QOS_HOST}:${QOS_PORT}/qos/host-health") || echo "0" + if [ $STATUS_CODE -eq 200 ]; then + break; + fi + + sleep 1 + done + + sleep 1 +} + +waitForProvisioned() { + local QOS_HOST="${1:?Missing qos host}" + local QOS_PORT="${2:?Missing qos port}" + + while true; do + local STATUS + STATUS=$(getEnclaveState "$QOS_HOST" "$QOS_PORT") + + case "${STATUS}" in + QuorumKeyProvisioned) + break + ;; + GenesisBooted|WaitingForQuorumShards|WaitingForBootInstruction|WaitingForForwardedKey) + echo "${QOS_HOST}:${QOS_PORT} - In ${STATUS}, waiting" >&2 + ;; + UnrecoverableError) + echo "${QOS_HOST}:${QOS_PORT} - In ${STATUS}, can't progress" >&2 + return 1 + ;; + *) + echo "${QOS_HOST}:${QOS_PORT} - Unknown ProtocolPhase: ${STATUS}" >&2 + return 1 + ;; + esac + + sleep 1 + done +} From ec3c54ff899b86fb2800843eaa9e913afab88353 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Thu, 18 Apr 2024 11:59:58 -0400 Subject: [PATCH 06/31] try getting enclave_reshard --- src/images/enclave_reshard/Containerfile | 1 + src/images/enclave_reshard/scripts/production-reshard | 9 ++++----- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/images/enclave_reshard/Containerfile b/src/images/enclave_reshard/Containerfile index 5ea1aae1..1697c7e9 100644 --- a/src/images/enclave_reshard/Containerfile +++ b/src/images/enclave_reshard/Containerfile @@ -1,6 +1,7 @@ FROM stagex/kubectl@sha256:5d0d1ad1790d9eed1b526134739778b0878b07ffc8120e4743190aea931da762 as kubectl FROM stagex/bash:sx2024.03.0@sha256:d1cbbb56847e6b1e7b879214aa6926b6fdfa210e9b42a2f612a6aea850ddeefc AS bash FROM stagex/curl:sx2024.03.0@sha256:925222b88f66aeed382b375343eb49ac1904575ab33d78896fd5285cdc709e7f AS curl +FROM ghcr.io/tkhq/qos_client as qos_client COPY --from=qos_client /qos_client /usr/local/bin/ diff --git a/src/images/enclave_reshard/scripts/production-reshard b/src/images/enclave_reshard/scripts/production-reshard index bae122f3..66d6a4f6 100755 --- a/src/images/enclave_reshard/scripts/production-reshard +++ b/src/images/enclave_reshard/scripts/production-reshard @@ -4,13 +4,12 @@ set -euo pipefail source ./qos-utils - QOS_RELEASE_DIR=/qos-dist PCR3_PRE_IMAGE_PATH=/etc/enclave/pcr3_preimage -UMP_QUORUM_KEY_PATH=/etc/quorum-keys/ump -SIGNER_QUORUM_KEY_PATH=/etc/quorum-keys/signer -NOTARIZER_QUORUM_KEY_PATH=/etc/quorum-keys/notarizer -EVM_PARSER_QUORUM_KEY_PATH=/etc/quorum-keys/evm-parser +UMP_QUORUM_KEY_PATH=/etc/quorum-keys/ump.pub +SIGNER_QUORUM_KEY_PATH=/etc/quorum-keys/signer.pub +NOTARIZER_QUORUM_KEY_PATH=/etc/quorum-keys/notarizer.pub +EVM_PARSER_QUORUM_KEY_PATH=/etc/quorum-keys/evm-parser.pub OLD_SHARE_SET_DIR=/etc/enclave-sets/old-share NEW_SHARE_SET_DIR=/etc/enclave-sets/new-share RESHARD_INPUT_PATH=/reshard-input From 0f28dd8c8cf146a0d30d09402ad2ffbd3913811f Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Thu, 18 Apr 2024 12:09:18 -0400 Subject: [PATCH 07/31] make sure no toolchain --- .gitmodules | 0 src/toolchain | 1 - 2 files changed, 1 deletion(-) delete mode 100644 .gitmodules delete mode 160000 src/toolchain diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index e69de29b..00000000 diff --git a/src/toolchain b/src/toolchain deleted file mode 160000 index 00ce00c2..00000000 --- a/src/toolchain +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 00ce00c246766b8a2ce09b96cf23b812a04d03a4 From add57ba0bf9e485c3ba0c0a34b461e75c47c0b27 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Thu, 18 Apr 2024 12:16:51 -0400 Subject: [PATCH 08/31] small fix --- src/images/enclave_reshard/Containerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/images/enclave_reshard/Containerfile b/src/images/enclave_reshard/Containerfile index 1697c7e9..d933004c 100644 --- a/src/images/enclave_reshard/Containerfile +++ b/src/images/enclave_reshard/Containerfile @@ -8,6 +8,6 @@ COPY --from=qos_client /qos_client /usr/local/bin/ ADD images/enclave_reshard/scripts/* /usr/local/bin/ RUN mkdir -p /qos-dist -ADD images/enclave_boot/nitro.pcrs /qos-dist/aws-x86_64.pcrs +ADD images/enclave_reshard/nitro.pcrs /qos-dist/aws-x86_64.pcrs USER 1000:1000 \ No newline at end of file From 9987767c307546efc23f897d172a09d2a1d037c6 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Thu, 18 Apr 2024 12:31:10 -0400 Subject: [PATCH 09/31] enclave_reshard builds --- src/images/enclave_reshard/Containerfile | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/src/images/enclave_reshard/Containerfile b/src/images/enclave_reshard/Containerfile index d933004c..4e7488bf 100644 --- a/src/images/enclave_reshard/Containerfile +++ b/src/images/enclave_reshard/Containerfile @@ -1,13 +1,12 @@ -FROM stagex/kubectl@sha256:5d0d1ad1790d9eed1b526134739778b0878b07ffc8120e4743190aea931da762 as kubectl -FROM stagex/bash:sx2024.03.0@sha256:d1cbbb56847e6b1e7b879214aa6926b6fdfa210e9b42a2f612a6aea850ddeefc AS bash -FROM stagex/curl:sx2024.03.0@sha256:925222b88f66aeed382b375343eb49ac1904575ab33d78896fd5285cdc709e7f AS curl -FROM ghcr.io/tkhq/qos_client as qos_client +FROM qos-local/build-base as base -COPY --from=qos_client /qos_client /usr/local/bin/ +COPY --from=ghcr.io/tkhq/qos_client /qos_client /usr/local/bin/ +COPY --from=stagex/curl . / +COPY --from=stagex/bash . / +COPY --from=stagex/kubectl . / ADD images/enclave_reshard/scripts/* /usr/local/bin/ -RUN mkdir -p /qos-dist ADD images/enclave_reshard/nitro.pcrs /qos-dist/aws-x86_64.pcrs USER 1000:1000 \ No newline at end of file From 21046c77c1e3a94bc7e3dd80ac3e781c3df581eb Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Thu, 18 Apr 2024 12:45:41 -0400 Subject: [PATCH 10/31] add enclave_reshard target --- .github/workflows/artifacts.yml | 106 +++++++++++++++++++++++++++++++- 1 file changed, 105 insertions(+), 1 deletion(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 2c4af331..1a92a145 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -16,6 +16,12 @@ jobs: # This is charged by the minute, so if you want to reduce cost change back to `runs-on: ubuntu-latest` runs-on: group: ubuntu-runners + strategy: + matrix: + include: + - target: qos_client.tar + - target: qos_host.tar + - target: qos_enclave.tar steps: - name: Checkout sources uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -24,4 +30,102 @@ jobs: - name: Run `make` shell: 'script -q -e -c "bash {0}"' run: | - make -j$(nproc) + make out/${{ matrix.target }} + - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: ${{ matrix.target }} + path: out/${{ matrix.target }} + retention-days: 1 + + upload_to_ecr: + name: Upload toolchain artifacts to ECR + # We use a special group that is configured to use github largest runner instance + # This is charged by the minute, so if you want to reduce cost change back to `runs-on: ubuntu-latest` + runs-on: + group: ubuntu-runners + needs: + - build + permissions: + id-token: write + contents: read + steps: + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + with: + aws-region: us-east-1 + role-to-assume: arn:aws:iam::799078726966:role/github-qos + + - name: Login to Amazon ECR + id: login-ecr + uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 + + - name: Download Artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + + - name: Upload images to ECR + env: + images: >- + qos_client + qos_enclave + qos_host + enclave_reshard + tags: >- + ${{ github.ref == format('refs/heads/{0}', 'main') && 'latest' || '' }} + ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.number) || '' }} + ${{ github.event_name == 'push' && github.ref_name || '' }} + run: | + skopeo login \ + --username "${{ steps.login-ecr.outputs.docker_username_799078726966_dkr_ecr_us_east_1_amazonaws_com }}" \ + --password "${{ steps.login-ecr.outputs.docker_password_799078726966_dkr_ecr_us_east_1_amazonaws_com }}" \ + ${{ steps.login-ecr.outputs.registry }} + for image in ${images}; do + skopeo copy --all --dest-decompress \ + "oci-archive:/home/runner/work/qos/qos/${image}.tar/${image}.tar" \ + "docker://${{ steps.login-ecr.outputs.registry }}/tkhq/${image}:sha-${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" + for tag in ${tags}; do + skopeo copy --all \ + "docker://${{ steps.login-ecr.outputs.registry }}/tkhq/${image}:sha-${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" \ + "docker://${{ steps.login-ecr.outputs.registry }}/tkhq/${image}:${tag}" + done + done + + upload_to_ghcr: + name: Upload toolchain artifacts to GHCR + # We use a special group that is configured to use github largest runner instance + # This is charged by the minute, so if you want to reduce cost change back to `runs-on: ubuntu-latest` + runs-on: + group: ubuntu-runners + needs: + - build + permissions: + contents: read + packages: write + steps: + - name: Download Artifacts + uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + - name: Upload images to GHCR + env: + images: >- + qos_client + qos_enclave + qos_host + tags: >- + ${{ github.ref == format('refs/heads/{0}', 'main') && 'latest' || '' }} + ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.number) || '' }} + ${{ github.event_name == 'push' && github.ref_name || '' }} + run: | + skopeo login \ + --username "${{ github.actor }}" \ + --password "${{ secrets.GITHUB_TOKEN }}" \ + ghcr.io + for image in ${images}; do + skopeo copy --all --dest-decompress \ + "oci-archive:/home/runner/work/qos/qos/${image}.tar/${image}.tar" \ + "docker://ghcr.io/tkhq/${image}:sha-${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" + for tag in ${tags}; do + skopeo copy --all \ + "docker://ghcr.io/tkhq/${image}:sha-${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" \ + "docker://ghcr.io/tkhq/${image}:${tag}" + done + done From c9fd19d8f6c56f02977ba3a6da3c6dbfdf29c380 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Thu, 18 Apr 2024 12:50:18 -0400 Subject: [PATCH 11/31] try fix perms issue --- .github/workflows/artifacts.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 1a92a145..5a3bb776 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -22,6 +22,10 @@ jobs: - target: qos_client.tar - target: qos_host.tar - target: qos_enclave.tar + permissions: + id-token: write + contents: read + packages: write steps: - name: Checkout sources uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 @@ -30,7 +34,9 @@ jobs: - name: Run `make` shell: 'script -q -e -c "bash {0}"' run: | - make out/${{ matrix.target }} + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + make -j$(nproc) out/${{ matrix.target }} - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: name: ${{ matrix.target }} From 54f56e3736ee3c4a0de68a88814e4d5bb9276b96 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Thu, 18 Apr 2024 18:57:20 -0400 Subject: [PATCH 12/31] remove enclave_reshard image (pivoting to mono) --- src/images/enclave_reshard/Containerfile | 12 --- src/images/enclave_reshard/nitro.pcrs | 3 - .../scripts/production-reshard | 31 ------- src/images/enclave_reshard/scripts/qos-utils | 83 ------------------- 4 files changed, 129 deletions(-) delete mode 100644 src/images/enclave_reshard/Containerfile delete mode 100644 src/images/enclave_reshard/nitro.pcrs delete mode 100755 src/images/enclave_reshard/scripts/production-reshard delete mode 100755 src/images/enclave_reshard/scripts/qos-utils diff --git a/src/images/enclave_reshard/Containerfile b/src/images/enclave_reshard/Containerfile deleted file mode 100644 index 4e7488bf..00000000 --- a/src/images/enclave_reshard/Containerfile +++ /dev/null @@ -1,12 +0,0 @@ -FROM qos-local/build-base as base - -COPY --from=ghcr.io/tkhq/qos_client /qos_client /usr/local/bin/ -COPY --from=stagex/curl . / -COPY --from=stagex/bash . / -COPY --from=stagex/kubectl . / - -ADD images/enclave_reshard/scripts/* /usr/local/bin/ - -ADD images/enclave_reshard/nitro.pcrs /qos-dist/aws-x86_64.pcrs - -USER 1000:1000 \ No newline at end of file diff --git a/src/images/enclave_reshard/nitro.pcrs b/src/images/enclave_reshard/nitro.pcrs deleted file mode 100644 index 3c5e5ab2..00000000 --- a/src/images/enclave_reshard/nitro.pcrs +++ /dev/null @@ -1,3 +0,0 @@ -0282aeaec6d1e83cf32e95bbec8888a2f5133a4c9506dfdb5309cc547b8f21fda4e1325beb9e0bc9a8dfb385f11afcca PCR0 -0282aeaec6d1e83cf32e95bbec8888a2f5133a4c9506dfdb5309cc547b8f21fda4e1325beb9e0bc9a8dfb385f11afcca PCR1 -21b9efbc184807662e966d34f390821309eeac6802309798826296bf3e8bec7c10edb30948c90ba67310f7b964fc500a PCR2 \ No newline at end of file diff --git a/src/images/enclave_reshard/scripts/production-reshard b/src/images/enclave_reshard/scripts/production-reshard deleted file mode 100755 index 66d6a4f6..00000000 --- a/src/images/enclave_reshard/scripts/production-reshard +++ /dev/null @@ -1,31 +0,0 @@ -#! /bin/bash - -set -euo pipefail - -source ./qos-utils - -QOS_RELEASE_DIR=/qos-dist -PCR3_PRE_IMAGE_PATH=/etc/enclave/pcr3_preimage -UMP_QUORUM_KEY_PATH=/etc/quorum-keys/ump.pub -SIGNER_QUORUM_KEY_PATH=/etc/quorum-keys/signer.pub -NOTARIZER_QUORUM_KEY_PATH=/etc/quorum-keys/notarizer.pub -EVM_PARSER_QUORUM_KEY_PATH=/etc/quorum-keys/evm-parser.pub -OLD_SHARE_SET_DIR=/etc/enclave-sets/old-share -NEW_SHARE_SET_DIR=/etc/enclave-sets/new-share -RESHARD_INPUT_PATH=/reshard-input - -qos_client generate-reshard-input \ - --qos-release-dir "${QOS_RELEASE_DIR}" \ - --pcr3-preimage-path "${PCR3_PRE_IMAGE_PATH}" \ - --quorum-key-path-multiple "${UMP_QUORUM_KEY_PATH}" \ - --quorum-key-path-multiple "${SIGNER_QUORUM_KEY_PATH}" \ - --quorum-key-path-multiple "${NOTARIZER_QUORUM_KEY_PATH}" \ - --quorum-key-path-multiple "${EVM_PARSER_QUORUM_KEY_PATH}" \ - --old-share-set-dir "${OLD_SHARE_SET_DIR}" \ - --new-share-set-dir "${NEW_SHARE_SET_DIR}" \ - --reshard-input-path "${RESHARD_INPUT_PATH}" - -qos_client boot-reshard \ - --reshard-input-path "${RESHARD_INPUT_PATH}" \ - --host-port "${QOS_PORT}" \ - --host-ip "${QOS_PORT}" diff --git a/src/images/enclave_reshard/scripts/qos-utils b/src/images/enclave_reshard/scripts/qos-utils deleted file mode 100755 index 174197e0..00000000 --- a/src/images/enclave_reshard/scripts/qos-utils +++ /dev/null @@ -1,83 +0,0 @@ -#! /bin/bash - -: "${LOOP:=}" - -genAppHash() { - sha256sum "${PIVOT_EXECUTABLE}" | awk '{print $1}' > "${PIVOT_HASH_PATH}" -} - -getAttestationDocument() { - local QOS_HOST="${1:?Missing qos host}" - local QOS_PORT="${2:?Missing qos port}" - qos_client \ - get-attestation-doc \ - --host-port "${QOS_PORT}" \ - --host-ip "${QOS_HOST}" \ - --attestation-doc-path "${ATTESTATION_DOC_PATH}" \ - --manifest-envelope-path "${UNUSED_MANIFEST_ENVELOPE_PATH}" -} - -getEnclaveState() { - local QOS_HOST="${1:?Missing qos host}" - local QOS_PORT="${2:?Missing qos port}" - local ENCLAVE_STATE - ENCLAVE_STATE=$(qos_client \ - enclave-status \ - --host-ip "${QOS_HOST}" \ - --host-port "${QOS_PORT}") - local STATUS_REGEX="Enclave phase: ([a-zA-Z]+)" - [[ $ENCLAVE_STATE =~ $STATUS_REGEX ]] && echo "${BASH_REMATCH[1]}" -} - -waitForEnclaveReady() { - while true; do - set -e # allow curl to fail - STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://${QOS_HOST}:8080") || echo "0" - set +e - if [ $STATUS_CODE -eq 200 ]; then - break; - fi - - sleep 1 - done - - while true; do - STATUS_CODE=$(curl -s -o /dev/null -w "%{http_code}" "http://${QOS_HOST}:${QOS_PORT}/qos/host-health") || echo "0" - if [ $STATUS_CODE -eq 200 ]; then - break; - fi - - sleep 1 - done - - sleep 1 -} - -waitForProvisioned() { - local QOS_HOST="${1:?Missing qos host}" - local QOS_PORT="${2:?Missing qos port}" - - while true; do - local STATUS - STATUS=$(getEnclaveState "$QOS_HOST" "$QOS_PORT") - - case "${STATUS}" in - QuorumKeyProvisioned) - break - ;; - GenesisBooted|WaitingForQuorumShards|WaitingForBootInstruction|WaitingForForwardedKey) - echo "${QOS_HOST}:${QOS_PORT} - In ${STATUS}, waiting" >&2 - ;; - UnrecoverableError) - echo "${QOS_HOST}:${QOS_PORT} - In ${STATUS}, can't progress" >&2 - return 1 - ;; - *) - echo "${QOS_HOST}:${QOS_PORT} - Unknown ProtocolPhase: ${STATUS}" >&2 - return 1 - ;; - esac - - sleep 1 - done -} From 7b8925f8967cf08bb002169d169e8a6cecd88d95 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Fri, 19 Apr 2024 11:44:39 -0600 Subject: [PATCH 13/31] revert ci changes --- .github/workflows/artifacts.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index 5a3bb776..aeb5775d 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -34,8 +34,6 @@ jobs: - name: Run `make` shell: 'script -q -e -c "bash {0}"' run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - make -j$(nproc) out/${{ matrix.target }} - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 with: From e08bb9d7e74154760f4fe6461393a2c84a4ae69f Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Fri, 19 Apr 2024 15:08:04 -0600 Subject: [PATCH 14/31] fix some comments --- src/qos_core/src/protocol/msg.rs | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/qos_core/src/protocol/msg.rs b/src/qos_core/src/protocol/msg.rs index 1df262ff..8548db5c 100644 --- a/src/qos_core/src/protocol/msg.rs +++ b/src/qos_core/src/protocol/msg.rs @@ -148,8 +148,8 @@ pub enum ProtocolMsg { }, /// Response to [`Self::BootReshardRequest`]. BootReshardResponse { - /// Should be `[NsmResponse::Attestation`]. `user_data` is the the - /// reshard_input + /// Should be `[NsmResponse::Attestation`]. The `user_data` field of + /// of the attestation document is the qos hash of [`ReshardInput`]. nsm_response: NsmResponse, }, @@ -157,8 +157,8 @@ pub enum ProtocolMsg { ReshardAttestationDocRequest, /// Response to [`Self::ReshardAttestationDocRequest`] ReshardAttestationDocResponse { - /// Should be `[NsmResponse::Attestation`]. `user_data` is the the - /// reshard_input + /// Should be `[NsmResponse::Attestation`]. The `user_data` field of + /// of the attestation document is the qos hash of [`ReshardInput`]. nsm_response: NsmResponse, /// The reshard parameters this enclave is setup for. reshard_input: ReshardInput, From dfc8996ad9df6e3e1e4a6c9f7ca89f7f8bf446f9 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Fri, 19 Apr 2024 15:11:28 -0600 Subject: [PATCH 15/31] another small comment fix --- src/qos_core/src/protocol/error.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/qos_core/src/protocol/error.rs b/src/qos_core/src/protocol/error.rs index 7eda6cd8..a5c3964b 100644 --- a/src/qos_core/src/protocol/error.rs +++ b/src/qos_core/src/protocol/error.rs @@ -148,7 +148,7 @@ pub enum ProtocolError { MissingReshardOutput, /// The same member was in the share set multiple times. DuplicateNewShareSetMember, - /// The share set member posted more or less shares then the number of + /// The share set member posted more or less shares than the number of /// quorum keys targeted for reconstruction. ShareCountDoesNotMatchExpectedQuorumKeyCount, /// Could not decrypt the share with the ephemeral key. From 8d1eca20d57f6454b9a6914787c42ac566dd96a5 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Fri, 19 Apr 2024 16:35:08 -0600 Subject: [PATCH 16/31] Update reshard service --- src/qos_core/src/protocol/error.rs | 7 ++++--- src/qos_crypto/src/n_choose_k.rs | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/qos_core/src/protocol/error.rs b/src/qos_core/src/protocol/error.rs index a5c3964b..18e2f922 100644 --- a/src/qos_core/src/protocol/error.rs +++ b/src/qos_core/src/protocol/error.rs @@ -141,10 +141,11 @@ pub enum ProtocolError { /// The new manifest was different from the old manifest when we expected /// them to be the same because they have the same nonce DifferentManifest, - /// Expected to have [crate::protocol::services::reshard::ReshardInput] in enclave state, but it was not found. + /// Expected to have [crate::protocol::services::reshard::ReshardInput] in + /// enclave state, but it was not found. MissingReshardInput, - /// Expected to have [crate::protocol::services::reshard::ReshardOutput] in enclave state, but it was not - /// found. + /// Expected to have [crate::protocol::services::reshard::ReshardOutput] in + /// enclave state, but it was not found. MissingReshardOutput, /// The same member was in the share set multiple times. DuplicateNewShareSetMember, diff --git a/src/qos_crypto/src/n_choose_k.rs b/src/qos_crypto/src/n_choose_k.rs index add5b706..91fe7c42 100644 --- a/src/qos_crypto/src/n_choose_k.rs +++ b/src/qos_crypto/src/n_choose_k.rs @@ -44,7 +44,7 @@ pub fn combinations(input: &[T], k: usize) -> Vec> { // the input let combination: Vec<_> = indices.iter().map(|&i| input[i].clone()).collect(); - combos.push(combination.clone()); + combos.push(combination); let mut i = k; while i > 1 && indices[i - 1] == n - k + i - 1 { From 69469a9bfe9dfcf392cd090463fff5ab0c6ec0bf Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Fri, 19 Apr 2024 18:11:26 -0600 Subject: [PATCH 17/31] small adjustment to default dev port --- src/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Makefile b/src/Makefile index 5b967a6d..0b90aa58 100644 --- a/src/Makefile +++ b/src/Makefile @@ -48,7 +48,7 @@ local-host: cargo run --bin qos_host \ -- \ --host-ip 127.0.0.1 \ - --host-port 3000 \ + --host-port 3001 \ --usock ./dev.sock .PHONY: vm-host From 3c0cfad911d2f520ce645a4953387a0840df5138 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Sat, 20 Apr 2024 22:40:23 -0600 Subject: [PATCH 18/31] ensure deterministic reshard input --- src/qos_client/src/cli/services.rs | 3 ++- src/qos_core/src/protocol/services/attestation.rs | 3 ++- src/qos_core/src/protocol/services/reshard.rs | 6 ++++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/src/qos_client/src/cli/services.rs b/src/qos_client/src/cli/services.rs index 0ecfa19f..67aae96a 100644 --- a/src/qos_client/src/cli/services.rs +++ b/src/qos_client/src/cli/services.rs @@ -1552,7 +1552,8 @@ pub(crate) fn reshard_re_encrypt_share( unsafe_auto_confirm, }: ReshardReEncryptShareArgs, ) -> Result<(), Error> { - let reshard_input = read_reshard_input(reshard_input_path)?; + let mut reshard_input = read_reshard_input(reshard_input_path)?; + reshard_input.deterministic(); let attestation_doc = read_attestation_doc(&attestation_doc_path, unsafe_skip_attestation)?; let mut new_share_set = get_share_set(&new_share_set_dir); diff --git a/src/qos_core/src/protocol/services/attestation.rs b/src/qos_core/src/protocol/services/attestation.rs index d3a402d4..df261a6a 100644 --- a/src/qos_core/src/protocol/services/attestation.rs +++ b/src/qos_core/src/protocol/services/attestation.rs @@ -27,10 +27,11 @@ pub(in crate::protocol) fn reshard_attestation_doc( ) -> Result { let ephemeral_public_key = state.handles.get_ephemeral_key()?.public_key().to_bytes(); - let reshard_input = state + let mut reshard_input = state .reshard_input .clone() .ok_or(ProtocolError::MissingReshardInput)?; + reshard_input.deterministic(); Ok(get_post_boot_attestation_doc( &*state.attestor, diff --git a/src/qos_core/src/protocol/services/reshard.rs b/src/qos_core/src/protocol/services/reshard.rs index fce811ca..a2ea8c4a 100644 --- a/src/qos_core/src/protocol/services/reshard.rs +++ b/src/qos_core/src/protocol/services/reshard.rs @@ -85,7 +85,8 @@ pub struct ReshardInput { } impl ReshardInput { - fn deterministic(&mut self) { + /// Make sure reshard input is deterministic + pub fn deterministic(&mut self) { self.quorum_keys.sort(); } @@ -249,12 +250,13 @@ pub(in crate::protocol) fn reshard_provision( input: ReshardProvisionInput, state: &mut ProtocolState, ) -> Result { - let reshard_input = state + let mut reshard_input = state .reshard_input .as_ref() .ok_or(ProtocolError::MissingReshardInput)? .clone(); + reshard_input.deterministic(); input.approval.verify(&reshard_input.qos_hash())?; if !reshard_input.old_share_set.members.contains(&input.approval.member) { From 3ab39ca8f81dbddbee47a55d6f64fe36d81ec0a6 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Sat, 20 Apr 2024 22:43:47 -0600 Subject: [PATCH 19/31] sort share set members --- src/qos_core/src/protocol/services/reshard.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/qos_core/src/protocol/services/reshard.rs b/src/qos_core/src/protocol/services/reshard.rs index a2ea8c4a..c07b2777 100644 --- a/src/qos_core/src/protocol/services/reshard.rs +++ b/src/qos_core/src/protocol/services/reshard.rs @@ -88,6 +88,8 @@ impl ReshardInput { /// Make sure reshard input is deterministic pub fn deterministic(&mut self) { self.quorum_keys.sort(); + self.new_share_set.members.sort(); + self.old_share_set.members.sort(); } fn validate(&mut self) -> Result<(), ProtocolError> { From 17b44ff5613a4b4cd29e5a3399a7e837e43e23bd Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Sat, 20 Apr 2024 22:49:21 -0600 Subject: [PATCH 20/31] ensure reshard input is deterministic when reading in --- src/qos_client/src/cli/services.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/qos_client/src/cli/services.rs b/src/qos_client/src/cli/services.rs index 67aae96a..64fce482 100644 --- a/src/qos_client/src/cli/services.rs +++ b/src/qos_client/src/cli/services.rs @@ -816,13 +816,15 @@ pub fn generate_reshard_input( let old_share_set = get_share_set(old_share_set_dir); let new_share_set = get_share_set(new_share_set_dir); - let reshard_input = ReshardInput { + let mut reshard_input = ReshardInput { quorum_keys, new_share_set, old_share_set, enclave: nitro_config, }; + reshard_input.deterministic(); + write_json_with_msg( reshard_input_path.as_ref(), &reshard_input, @@ -2461,8 +2463,9 @@ fn read_reshard_input(file: String) -> Result { error: e.to_string(), })?; - let reshard_input: ReshardInput = serde_json::from_slice(&buf) + let mut reshard_input: ReshardInput = serde_json::from_slice(&buf) .map_err(|e| Error::FileDidNotHaveValidReshardInput(e.to_string()))?; + reshard_input.deterministic(); Ok(reshard_input) } From a33541759abcaaec3b8e2dde1acb3a852faf25f8 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Tue, 23 Apr 2024 13:50:43 -0400 Subject: [PATCH 21/31] update ci --- .github/workflows/artifacts.yml | 141 ++++++++++++-------------------- 1 file changed, 53 insertions(+), 88 deletions(-) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/artifacts.yml index aeb5775d..407fdebe 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/artifacts.yml @@ -1,4 +1,4 @@ -name: artifacts-build +name: stagex-build on: push: @@ -11,7 +11,7 @@ on: jobs: build: - name: build artifacts + name: stagex build+push # We use a special group that is configured to use github largest runner instance # This is charged by the minute, so if you want to reduce cost change back to `runs-on: ubuntu-latest` runs-on: @@ -19,9 +19,9 @@ jobs: strategy: matrix: include: - - target: qos_client.tar - - target: qos_host.tar - - target: qos_enclave.tar + - target: qos_client + - target: qos_host + - target: qos_enclave permissions: id-token: write contents: read @@ -29,30 +29,39 @@ jobs: steps: - name: Checkout sources uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - name: Setup Docker uses: ./.github/actions/docker-setup + - name: Run `make` shell: 'script -q -e -c "bash {0}"' run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin make -j$(nproc) out/${{ matrix.target }} - - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 - with: - name: ${{ matrix.target }} - path: out/${{ matrix.target }} - retention-days: 1 - upload_to_ecr: - name: Upload toolchain artifacts to ECR - # We use a special group that is configured to use github largest runner instance - # This is charged by the minute, so if you want to reduce cost change back to `runs-on: ubuntu-latest` - runs-on: - group: ubuntu-runners - needs: - - build - permissions: - id-token: write - contents: read - steps: + - name: Run `make ${{ matrix.target }}` + shell: 'script -q -e -c "bash {0}"' + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + make -j$(nproc) out/${{ matrix.target }}/index.json + + - name: upload to GHCR + run: | + echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin + + env -C out/${{ matrix.target }} tar -cf - . | docker load + + for tag in ${tags}; do + docker tag "qos-local/${{ matrix.target }}:latest" "ghcr.io/tkhq/${{ matrix.target }}:${tag}" + done + + docker image push --all-tags ghcr.io/tkhq/${{ matrix.target }} + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 + with: + aws-region: us-east-1 + role-to-assume: arn:aws:iam::799078726966:role/github-mono - name: Configure AWS Credentials uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 @@ -64,72 +73,28 @@ jobs: id: login-ecr uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - - name: Download Artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - - - name: Upload images to ECR - env: - images: >- - qos_client - qos_enclave - qos_host - enclave_reshard - tags: >- - ${{ github.ref == format('refs/heads/{0}', 'main') && 'latest' || '' }} - ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.number) || '' }} - ${{ github.event_name == 'push' && github.ref_name || '' }} + - name: Upload to ECR + shell: 'script -q -e -c "bash {0}"' run: | - skopeo login \ - --username "${{ steps.login-ecr.outputs.docker_username_799078726966_dkr_ecr_us_east_1_amazonaws_com }}" \ - --password "${{ steps.login-ecr.outputs.docker_password_799078726966_dkr_ecr_us_east_1_amazonaws_com }}" \ - ${{ steps.login-ecr.outputs.registry }} - for image in ${images}; do - skopeo copy --all --dest-decompress \ - "oci-archive:/home/runner/work/qos/qos/${image}.tar/${image}.tar" \ - "docker://${{ steps.login-ecr.outputs.registry }}/tkhq/${image}:sha-${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" - for tag in ${tags}; do - skopeo copy --all \ - "docker://${{ steps.login-ecr.outputs.registry }}/tkhq/${image}:sha-${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" \ - "docker://${{ steps.login-ecr.outputs.registry }}/tkhq/${image}:${tag}" - done - done + echo "${{ steps.login-ecr.outputs.docker_password_799078726966_dkr_ecr_us_east_1_amazonaws_com }}" | \ + docker login \ + ${{ steps.login-ecr.outputs.registry }} \ + -u "${{ steps.login-ecr.outputs.docker_username_799078726966_dkr_ecr_us_east_1_amazonaws_com }}" \ + --password-stdin - upload_to_ghcr: - name: Upload toolchain artifacts to GHCR - # We use a special group that is configured to use github largest runner instance - # This is charged by the minute, so if you want to reduce cost change back to `runs-on: ubuntu-latest` - runs-on: - group: ubuntu-runners - needs: - - build - permissions: - contents: read - packages: write - steps: - - name: Download Artifacts - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 - - name: Upload images to GHCR - env: - images: >- - qos_client - qos_enclave - qos_host - tags: >- - ${{ github.ref == format('refs/heads/{0}', 'main') && 'latest' || '' }} - ${{ github.event_name == 'pull_request' && format('pr-{0}', github.event.number) || '' }} - ${{ github.event_name == 'push' && github.ref_name || '' }} - run: | - skopeo login \ - --username "${{ github.actor }}" \ - --password "${{ secrets.GITHUB_TOKEN }}" \ - ghcr.io - for image in ${images}; do - skopeo copy --all --dest-decompress \ - "oci-archive:/home/runner/work/qos/qos/${image}.tar/${image}.tar" \ - "docker://ghcr.io/tkhq/${image}:sha-${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" - for tag in ${tags}; do - skopeo copy --all \ - "docker://ghcr.io/tkhq/${image}:sha-${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" \ - "docker://ghcr.io/tkhq/${image}:${tag}" - done + export BASE_IMAGE_NAME="${{ steps.login-ecr.outputs.registry }}/tkhq/${{ matrix.target }}" + export IMAGE_NAME="${BASE_IMAGE_NAME}:sha-${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" + export DIGEST_FILE=/tmp/image-digest-${{ matrix.target }}.sha256 + + cat out/${{ matrix.target }}/index.json | jq -r .manifests[].digest > "${DIGEST_FILE}" + + docker tag "qos-local/${{ matrix.target }}:latest" "$IMAGE_NAME" + + for tag in ${tags}; do + docker tag "$IMAGE_NAME" "$BASE_IMAGE_NAME:${tag}" done + + docker image push --all-tags "$BASE_IMAGE_NAME" + + echo "Uploaded image $IMAGE_NAME (SHA-256 digest: $(cat $DIGEST_FILE))" + From 5342ff3ac0f31c3c412bd441880469753b5bc5d6 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Tue, 23 Apr 2024 14:02:00 -0400 Subject: [PATCH 22/31] address comments; fix tests --- src/qos_client/src/cli/mod.rs | 8 ++++---- src/qos_client/src/cli/services.rs | 9 +++++---- src/qos_core/src/protocol/services/reshard.rs | 5 ++++- 3 files changed, 13 insertions(+), 9 deletions(-) diff --git a/src/qos_client/src/cli/mod.rs b/src/qos_client/src/cli/mod.rs index e9e6a24d..c164a767 100644 --- a/src/qos_client/src/cli/mod.rs +++ b/src/qos_client/src/cli/mod.rs @@ -430,7 +430,7 @@ impl Command { fn share_dir_token() -> Token { Token::new( SHARE_DIR, - "directory to read/write subdirectories that contain quorum key and your associated share.", + "Directory to read/write subdirectories that contain quorum key and your associated share.", ) .takes_value(true) .required(true) @@ -631,7 +631,7 @@ impl Command { fn quorum_key_path_multiple_token() -> Token { Token::new( QUORUM_KEY_PATH_MULTIPLE, - "use multiple times to specify multiple quorum public key files.", + "Use multiple times to specify multiple quorum public key files.", ) .required(true) .takes_value(true) @@ -640,7 +640,7 @@ impl Command { fn quorum_share_dir_multiple_token() -> Token { Token::new( QUORUM_SHARE_DIR_MULTIPLE, - "path to directory with just your share and the associated quorum key." + "Path to directory with just your share and the associated quorum key." ) .required(true) .takes_value(true) @@ -649,7 +649,7 @@ impl Command { fn provision_input_path_token() -> Token { Token::new( PROVISION_INPUT_PATH, - "path to file to read/write ReshardProvisionInput.", + "Path to file to read/write ReshardProvisionInput.", ) .required(true) .takes_value(true) diff --git a/src/qos_client/src/cli/services.rs b/src/qos_client/src/cli/services.rs index 64fce482..7cd1c856 100644 --- a/src/qos_client/src/cli/services.rs +++ b/src/qos_client/src/cli/services.rs @@ -151,11 +151,12 @@ pub enum Error { /// Given quorum key seed does not match the hash of the expected quorum /// key seed. SecretDoesNotMatch, - FileDidNotHaveValidReshardInput(String), + /// The contents of the file could not be deserialized as `ReshardInput`. + FailedToDeserializeReshardInput(String), /// Could not read the file with the reshard output FailedToReadReshardOutput(std::io::Error), /// Could not serialize the reshard output in the file - FileDidNotHaveReshardOutput(String), + FailedToDeserializeReshardOutput(String), /// Could not find key in new genesis member outputs KeyNotInNewShareSet, /// Failed to write to the file specified for the encrypted share. @@ -2464,7 +2465,7 @@ fn read_reshard_input(file: String) -> Result { })?; let mut reshard_input: ReshardInput = serde_json::from_slice(&buf) - .map_err(|e| Error::FileDidNotHaveValidReshardInput(e.to_string()))?; + .map_err(|e| Error::FailedToDeserializeReshardInput(e.to_string()))?; reshard_input.deterministic(); Ok(reshard_input) @@ -2477,7 +2478,7 @@ fn read_reshard_output(file: String) -> Result { })?; serde_json::from_slice(&buf) - .map_err(|e| Error::FileDidNotHaveReshardOutput(e.to_string())) + .map_err(|e| Error::FailedToDeserializeReshardOutput(e.to_string())) } fn read_attestation_approval>( diff --git a/src/qos_core/src/protocol/services/reshard.rs b/src/qos_core/src/protocol/services/reshard.rs index c07b2777..a212f552 100644 --- a/src/qos_core/src/protocol/services/reshard.rs +++ b/src/qos_core/src/protocol/services/reshard.rs @@ -393,7 +393,7 @@ mod tests { }) .collect(); - let reshard_input = ReshardInput { + let mut reshard_input = ReshardInput { quorum_keys, new_share_set: ShareSet { threshold: 2, @@ -414,6 +414,9 @@ mod tests { }, }; + // SUPER IMPORTANT: we need to ensure we always sign the correct hash. + reshard_input.deterministic(); + let provision_inputs: Vec<_> = old_members .into_iter() .enumerate() From b8878109ee45737674fcb65649711014c15cadfb Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Tue, 23 Apr 2024 14:14:09 -0400 Subject: [PATCH 23/31] review comments; try fix CI --- .github/workflows/{artifacts.yml => stagex.yml} | 6 ------ src/qos_client/src/cli/services.rs | 4 ++-- src/qos_core/src/protocol/services/reshard.rs | 1 - 3 files changed, 2 insertions(+), 9 deletions(-) rename .github/workflows/{artifacts.yml => stagex.yml} (93%) diff --git a/.github/workflows/artifacts.yml b/.github/workflows/stagex.yml similarity index 93% rename from .github/workflows/artifacts.yml rename to .github/workflows/stagex.yml index 407fdebe..d5291486 100644 --- a/.github/workflows/artifacts.yml +++ b/.github/workflows/stagex.yml @@ -33,12 +33,6 @@ jobs: - name: Setup Docker uses: ./.github/actions/docker-setup - - name: Run `make` - shell: 'script -q -e -c "bash {0}"' - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - make -j$(nproc) out/${{ matrix.target }} - - name: Run `make ${{ matrix.target }}` shell: 'script -q -e -c "bash {0}"' run: | diff --git a/src/qos_client/src/cli/services.rs b/src/qos_client/src/cli/services.rs index 7cd1c856..fca937e8 100644 --- a/src/qos_client/src/cli/services.rs +++ b/src/qos_client/src/cli/services.rs @@ -1858,8 +1858,8 @@ pub(crate) fn verify_reshard_output( panic!("failed to create dir {:?}: {}", dir_path, e) }) .unwrap(); - let quorum_key_path = dir_path.clone().join("quorum_key.pub"); - let share_path = dir_path.join(format!("{}.share", alias)); + let quorum_key_path = dir_path.clone().join(format!("quorum_key.{}", PUB_EXT)); + let share_path = dir_path.join(format!("{}.{}", alias, SHARE_EXT)); fs::write(&quorum_key_path, qos_hex::encode(&quorum_key)).map_err( |e| Error::FailedToWrite { diff --git a/src/qos_core/src/protocol/services/reshard.rs b/src/qos_core/src/protocol/services/reshard.rs index a212f552..959c9a3b 100644 --- a/src/qos_core/src/protocol/services/reshard.rs +++ b/src/qos_core/src/protocol/services/reshard.rs @@ -292,7 +292,6 @@ pub(in crate::protocol) fn reshard_provision( // Now, lets create the new shards let member_outputs = zip(shares, reshard_input.new_share_set.members.iter().cloned()) - // .map(|(share, share_set_member)| -> Result { .map(|(share, share_set_member)| -> Result { let personal_pub = P256Public::from_bytes(&share_set_member.pub_key)?; let encrypted_quorum_key_share = personal_pub.encrypt(&share)?; From 1608d8c54a07179cc653cbd3ad41a42a95158898 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Tue, 23 Apr 2024 14:17:39 -0400 Subject: [PATCH 24/31] lint --- src/qos_client/src/cli/services.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/qos_client/src/cli/services.rs b/src/qos_client/src/cli/services.rs index fca937e8..ff1e44ef 100644 --- a/src/qos_client/src/cli/services.rs +++ b/src/qos_client/src/cli/services.rs @@ -1858,7 +1858,8 @@ pub(crate) fn verify_reshard_output( panic!("failed to create dir {:?}: {}", dir_path, e) }) .unwrap(); - let quorum_key_path = dir_path.clone().join(format!("quorum_key.{}", PUB_EXT)); + let quorum_key_path = + dir_path.clone().join(format!("quorum_key.{}", PUB_EXT)); let share_path = dir_path.join(format!("{}.{}", alias, SHARE_EXT)); fs::write(&quorum_key_path, qos_hex::encode(&quorum_key)).map_err( From 7427f33c3dfadcbd4e98d10116ddb6d42df9edf1 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Tue, 23 Apr 2024 14:34:43 -0400 Subject: [PATCH 25/31] update common image tags --- src/images/common/Containerfile | 48 ++++++++++++++++----------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/src/images/common/Containerfile b/src/images/common/Containerfile index b723b21a..5096487c 100644 --- a/src/images/common/Containerfile +++ b/src/images/common/Containerfile @@ -1,27 +1,27 @@ -FROM stagex/bash:sx2024.03.0@sha256:d1cbbb56847e6b1e7b879214aa6926b6fdfa210e9b42a2f612a6aea850ddeefc AS bash -FROM stagex/binutils:sx2024.03.0@sha256:3af41227e1fe6a8f9b3df9916ef4876840f33eaa172168e1db1d8f457ba011d5 AS binutils -FROM stagex/ca-certificates:sx2024.03.0@sha256:6746d2d203be3455bfc5ffd5a051c8edb73ecfd7be77c3da5a2973003a30794f AS ca-certificates -FROM stagex/coreutils:sx2024.03.0@sha256:cf4032ca6b5f912a8b9d572d527d388401b68a0c9224cc086173e46bc4e1eabe AS coreutils -FROM stagex/eif_build:sx2024.03.0@sha256:6f3fed0aeaf9f9eebb43a370a5495fab92fcb21119fc23e261f0f24e1174009c AS eif_build -FROM stagex/file:sx2024.03.0@sha256:7fd68d1e7d5e1d3b1e52433bb6709f28d3e362ea89c9e13586b852ca0412f640 AS file -FROM stagex/filesystem:sx2024.03.0@sha256:42c8353db508ac79599df38c684502e50167352de2cddc5aea9b89486e7f8498 AS filesystem -FROM stagex/findutils:sx2024.03.0@sha256:475ea3488840297454f0f20b58e1b8292bf9b3944f901e3fce432fa4afeaa4cd AS findutils -FROM stagex/gcc:sx2024.03.0@sha256:25798fdde278a9f1f27e4092a1668e93d2766d4f8b089fba38d4684b20a9b0f7 AS gcc -FROM stagex/gen_initramfs:sx2024.03.0@sha256:a51c840a1c82dbc00c0a813964195d4f4bcb20463701083999320f826ffa49bf AS gen_initramfs -FROM stagex/git:sx2024.03.0@sha256:2c11f2daf9b8c1738cbd966b6de5dd0bcfaf81b675c2d268d30f972ddab9d9df AS git -FROM stagex/grep:sx2024.03.0@sha256:589465adc0125128c21534eb560299c335a41935e0ce182a632f4b739bf25c60 AS grep -FROM stagex/libunwind:sx2024.03.0@sha256:e74819e47c79f68a008302927ef02a5aa39cf12e859a8dfeccf9d1b4769b4833 AS libunwind -FROM stagex/linux-nitro:sx2024.03.0@sha256:073c4603686e3bdc0ed6755fee3203f6f6f1512e0ded09eaea8866b002b04264 AS linux-nitro -FROM stagex/llvm13:sx2024.03.0@sha256:97d0f3d32f58dca648cd70b0d58364d9bea5170bb99054c0a0b19ef57a7da7b1 AS llvm13 -FROM stagex/llvm:sx2024.03.0@sha256:8e361f1da92e956d947e37b6fc0a3951fcc1130863e2d3a9b4fca40ab4fd07f6 AS llvm -FROM stagex/musl-fts:sx2024.03.0@sha256:73c3c4647010f7151c711ed5005ef946c7c1a19c6e8921e057b5dbc15ef9559a AS musl-fts -FROM stagex/musl:sx2024.03.0@sha256:7db05e6817058a512a66ea82f3b99163069424c281363c2e9a48091d0d1d3bd9 AS musl -FROM stagex/musl-obstack:sx2024.03.0@sha256:4b6737815460908f666fa7a8e91138610d0a0909c408165a575ffb42bf21cd66 AS musl-obstack -FROM stagex/openssl:sx2024.03.0@sha256:1a2f656ced34d1ade99279c5663fcf0ec4f6526bcc50142079ef8adc080be3a9 AS openssl -FROM stagex/pcsc-lite:sx2024.03.0@sha256:e720e1795706c7c8c1db14bf730b10521e3ff42e4bed90addc590f7446aac8af AS pcsc-lite -FROM stagex/pkgconf:sx2024.03.0@sha256:31ce4eddaf4e777ddb51f01923089f3321ec5272ca0aa834d475f644279209b8 AS pkgconf -FROM stagex/rust:sx2024.03.0@sha256:fe22a0fcdb569cb70b8147378463fb6ff800e642be9d50542f8e25a38d90ec7f AS rust -FROM stagex/zlib:sx2024.03.0@sha256:de8f56f3ece28b14d575329bead53fc5318962ae3cb8f161a2d69710f7ec51f4 AS zlib +FROM stagex/bash:sx2024.04.2@sha256:6972f21bb1e1828d875c5adef57a979e97cf2487a55f454920ee43682731aa43 AS bash +FROM stagex/binutils:sx2024.04.2@sha256:311f8c2bd2b586bf7210c40dde43d0e0d5e76af4e1e688ad129f945691e3e105 AS binutils +FROM stagex/ca-certificates:sx2024.04.2@sha256:f9fe6e67df91083fee3d88cf221f84ef77f0b67480fb5b0689e890509a712533 AS ca-certificates +FROM stagex/coreutils:sx2024.04.2@sha256:6c02453e989104f475b606566e90f8abf71a9c2d591c91708c25b30a80b966d9 AS coreutils +FROM stagex/eif_build:sx2024.04.2@sha256:adf4c5293906b720fe39912207871872e89170eede20c95de05ea44e0a7ce2ae AS eif_build +FROM stagex/file:sx2024.04.2@sha256:fc3e7e06ae03aec2e969d90d80279f468de7861751abee7169ca285a54482570 AS file +FROM stagex/filesystem:sx2024.04.2@sha256:c504b17edae1bea8c139d058b20bebd383a0be1b4f57565d92cb578012f9c0f8 AS filesystem +FROM stagex/findutils:sx2024.04.2@sha256:43a6ae4b35574b1e36b2e95fc47eecc8b4d0f176f435e21259dbdf59d1c3f17d AS findutils +FROM stagex/gcc:sx2024.04.2@sha256:c67989de74d82eddeaf0d458edb1ca35b88064d3a66d5631c3530d5f10975f5e AS gcc +FROM stagex/gen_initramfs:sx2024.04.2@sha256:32f4993001fe799352b5fc12d79acc7b6bf746451557e924fd498e69a5270e1f AS gen_initramfs +FROM stagex/git:sx2024.04.2@sha256:16dc35c005f8e6cab3a20f116ce9ab7ea09c18d554f818f932a4fb3c44c90fdb AS git +FROM stagex/grep:sx2024.04.2@sha256:1a81d8d427b6ad73bd9243fe4031c12e66128bd4c2013132180a49e9034310ae AS grep +FROM stagex/libunwind:sx2024.04.2@sha256:622bbc0bcd502d349624fe90bd3cfc63595a71d450702d4e746abf8918351e2b AS libunwind +FROM stagex/linux-nitro:sx2024.04.2@sha256:3dd38f1acd01fa9c36eac432dc036c28149e1510be95c9538c7a66bf2ca9ac00 AS linux-nitro +FROM stagex/llvm13:sx2024.04.2@sha256:780b11314eaf2724e2ae84e7f84255a2d0ab6d3cd0e216707a2bbc27e6c7c6ff AS llvm13 +FROM stagex/llvm:sx2024.04.2@sha256:ae430dbdd1a6d546bb8aef817d09d3fb4e0145f81c071647666b7a9f7b69f8a1 AS llvm +FROM stagex/musl-fts:sx2024.04.2@sha256:5d815d099ccd6ebd5be9e24d96b303c1f2a36cc82fc82e3edf3602c1b50d424d AS musl-fts +FROM stagex/musl:sx2024.04.2@sha256:f888fcf45fabaaae3d0268bcec902ceb94edba7bf8d09ef6966ebb20e00b7127 AS musl +FROM stagex/musl-obstack:sx2024.04.2@sha256:9314fdf086b3663833c7df453a7aa1cd5832330a275939ceb39c0ab987e96088 AS musl-obstack +FROM stagex/openssl:sx2024.04.2@sha256:e719432f169a5ff606f52004a554e58780335f9635b1cf271d002ca7b1d1a206 AS openssl +FROM stagex/pcsc-lite:sx2024.04.2@sha256:be32e4ce71660ed0df3880bf536412c2bd598ba61464edbea86969a62eb41df5 AS pcsc-lite +FROM stagex/pkgconf:sx2024.04.2@sha256:31ce4eddaf4e777ddb51f01923089f3321ec5272ca0aa834d475f644279209b8 AS pkgconf +FROM stagex/rust:sx2024.04.2@sha256:e7882823078d49da4302578526dc35d6b2afad7507740c2f9bcd406a79befbc9 AS rust +FROM stagex/zlib:sx2024.04.2@sha256:d5b923b8f1b0382b8bdde96f36c0b8b9f694c97b14a9071bd96f2ffc46124e03 AS zlib FROM scratch as base ENV TARGET=x86_64-unknown-linux-musl From e51d28133a2ccea958e5b42a4c05cf4118e18989 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Tue, 23 Apr 2024 14:56:43 -0400 Subject: [PATCH 26/31] remove fail fast --- .github/workflows/stagex.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/stagex.yml b/.github/workflows/stagex.yml index d5291486..d34d8bd5 100644 --- a/.github/workflows/stagex.yml +++ b/.github/workflows/stagex.yml @@ -17,6 +17,7 @@ jobs: runs-on: group: ubuntu-runners strategy: + fail-fast: false matrix: include: - target: qos_client From 13a7aed4cab9a2e2af6dba6aae249b9105975992 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Tue, 23 Apr 2024 16:49:15 -0400 Subject: [PATCH 27/31] make sure to pull only hash pinned --- src/images/common/Containerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/images/common/Containerfile b/src/images/common/Containerfile index 5096487c..212780f1 100644 --- a/src/images/common/Containerfile +++ b/src/images/common/Containerfile @@ -28,8 +28,8 @@ ENV TARGET=x86_64-unknown-linux-musl ENV RUSTFLAGS="-C target-feature=+crt-static" ENV CARGOFLAGS="--locked --no-default-features --release --target ${TARGET}" ENV OPENSSL_STATIC=true -COPY --from=stagex/busybox . / -COPY --from=stagex/bash . / +COPY --from=busybox . / +COPY --from=bash . / COPY --from=coreutils . / COPY --from=findutils . / COPY --from=grep . / From 3063310011d0ead70e9b568b7d1608163d235374 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Wed, 24 Apr 2024 11:22:07 -0400 Subject: [PATCH 28/31] just run make; no updload --- .github/workflows/stagex.yml | 74 ++---------------------------------- 1 file changed, 3 insertions(+), 71 deletions(-) diff --git a/.github/workflows/stagex.yml b/.github/workflows/stagex.yml index d34d8bd5..877be2c1 100644 --- a/.github/workflows/stagex.yml +++ b/.github/workflows/stagex.yml @@ -11,85 +11,17 @@ on: jobs: build: - name: stagex build+push + name: build artifacts # We use a special group that is configured to use github largest runner instance # This is charged by the minute, so if you want to reduce cost change back to `runs-on: ubuntu-latest` runs-on: group: ubuntu-runners - strategy: - fail-fast: false - matrix: - include: - - target: qos_client - - target: qos_host - - target: qos_enclave - permissions: - id-token: write - contents: read - packages: write steps: - name: Checkout sources uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - - name: Setup Docker uses: ./.github/actions/docker-setup - - - name: Run `make ${{ matrix.target }}` - shell: 'script -q -e -c "bash {0}"' - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - make -j$(nproc) out/${{ matrix.target }}/index.json - - - name: upload to GHCR - run: | - echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u "${{ github.actor }}" --password-stdin - - env -C out/${{ matrix.target }} tar -cf - . | docker load - - for tag in ${tags}; do - docker tag "qos-local/${{ matrix.target }}:latest" "ghcr.io/tkhq/${{ matrix.target }}:${tag}" - done - - docker image push --all-tags ghcr.io/tkhq/${{ matrix.target }} - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - with: - aws-region: us-east-1 - role-to-assume: arn:aws:iam::799078726966:role/github-mono - - - name: Configure AWS Credentials - uses: aws-actions/configure-aws-credentials@010d0da01d0b5a38af31e9c3470dbfdabdecca3a # v4.0.1 - with: - aws-region: us-east-1 - role-to-assume: arn:aws:iam::799078726966:role/github-qos - - - name: Login to Amazon ECR - id: login-ecr - uses: aws-actions/amazon-ecr-login@062b18b96a7aff071d4dc91bc00c4c1a7945b076 # v2.0.1 - - - name: Upload to ECR + - name: Run `make` shell: 'script -q -e -c "bash {0}"' run: | - echo "${{ steps.login-ecr.outputs.docker_password_799078726966_dkr_ecr_us_east_1_amazonaws_com }}" | \ - docker login \ - ${{ steps.login-ecr.outputs.registry }} \ - -u "${{ steps.login-ecr.outputs.docker_username_799078726966_dkr_ecr_us_east_1_amazonaws_com }}" \ - --password-stdin - - export BASE_IMAGE_NAME="${{ steps.login-ecr.outputs.registry }}/tkhq/${{ matrix.target }}" - export IMAGE_NAME="${BASE_IMAGE_NAME}:sha-${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}" - export DIGEST_FILE=/tmp/image-digest-${{ matrix.target }}.sha256 - - cat out/${{ matrix.target }}/index.json | jq -r .manifests[].digest > "${DIGEST_FILE}" - - docker tag "qos-local/${{ matrix.target }}:latest" "$IMAGE_NAME" - - for tag in ${tags}; do - docker tag "$IMAGE_NAME" "$BASE_IMAGE_NAME:${tag}" - done - - docker image push --all-tags "$BASE_IMAGE_NAME" - - echo "Uploaded image $IMAGE_NAME (SHA-256 digest: $(cat $DIGEST_FILE))" - + make -j$(nproc) From 553043c1fd17506b3bb4c10547aac1c00b62e455 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Mon, 6 May 2024 20:17:38 -0400 Subject: [PATCH 29/31] respond to some feedback --- src/qos_client/RESHARD_GUIDE.md | 6 +++--- src/qos_core/src/protocol/services/reshard.rs | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/src/qos_client/RESHARD_GUIDE.md b/src/qos_client/RESHARD_GUIDE.md index c7fa4489..3cc9b1fb 100644 --- a/src/qos_client/RESHARD_GUIDE.md +++ b/src/qos_client/RESHARD_GUIDE.md @@ -41,7 +41,7 @@ qos_client boot-reshard \ ### 3 - Get attestation doc (Old Share Holder) -Get the attestation doc from the enclave. The attestation doc contains a refference to the reshard input and the ephemeral key which shares will be encrypted. +Get the attestation doc from the enclave. The attestation doc contains a reference to the reshard input and the ephemeral key which shares will be encrypted. ```sh qos_client get-reshard-attestation-doc \ @@ -62,7 +62,7 @@ For each quorum key being resharded, the user will need a separate directory wit - my_alias.share ``` -Note that the logic looks at the extension of the file to determine if its a share or the quorum key. +Note that the logic looks at the extension of the file to determine if it's a share or the quorum key. ```sh qos_client reshard-re-encrypt-share \ @@ -84,7 +84,7 @@ Post the re-encrypted shares from last step in order to reconstruct the quorum k ```sh qos_client reshard-post-share - --provision-input-path \ + --provision-input-path \ --host-port 3001 \ --host-ip localhost ``` diff --git a/src/qos_core/src/protocol/services/reshard.rs b/src/qos_core/src/protocol/services/reshard.rs index 959c9a3b..46fa015b 100644 --- a/src/qos_core/src/protocol/services/reshard.rs +++ b/src/qos_core/src/protocol/services/reshard.rs @@ -74,9 +74,9 @@ pub struct ReshardProvisionInput { pub struct ReshardInput { /// List of quorum public keys pub quorum_keys: Vec>, - /// The set and threshold to shard the key. + /// The share set and threshold to shard the quorum keys to. pub new_share_set: ShareSet, - /// The set the key is currently sharded too. + /// The share set the quorum keys are currently sharded too. pub old_share_set: ShareSet, /// The expected configuration of the enclave. Useful to verify the /// attestation document against. We also want those posting shares to @@ -289,7 +289,7 @@ pub(in crate::protocol) fn reshard_provision( reshard_input.new_share_set.threshold as usize, ); - // Now, lets create the new shards + // Now, let's create the new shards let member_outputs = zip(shares, reshard_input.new_share_set.members.iter().cloned()) .map(|(share, share_set_member)| -> Result { From 72b6de291f405368e87174bf710b25ee8b0ef747 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Mon, 6 May 2024 20:45:47 -0400 Subject: [PATCH 30/31] remove lfs and release section --- .lfsconfig | 2 -- README.md | 28 ---------------------------- 2 files changed, 30 deletions(-) delete mode 100644 .lfsconfig diff --git a/.lfsconfig b/.lfsconfig deleted file mode 100644 index 0c9ce0f4..00000000 --- a/.lfsconfig +++ /dev/null @@ -1,2 +0,0 @@ -[lfs] - url = https://turnkey.engineering/if-you-see-this-lfs-is-not-configured-correctly diff --git a/README.md b/README.md index 86ca01fb..624287a2 100644 --- a/README.md +++ b/README.md @@ -152,36 +152,8 @@ make toolchain-shell make toolchain-update ``` - -### Release Process - - 0. Determine the release semver version by consulting the [changelog](./CHANGELOG.MD). - 1. Create a branch for your release e.g. - `git checkout -b release/v1.0.0` - 2. Run `make dist` as described in ["Release" section](#release) - 3. Commit the new dist folder `git commit -m "Release v1.0.0" -- dist/` - 4. Push up your branch to github, and make a pull request. - 5. You may also create and push a signed `-rcX` git tag where the number after `rc` doesn't already exist. - `git tag -S v1.0.0-rc0 -m v1.0.0-rc0` - `git push origin v1.0.0-rc0` - 6. Wait for others to replicate your build, see ["Verify" section](#verify) - 7. Once the release has enough `git sig` signatures, make the final tag and merge the pull request. - `git tag -S v1.0.0 -m v1.0.0` - `git push origin v1.0.0` - - [gs]: https://codeberg.org/distrust/git-sig -### LFS setup - -This repository externalises large files so that they do not bulk up the git repo itself. -This is done through a tool called `git-lfs`, which must be installed for it to work. -Additionally, we use a custom agent to store our LFS objects in S3 (rather than the default and more expensive Github LFS service). - -In order to setup our s3 based lfs: - -1) Install [tkinfra](https://github.com/tkhq/mono/tree/main/src/go/tkinfra) -2) Run `./scripts/setup-lfs.sh` #### Troubleshooting From a3ef37a6154832a2b930a64d2b4492dec8245736 Mon Sep 17 00:00:00 2001 From: Zeke Mostov Date: Tue, 18 Jun 2024 12:49:12 -0400 Subject: [PATCH 31/31] revert container file --- src/images/common/Containerfile | 54 ++++++++++++++++----------------- 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/src/images/common/Containerfile b/src/images/common/Containerfile index 212780f1..b380c077 100644 --- a/src/images/common/Containerfile +++ b/src/images/common/Containerfile @@ -1,35 +1,35 @@ -FROM stagex/bash:sx2024.04.2@sha256:6972f21bb1e1828d875c5adef57a979e97cf2487a55f454920ee43682731aa43 AS bash -FROM stagex/binutils:sx2024.04.2@sha256:311f8c2bd2b586bf7210c40dde43d0e0d5e76af4e1e688ad129f945691e3e105 AS binutils -FROM stagex/ca-certificates:sx2024.04.2@sha256:f9fe6e67df91083fee3d88cf221f84ef77f0b67480fb5b0689e890509a712533 AS ca-certificates -FROM stagex/coreutils:sx2024.04.2@sha256:6c02453e989104f475b606566e90f8abf71a9c2d591c91708c25b30a80b966d9 AS coreutils -FROM stagex/eif_build:sx2024.04.2@sha256:adf4c5293906b720fe39912207871872e89170eede20c95de05ea44e0a7ce2ae AS eif_build -FROM stagex/file:sx2024.04.2@sha256:fc3e7e06ae03aec2e969d90d80279f468de7861751abee7169ca285a54482570 AS file -FROM stagex/filesystem:sx2024.04.2@sha256:c504b17edae1bea8c139d058b20bebd383a0be1b4f57565d92cb578012f9c0f8 AS filesystem -FROM stagex/findutils:sx2024.04.2@sha256:43a6ae4b35574b1e36b2e95fc47eecc8b4d0f176f435e21259dbdf59d1c3f17d AS findutils -FROM stagex/gcc:sx2024.04.2@sha256:c67989de74d82eddeaf0d458edb1ca35b88064d3a66d5631c3530d5f10975f5e AS gcc -FROM stagex/gen_initramfs:sx2024.04.2@sha256:32f4993001fe799352b5fc12d79acc7b6bf746451557e924fd498e69a5270e1f AS gen_initramfs -FROM stagex/git:sx2024.04.2@sha256:16dc35c005f8e6cab3a20f116ce9ab7ea09c18d554f818f932a4fb3c44c90fdb AS git -FROM stagex/grep:sx2024.04.2@sha256:1a81d8d427b6ad73bd9243fe4031c12e66128bd4c2013132180a49e9034310ae AS grep -FROM stagex/libunwind:sx2024.04.2@sha256:622bbc0bcd502d349624fe90bd3cfc63595a71d450702d4e746abf8918351e2b AS libunwind -FROM stagex/linux-nitro:sx2024.04.2@sha256:3dd38f1acd01fa9c36eac432dc036c28149e1510be95c9538c7a66bf2ca9ac00 AS linux-nitro -FROM stagex/llvm13:sx2024.04.2@sha256:780b11314eaf2724e2ae84e7f84255a2d0ab6d3cd0e216707a2bbc27e6c7c6ff AS llvm13 -FROM stagex/llvm:sx2024.04.2@sha256:ae430dbdd1a6d546bb8aef817d09d3fb4e0145f81c071647666b7a9f7b69f8a1 AS llvm -FROM stagex/musl-fts:sx2024.04.2@sha256:5d815d099ccd6ebd5be9e24d96b303c1f2a36cc82fc82e3edf3602c1b50d424d AS musl-fts -FROM stagex/musl:sx2024.04.2@sha256:f888fcf45fabaaae3d0268bcec902ceb94edba7bf8d09ef6966ebb20e00b7127 AS musl -FROM stagex/musl-obstack:sx2024.04.2@sha256:9314fdf086b3663833c7df453a7aa1cd5832330a275939ceb39c0ab987e96088 AS musl-obstack -FROM stagex/openssl:sx2024.04.2@sha256:e719432f169a5ff606f52004a554e58780335f9635b1cf271d002ca7b1d1a206 AS openssl -FROM stagex/pcsc-lite:sx2024.04.2@sha256:be32e4ce71660ed0df3880bf536412c2bd598ba61464edbea86969a62eb41df5 AS pcsc-lite -FROM stagex/pkgconf:sx2024.04.2@sha256:31ce4eddaf4e777ddb51f01923089f3321ec5272ca0aa834d475f644279209b8 AS pkgconf -FROM stagex/rust:sx2024.04.2@sha256:e7882823078d49da4302578526dc35d6b2afad7507740c2f9bcd406a79befbc9 AS rust -FROM stagex/zlib:sx2024.04.2@sha256:d5b923b8f1b0382b8bdde96f36c0b8b9f694c97b14a9071bd96f2ffc46124e03 AS zlib +FROM stagex/bash:sx2024.03.0@sha256:d1cbbb56847e6b1e7b879214aa6926b6fdfa210e9b42a2f612a6aea850ddeefc AS bash +FROM stagex/binutils:sx2024.03.0@sha256:3af41227e1fe6a8f9b3df9916ef4876840f33eaa172168e1db1d8f457ba011d5 AS binutils +FROM stagex/ca-certificates:sx2024.03.0@sha256:6746d2d203be3455bfc5ffd5a051c8edb73ecfd7be77c3da5a2973003a30794f AS ca-certificates +FROM stagex/coreutils:sx2024.03.0@sha256:cf4032ca6b5f912a8b9d572d527d388401b68a0c9224cc086173e46bc4e1eabe AS coreutils +FROM stagex/eif_build:sx2024.03.0@sha256:6f3fed0aeaf9f9eebb43a370a5495fab92fcb21119fc23e261f0f24e1174009c AS eif_build +FROM stagex/file:sx2024.03.0@sha256:7fd68d1e7d5e1d3b1e52433bb6709f28d3e362ea89c9e13586b852ca0412f640 AS file +FROM stagex/filesystem:sx2024.03.0@sha256:42c8353db508ac79599df38c684502e50167352de2cddc5aea9b89486e7f8498 AS filesystem +FROM stagex/findutils:sx2024.03.0@sha256:475ea3488840297454f0f20b58e1b8292bf9b3944f901e3fce432fa4afeaa4cd AS findutils +FROM stagex/gcc:sx2024.03.0@sha256:25798fdde278a9f1f27e4092a1668e93d2766d4f8b089fba38d4684b20a9b0f7 AS gcc +FROM stagex/gen_initramfs:sx2024.03.0@sha256:a51c840a1c82dbc00c0a813964195d4f4bcb20463701083999320f826ffa49bf AS gen_initramfs +FROM stagex/git:sx2024.03.0@sha256:2c11f2daf9b8c1738cbd966b6de5dd0bcfaf81b675c2d268d30f972ddab9d9df AS git +FROM stagex/grep:sx2024.03.0@sha256:589465adc0125128c21534eb560299c335a41935e0ce182a632f4b739bf25c60 AS grep +FROM stagex/libunwind:sx2024.03.0@sha256:e74819e47c79f68a008302927ef02a5aa39cf12e859a8dfeccf9d1b4769b4833 AS libunwind +FROM stagex/linux-nitro:sx2024.03.0@sha256:073c4603686e3bdc0ed6755fee3203f6f6f1512e0ded09eaea8866b002b04264 AS linux-nitro +FROM stagex/llvm13:sx2024.03.0@sha256:97d0f3d32f58dca648cd70b0d58364d9bea5170bb99054c0a0b19ef57a7da7b1 AS llvm13 +FROM stagex/llvm:sx2024.03.0@sha256:8e361f1da92e956d947e37b6fc0a3951fcc1130863e2d3a9b4fca40ab4fd07f6 AS llvm +FROM stagex/musl-fts:sx2024.03.0@sha256:73c3c4647010f7151c711ed5005ef946c7c1a19c6e8921e057b5dbc15ef9559a AS musl-fts +FROM stagex/musl:sx2024.03.0@sha256:7db05e6817058a512a66ea82f3b99163069424c281363c2e9a48091d0d1d3bd9 AS musl +FROM stagex/musl-obstack:sx2024.03.0@sha256:4b6737815460908f666fa7a8e91138610d0a0909c408165a575ffb42bf21cd66 AS musl-obstack +FROM stagex/openssl:sx2024.03.0@sha256:1a2f656ced34d1ade99279c5663fcf0ec4f6526bcc50142079ef8adc080be3a9 AS openssl +FROM stagex/pcsc-lite:sx2024.03.0@sha256:e720e1795706c7c8c1db14bf730b10521e3ff42e4bed90addc590f7446aac8af AS pcsc-lite +FROM stagex/pkgconf:sx2024.03.0@sha256:31ce4eddaf4e777ddb51f01923089f3321ec5272ca0aa834d475f644279209b8 AS pkgconf +FROM stagex/rust:sx2024.03.0@sha256:fe22a0fcdb569cb70b8147378463fb6ff800e642be9d50542f8e25a38d90ec7f AS rust +FROM stagex/zlib:sx2024.03.0@sha256:de8f56f3ece28b14d575329bead53fc5318962ae3cb8f161a2d69710f7ec51f4 AS zlib FROM scratch as base ENV TARGET=x86_64-unknown-linux-musl ENV RUSTFLAGS="-C target-feature=+crt-static" ENV CARGOFLAGS="--locked --no-default-features --release --target ${TARGET}" ENV OPENSSL_STATIC=true -COPY --from=busybox . / -COPY --from=bash . / +COPY --from=stagex/busybox . / +COPY --from=stagex/bash . / COPY --from=coreutils . / COPY --from=findutils . / COPY --from=grep . / @@ -50,4 +50,4 @@ COPY --from=file . / COPY --from=gcc . / COPY --from=linux-nitro /bzImage . COPY --from=linux-nitro /nsm.ko . -COPY --from=linux-nitro /linux.config . +COPY --from=linux-nitro /linux.config . \ No newline at end of file