diff --git a/ipa-core/src/bin/crypto_util.rs b/ipa-core/src/bin/crypto_util.rs index 99556089f..c884560f5 100644 --- a/ipa-core/src/bin/crypto_util.rs +++ b/ipa-core/src/bin/crypto_util.rs @@ -2,7 +2,7 @@ use std::fmt::Debug; use clap::{Parser, Subcommand}; use ipa_core::{ - cli::crypto::{DecryptArgs, EncryptArgs}, + cli::crypto::{DecryptArgs, EncryptArgs, HybridDecryptArgs, HybridEncryptArgs}, error::BoxError, }; @@ -17,7 +17,9 @@ struct Args { #[derive(Debug, Subcommand)] enum CryptoUtilCommand { Encrypt(EncryptArgs), + HybridEncrypt(HybridEncryptArgs), Decrypt(DecryptArgs), + HybridDecrypt(HybridDecryptArgs), } #[tokio::main] @@ -25,7 +27,11 @@ async fn main() -> Result<(), BoxError> { let args = Args::parse(); match args.action { CryptoUtilCommand::Encrypt(encrypt_args) => encrypt_args.encrypt()?, + CryptoUtilCommand::HybridEncrypt(hybrid_encrypt_args) => hybrid_encrypt_args.encrypt()?, CryptoUtilCommand::Decrypt(decrypt_args) => decrypt_args.decrypt_and_reconstruct().await?, + CryptoUtilCommand::HybridDecrypt(hybrid_decrypt_args) => { + hybrid_decrypt_args.decrypt_and_reconstruct().await? + } } Ok(()) } diff --git a/ipa-core/src/cli/crypto/hybrid_decrypt.rs b/ipa-core/src/cli/crypto/hybrid_decrypt.rs new file mode 100644 index 000000000..ed5747384 --- /dev/null +++ b/ipa-core/src/cli/crypto/hybrid_decrypt.rs @@ -0,0 +1,295 @@ +use std::{ + fs::{File, OpenOptions}, + io::{BufRead, BufReader, Write}, + path::{Path, PathBuf}, +}; + +use clap::Parser; + +use crate::{ + config::{hpke_registry, HpkeServerConfig}, + error::BoxError, + ff::{ + boolean_array::{BA3, BA8}, + U128Conversions, + }, + hpke::{KeyRegistry, PrivateKeyOnly}, + report::hybrid::{EncryptedHybridReport, HybridReport}, + test_fixture::Reconstruct, +}; + +#[derive(Debug, Parser)] +#[clap(name = "test_hybrid_decrypt", about = "Test Hybrid Decrypt")] +#[command(about)] +pub struct HybridDecryptArgs { + /// Path to helper1 file to decrypt + #[arg(long)] + input_file1: PathBuf, + + /// Helper1 Private key for decrypting match keys + #[arg(long)] + mk_private_key1: PathBuf, + + /// Path to helper2 file to decrypt + #[arg(long)] + input_file2: PathBuf, + + /// Helper2 Private key for decrypting match keys + #[arg(long)] + mk_private_key2: PathBuf, + + /// Path to helper3 file to decrypt + #[arg(long)] + input_file3: PathBuf, + + /// Helper3 Private key for decrypting match keys + #[arg(long)] + mk_private_key3: PathBuf, + + /// The destination file for decrypted output. + #[arg(long, value_name = "FILE")] + output_file: PathBuf, +} + +impl HybridDecryptArgs { + #[must_use] + pub fn new( + input_file1: &Path, + input_file2: &Path, + input_file3: &Path, + mk_private_key1: &Path, + mk_private_key2: &Path, + mk_private_key3: &Path, + output_file: &Path, + ) -> Self { + Self { + input_file1: input_file1.to_path_buf(), + mk_private_key1: mk_private_key1.to_path_buf(), + input_file2: input_file2.to_path_buf(), + mk_private_key2: mk_private_key2.to_path_buf(), + input_file3: input_file3.to_path_buf(), + mk_private_key3: mk_private_key3.to_path_buf(), + output_file: output_file.to_path_buf(), + } + } + + /// # Panics + // if input files or private_keys are not correctly formatted + /// # Errors + /// if it cannot open the files + pub async fn decrypt_and_reconstruct(self) -> Result<(), BoxError> { + let Self { + input_file1, + mk_private_key1, + input_file2, + mk_private_key2, + input_file3, + mk_private_key3, + output_file, + } = self; + let key_registry1 = build_hpke_registry(mk_private_key1).await?; + let key_registry2 = build_hpke_registry(mk_private_key2).await?; + let key_registry3 = build_hpke_registry(mk_private_key3).await?; + let decrypted_reports1 = DecryptedHybridReports::new(&input_file1, key_registry1); + let decrypted_reports2 = DecryptedHybridReports::new(&input_file2, key_registry2); + let decrypted_reports3 = DecryptedHybridReports::new(&input_file3, key_registry3); + + let mut writer = Box::new( + OpenOptions::new() + .write(true) + .create_new(true) + .open(output_file)?, + ); + + for (dec_report1, (dec_report2, dec_report3)) in + decrypted_reports1.zip(decrypted_reports2.zip(decrypted_reports3)) + { + match (dec_report1, dec_report2, dec_report3) { + ( + HybridReport::Impression(impression_report1), + HybridReport::Impression(impression_report2), + HybridReport::Impression(impression_report3), + ) => { + let match_key = [ + impression_report1.match_key, + impression_report2.match_key, + impression_report3.match_key, + ] + .reconstruct() + .as_u128(); + + let breakdown_key = [ + impression_report1.breakdown_key, + impression_report2.breakdown_key, + impression_report3.breakdown_key, + ] + .reconstruct() + .as_u128(); + let key_id = impression_report1.info.key_id; + let helper_origin = impression_report1.info.helper_origin; + + writeln!( + writer, + "i,{match_key},{breakdown_key},{key_id},{helper_origin}" + )?; + } + ( + HybridReport::Conversion(conversion_report1), + HybridReport::Conversion(conversion_report2), + HybridReport::Conversion(conversion_report3), + ) => { + let match_key = [ + conversion_report1.match_key, + conversion_report2.match_key, + conversion_report3.match_key, + ] + .reconstruct() + .as_u128(); + + let value = [ + conversion_report1.value, + conversion_report2.value, + conversion_report3.value, + ] + .reconstruct() + .as_u128(); + let key_id = conversion_report1.info.key_id; + let helper_origin = conversion_report1.info.helper_origin; + let conversion_site_domain = conversion_report1.info.conversion_site_domain; + let timestamp = conversion_report1.info.timestamp; + let epsilon = conversion_report1.info.epsilon; + let sensitivity = conversion_report1.info.sensitivity; + writeln!(writer, "c,{match_key},{value},{key_id},{helper_origin},{conversion_site_domain},{timestamp},{epsilon},{sensitivity}")?; + } + _ => { + panic!("Reports are not all the same type"); + } + } + } + + Ok(()) + } +} + +struct DecryptedHybridReports { + reader: BufReader, + key_registry: KeyRegistry, +} + +impl Iterator for DecryptedHybridReports { + type Item = HybridReport; + + fn next(&mut self) -> Option { + let mut line = String::new(); + if self.reader.read_line(&mut line).unwrap() > 0 { + let encrypted_report_bytes = hex::decode(line.trim()).unwrap(); + let enc_report = + EncryptedHybridReport::from_bytes(encrypted_report_bytes.into()).unwrap(); + let dec_report: HybridReport = + enc_report.decrypt(&self.key_registry).unwrap(); + Some(dec_report) + } else { + None + } + } +} + +impl DecryptedHybridReports { + fn new(filename: &PathBuf, key_registry: KeyRegistry) -> Self { + let file = File::open(filename) + .unwrap_or_else(|e| panic!("unable to open file {filename:?}. {e}")); + let reader = BufReader::new(file); + Self { + reader, + key_registry, + } + } +} + +async fn build_hpke_registry( + private_key_file: PathBuf, +) -> Result, BoxError> { + let mk_encryption = Some(HpkeServerConfig::File { private_key_file }); + let key_registry = hpke_registry(mk_encryption.as_ref()).await?; + Ok(key_registry) +} + +#[cfg(test)] +mod tests { + + use tempfile::tempdir; + + use crate::cli::crypto::{ + hybrid_decrypt::HybridDecryptArgs, hybrid_encrypt::HybridEncryptArgs, hybrid_sample_data, + }; + + #[tokio::test] + #[should_panic = "No such file or directory (os error 2)"] + async fn decrypt_no_enc_file() { + let input_file = + hybrid_sample_data::write_csv(hybrid_sample_data::test_hybrid_data().take(10)).unwrap(); + + let output_dir = tempdir().unwrap(); + let network_file = hybrid_sample_data::test_keys().network_config(); + HybridEncryptArgs::new(input_file.path(), output_dir.path(), network_file.path()) + .encrypt() + .unwrap(); + + let decrypt_output = output_dir.path().join("output"); + let enc1 = output_dir.path().join("DOES_NOT_EXIST.enc"); + let enc2 = output_dir.path().join("helper2.enc"); + let enc3 = output_dir.path().join("helper3.enc"); + + let [mk_private_key1, mk_private_key2, mk_private_key3] = + hybrid_sample_data::test_keys().sk_files(); + + let decrypt_args = HybridDecryptArgs::new( + enc1.as_path(), + enc2.as_path(), + enc3.as_path(), + mk_private_key1.path(), + mk_private_key2.path(), + mk_private_key3.path(), + &decrypt_output, + ); + decrypt_args.decrypt_and_reconstruct().await.unwrap(); + } + + #[tokio::test] + #[should_panic = "called `Result::unwrap()` on an `Err` value: Crypt(Other)"] + async fn decrypt_bad_private_key() { + let input_file = + hybrid_sample_data::write_csv(hybrid_sample_data::test_hybrid_data().take(10)).unwrap(); + + let network_file = hybrid_sample_data::test_keys().network_config(); + let output_dir = tempdir().unwrap(); + HybridEncryptArgs::new(input_file.path(), output_dir.path(), network_file.path()) + .encrypt() + .unwrap(); + + let decrypt_output = output_dir.path().join("output"); + let enc1 = output_dir.path().join("helper1.enc"); + let enc2 = output_dir.path().join("helper2.enc"); + let enc3 = output_dir.path().join("helper3.enc"); + + // corrupt the secret key for H1 + let mut keys = hybrid_sample_data::test_keys().clone(); + let mut sk = keys.get_sk(0); + sk[0] = !sk[0]; + keys.set_sk(0, sk); + let [mk_private_key1, mk_private_key2, mk_private_key3] = keys.sk_files(); + + HybridDecryptArgs::new( + enc1.as_path(), + enc2.as_path(), + enc3.as_path(), + mk_private_key1.path(), + mk_private_key2.path(), + mk_private_key3.path(), + &decrypt_output, + ) + .decrypt_and_reconstruct() + .await + .unwrap(); + } +} diff --git a/ipa-core/src/cli/crypto/hybrid_encrypt.rs b/ipa-core/src/cli/crypto/hybrid_encrypt.rs new file mode 100644 index 000000000..e0c749faf --- /dev/null +++ b/ipa-core/src/cli/crypto/hybrid_encrypt.rs @@ -0,0 +1,214 @@ +use std::{ + fs::{read_to_string, OpenOptions}, + io::Write, + iter::zip, + path::{Path, PathBuf}, +}; + +use clap::Parser; +use rand::thread_rng; + +use crate::{ + cli::{ + config_parse::HelperNetworkConfigParseExt, + playbook::{BreakdownKey, InputSource, TriggerValue}, + }, + config::{KeyRegistries, NetworkConfig}, + error::BoxError, + report::hybrid::{HybridReport, DEFAULT_KEY_ID}, + secret_sharing::IntoShares, + test_fixture::hybrid::TestHybridRecord, +}; + +#[derive(Debug, Parser)] +#[clap(name = "test_hybrid_encrypt", about = "Test Hybrid Encrypt")] +#[command(about)] +pub struct HybridEncryptArgs { + /// Path to file to secret share and encrypt + #[arg(long)] + input_file: PathBuf, + /// The destination dir for encrypted output. + /// In that dir, it will create helper1.enc, + /// helper2.enc, and helper3.enc + #[arg(long, value_name = "FILE")] + output_dir: PathBuf, + /// Path to helper network configuration file + #[arg(long)] + network: PathBuf, +} + +impl HybridEncryptArgs { + #[must_use] + pub fn new(input_file: &Path, output_dir: &Path, network: &Path) -> Self { + Self { + input_file: input_file.to_path_buf(), + output_dir: output_dir.to_path_buf(), + network: network.to_path_buf(), + } + } + + /// # Panics + /// if input file or network file are not correctly formatted + /// # Errors + /// if it cannot open the files + pub fn encrypt(&self) -> Result<(), BoxError> { + let input = InputSource::from_file(&self.input_file); + + let mut rng = thread_rng(); + let mut key_registries = KeyRegistries::default(); + + let network = + NetworkConfig::from_toml_str(&read_to_string(&self.network).unwrap_or_else(|e| { + panic!("Failed to open network file: {:?}. {}", &self.network, e) + })) + .unwrap_or_else(|e| { + panic!( + "Failed to parse network file into toml: {:?}. {}", + &self.network, e + ) + }); + let Some(key_registries) = key_registries.init_from(&network) else { + panic!("could not load network file") + }; + + let shares: [Vec>; 3] = + input.iter::().share(); + + for (index, (shares, key_registry)) in zip(shares, key_registries).enumerate() { + let output_filename = format!("helper{}.enc", index + 1); + let mut writer = OpenOptions::new() + .write(true) + .create_new(true) + .open(self.output_dir.join(&output_filename)) + .unwrap_or_else(|e| panic!("unable write to {}. {}", &output_filename, e)); + + for share in shares { + let output = share + .encrypt(DEFAULT_KEY_ID, key_registry, &mut rng) + .unwrap(); + let hex_output = hex::encode(&output); + writeln!(writer, "{hex_output}")?; + } + } + + Ok(()) + } +} + +#[cfg(all(test, unit_test))] +mod tests { + use std::io::Write; + + use tempfile::{tempdir, NamedTempFile}; + + use crate::{ + cli::{ + crypto::{hybrid_encrypt::HybridEncryptArgs, sample_data}, + CsvSerializer, + }, + test_fixture::hybrid::TestHybridRecord, + }; + + #[tokio::test] + async fn try_encrypting_something() { + let helper_origin = "HELPER_ORIGIN".to_string(); + let conversion_site_domain = "meta.com".to_string(); + let records = vec![ + TestHybridRecord::TestConversion { + match_key: 12345, + value: 2, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 100, + epsilon: 0.0, + sensitivity: 0.0, + }, + TestHybridRecord::TestConversion { + match_key: 12345, + value: 5, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 101, + epsilon: 0.0, + sensitivity: 0.0, + }, + TestHybridRecord::TestImpression { + match_key: 23456, + breakdown_key: 4, + key_id: 0, + helper_origin: helper_origin.clone(), + }, + ]; + let mut input_file = NamedTempFile::new().unwrap(); + + for event in records { + event.to_csv(input_file.as_file_mut()).unwrap(); + writeln!(input_file.as_file()).unwrap(); + } + input_file.flush().unwrap(); + + let output_dir = tempdir().unwrap(); + let network_file = sample_data::test_keys().network_config(); + + HybridEncryptArgs::new(input_file.path(), output_dir.path(), network_file.path()) + .encrypt() + .unwrap(); + } + + #[test] + #[should_panic = "Failed to open network file:"] + fn encrypt_no_network_file() { + let input_file = sample_data::write_csv(sample_data::test_ipa_data().take(10)).unwrap(); + + let output_dir = tempdir().unwrap(); + let network_dir = tempdir().unwrap(); + let network_file = network_dir.path().join("does_not_exist"); + HybridEncryptArgs::new(input_file.path(), output_dir.path(), &network_file) + .encrypt() + .unwrap(); + } + + #[test] + #[should_panic = "TOML parse error at"] + fn encrypt_bad_network_file() { + let input_file = sample_data::write_csv(sample_data::test_ipa_data().take(10)).unwrap(); + let output_dir = tempdir().unwrap(); + let network_data = r" +this is not toml! +%^& weird characters +(\deadbeef>? +"; + let mut network_file = NamedTempFile::new().unwrap(); + writeln!(network_file.as_file_mut(), "{network_data}").unwrap(); + + HybridEncryptArgs::new(input_file.path(), output_dir.path(), network_file.path()) + .encrypt() + .unwrap(); + } + + #[test] + #[should_panic(expected = "Failed to parse network file into toml")] + fn encrypt_incomplete_network_file() { + let input_file = sample_data::write_csv(sample_data::test_ipa_data().take(10)).unwrap(); + + let output_dir = tempdir().unwrap(); + let network_data = r#" +[[peers]] +url = "helper1.test" +[peers.hpke] +public_key = "92a6fb666c37c008defd74abf3204ebea685742eab8347b08e2f7c759893947a" +[[peers]] +url = "helper2.test" +[peers.hpke] +public_key = "cfdbaaff16b30aa8a4ab07eaad2cdd80458208a1317aefbb807e46dce596617e" +"#; + let mut network_file = NamedTempFile::new().unwrap(); + writeln!(network_file.as_file_mut(), "{network_data}").unwrap(); + + HybridEncryptArgs::new(input_file.path(), output_dir.path(), network_file.path()) + .encrypt() + .unwrap(); + } +} diff --git a/ipa-core/src/cli/crypto/mod.rs b/ipa-core/src/cli/crypto/mod.rs index 0bcb1f629..ac0fd1a06 100644 --- a/ipa-core/src/cli/crypto/mod.rs +++ b/ipa-core/src/cli/crypto/mod.rs @@ -1,8 +1,12 @@ mod decrypt; mod encrypt; +mod hybrid_decrypt; +mod hybrid_encrypt; pub use decrypt::DecryptArgs; pub use encrypt::EncryptArgs; +pub use hybrid_decrypt::HybridDecryptArgs; +pub use hybrid_encrypt::HybridEncryptArgs; #[cfg(test)] mod sample_data { @@ -137,6 +141,142 @@ mod sample_data { } } +#[cfg(test)] +mod hybrid_sample_data { + use std::{io, io::Write, sync::OnceLock}; + + use hpke::{Deserializable, Serializable}; + use rand::thread_rng; + use tempfile::NamedTempFile; + + use crate::{ + cli::CsvSerializer, + hpke::{IpaPrivateKey, IpaPublicKey}, + test_fixture::{ + hybrid::TestHybridRecord, hybrid_event_gen::ConversionDistribution, + HybridEventGenerator, HybridGeneratorConfig, + }, + }; + + /// Keys that are used in crypto tests + #[derive(Clone)] + pub(super) struct TestKeys { + key_pairs: [(IpaPublicKey, IpaPrivateKey); 3], + } + + static TEST_KEYS: OnceLock = OnceLock::new(); + pub fn test_keys() -> &'static TestKeys { + TEST_KEYS.get_or_init(TestKeys::new) + } + + impl TestKeys { + pub fn new() -> Self { + Self { + key_pairs: [ + ( + decode_key::<_, IpaPublicKey>( + "92a6fb666c37c008defd74abf3204ebea685742eab8347b08e2f7c759893947a", + ), + decode_key::<_, IpaPrivateKey>( + "53d58e022981f2edbf55fec1b45dbabd08a3442cb7b7c598839de5d7a5888bff", + ), + ), + ( + decode_key::<_, IpaPublicKey>( + "cfdbaaff16b30aa8a4ab07eaad2cdd80458208a1317aefbb807e46dce596617e", + ), + decode_key::<_, IpaPrivateKey>( + "3a0a993a3cfc7e8d381addac586f37de50c2a14b1a6356d71e94ca2afaeb2569", + ), + ), + ( + decode_key::<_, IpaPublicKey>( + "b900be35da06106a83ed73c33f733e03e4ea5888b7ea4c912ab270b0b0f8381e", + ), + decode_key::<_, IpaPrivateKey>( + "1fb5c5274bf85fbe6c7935684ef05499f6cfb89ac21640c28330135cc0e8a0f7", + ), + ), + ], + } + } + + pub fn network_config(&self) -> NamedTempFile { + let mut file = NamedTempFile::new().unwrap(); + let [pk1, pk2, pk3] = self.key_pairs.each_ref().map(|(pk, _)| pk); + let [pk1, pk2, pk3] = [ + hex::encode(pk1.to_bytes()), + hex::encode(pk2.to_bytes()), + hex::encode(pk3.to_bytes()), + ]; + let network_data = format!( + r#" + [[peers]] + url = "helper1.test" + [peers.hpke] + public_key = "{pk1}" + [[peers]] + url = "helper2.test" + [peers.hpke] + public_key = "{pk2}" + [[peers]] + url = "helper3.test" + [peers.hpke] + public_key = "{pk3}" + "# + ); + file.write_all(network_data.as_bytes()).unwrap(); + + file + } + + pub fn set_sk>(&mut self, idx: usize, data: I) { + self.key_pairs[idx].1 = IpaPrivateKey::from_bytes(data.as_ref()).unwrap(); + } + + pub fn get_sk(&self, idx: usize) -> Vec { + self.key_pairs[idx].1.to_bytes().to_vec() + } + + pub fn sk_files(&self) -> [NamedTempFile; 3] { + self.key_pairs.each_ref().map(|(_, sk)| sk).map(|sk| { + let mut file = NamedTempFile::new().unwrap(); + file.write_all(hex::encode(sk.to_bytes()).as_bytes()) + .unwrap(); + file.flush().unwrap(); + + file + }) + } + } + + fn decode_key, T: Deserializable>(input: I) -> T { + let bytes = hex::decode(input).unwrap(); + T::from_bytes(&bytes).unwrap() + } + + pub fn test_hybrid_data() -> impl Iterator { + let rng = thread_rng(); + let event_gen_args = HybridGeneratorConfig::new(3, 50, ConversionDistribution::Default); + + HybridEventGenerator::with_config(rng, event_gen_args) + } + + pub fn write_csv( + data: impl Iterator, + ) -> Result { + let mut file = NamedTempFile::new()?; + for event in data { + let () = event.to_csv(&mut file)?; + writeln!(file)?; + } + + file.flush()?; + + Ok(file) + } +} + #[cfg(all(test, unit_test))] mod tests { use std::{ @@ -147,7 +287,10 @@ mod tests { use tempfile::tempdir; - use crate::cli::crypto::{decrypt::DecryptArgs, encrypt::EncryptArgs, sample_data}; + use crate::cli::crypto::{ + decrypt::DecryptArgs, encrypt::EncryptArgs, hybrid_decrypt::HybridDecryptArgs, + hybrid_encrypt::HybridEncryptArgs, hybrid_sample_data, sample_data, + }; fn are_files_equal(file1: &Path, file2: &Path) { let file1 = @@ -195,4 +338,37 @@ mod tests { are_files_equal(input_file.path(), &decrypt_output); } + + #[tokio::test] + async fn hybrid_encrypt_and_decrypt() { + let output_dir = tempdir().unwrap(); + let input = hybrid_sample_data::test_hybrid_data().take(10); + let input_file = hybrid_sample_data::write_csv(input).unwrap(); + let network_file = hybrid_sample_data::test_keys().network_config(); + HybridEncryptArgs::new(input_file.path(), output_dir.path(), network_file.path()) + .encrypt() + .unwrap(); + + let decrypt_output = output_dir.path().join("output"); + let enc1 = output_dir.path().join("helper1.enc"); + let enc2 = output_dir.path().join("helper2.enc"); + let enc3 = output_dir.path().join("helper3.enc"); + let [mk_private_key1, mk_private_key2, mk_private_key3] = + hybrid_sample_data::test_keys().sk_files(); + + HybridDecryptArgs::new( + enc1.as_path(), + enc2.as_path(), + enc3.as_path(), + mk_private_key1.path(), + mk_private_key2.path(), + mk_private_key3.path(), + &decrypt_output, + ) + .decrypt_and_reconstruct() + .await + .unwrap(); + + are_files_equal(input_file.path(), &decrypt_output); + } }