From cf7efd485096b3132e44366a6c0aad7d263ecd21 Mon Sep 17 00:00:00 2001 From: tyurek Date: Tue, 3 Dec 2024 15:50:27 -0800 Subject: [PATCH] Add info fields to Hybrid Report Structs (#1469) Add info fields to Hybrid Report Structs and change encryption interfaces to no longer require info More specifically: Removes Serializable trait from HybridReport Rewrites HybridInfo and related structs to be serializable and deserializable Adds these info structs to report structs (as a struct for HybridReport, as bytes for EncryptedHybridReport) Changes the encryption/decryption interfaces to not require an Info argument (grabbed from the report instead) Changes TestHybridRecord to include info fields --- ipa-core/src/cli/csv.rs | 20 +- ipa-core/src/cli/playbook/input.rs | 99 +++-- ipa-core/src/protocol/hybrid/agg.rs | 63 +++ ipa-core/src/protocol/hybrid/oprf.rs | 25 ++ ipa-core/src/query/runner/hybrid.rs | 58 +-- ipa-core/src/report/hybrid.rs | 361 +++++++++--------- ipa-core/src/report/hybrid_info.rs | 221 +++++++++-- ipa-core/src/test_fixture/hybrid.rs | 115 +++++- ipa-core/src/test_fixture/hybrid_event_gen.rs | 13 +- 9 files changed, 685 insertions(+), 290 deletions(-) diff --git a/ipa-core/src/cli/csv.rs b/ipa-core/src/cli/csv.rs index 621c9b352..9ea132173 100644 --- a/ipa-core/src/cli/csv.rs +++ b/ipa-core/src/cli/csv.rs @@ -28,11 +28,25 @@ impl Serializer for crate::test_fixture::hybrid::TestHybridRecord { crate::test_fixture::hybrid::TestHybridRecord::TestImpression { match_key, breakdown_key, + key_id, + helper_origin, } => { - write!(buf, "i,{match_key},{breakdown_key}")?; + write!( + buf, + "i,{match_key},{breakdown_key},{key_id},{helper_origin}" + )?; } - crate::test_fixture::hybrid::TestHybridRecord::TestConversion { match_key, value } => { - write!(buf, "c,{match_key},{value}")?; + crate::test_fixture::hybrid::TestHybridRecord::TestConversion { + match_key, + value, + key_id, + helper_origin, + conversion_site_domain, + timestamp, + epsilon, + sensitivity, + } => { + write!(buf, "c,{match_key},{value},{key_id},{helper_origin},{conversion_site_domain},{timestamp},{epsilon},{sensitivity}")?; } } diff --git a/ipa-core/src/cli/playbook/input.rs b/ipa-core/src/cli/playbook/input.rs index 5015d5587..85bdac631 100644 --- a/ipa-core/src/cli/playbook/input.rs +++ b/ipa-core/src/cli/playbook/input.rs @@ -59,34 +59,81 @@ impl InputItem for TestRawDataRecord { impl InputItem for TestHybridRecord { fn from_str(s: &str) -> Self { - if let [event_type, match_key, number] = s.splitn(3, ',').collect::>()[..] { - let match_key: u64 = match_key - .parse() - .unwrap_or_else(|e| panic!("Expected an u64, got {match_key}: {e}")); - - let number: u32 = number - .parse() - .unwrap_or_else(|e| panic!("Expected an u32, got {number}: {e}")); - - match event_type { - "i" => TestHybridRecord::TestImpression { - match_key, - breakdown_key: number, - }, - - "c" => TestHybridRecord::TestConversion { - match_key, - value: number, - }, - _ => panic!( - "{}", - format!( + let event_type = s.chars().nth(0).unwrap(); + match event_type { + 'i' => { + if let [_, match_key, number, key_id, helper_origin] = + s.splitn(5, ',').collect::>()[..] + { + let match_key: u64 = match_key + .parse() + .unwrap_or_else(|e| panic!("Expected a u64, got {match_key}: {e}")); + + let number: u32 = number + .parse() + .unwrap_or_else(|e| panic!("Expected a u32, got {number}: {e}")); + + let key_id: u8 = key_id + .parse() + .unwrap_or_else(|e| panic!("Expected a u8, got {key_id}: {e}")); + TestHybridRecord::TestImpression { + match_key, + breakdown_key: number, + key_id, + helper_origin: helper_origin.to_string(), + } + } else { + panic!("{s} is not a valid {}", type_name::()) + } + } + + 'c' => { + if let [_, match_key, number, key_id, helper_origin, conversion_site_domain, timestamp, epsilon, sensitivity] = + s.splitn(9, ',').collect::>()[..] + { + let match_key: u64 = match_key + .parse() + .unwrap_or_else(|e| panic!("Expected a u64, got {match_key}: {e}")); + + let number: u32 = number + .parse() + .unwrap_or_else(|e| panic!("Expected a u32, got {number}: {e}")); + + let key_id: u8 = key_id + .parse() + .unwrap_or_else(|e| panic!("Expected a u8, got {key_id}: {e}")); + + let timestamp: u64 = timestamp + .parse() + .unwrap_or_else(|e| panic!("Expected a u64, got {timestamp}: {e}")); + + let epsilon: f64 = epsilon + .parse() + .unwrap_or_else(|e| panic!("Expected an f64, got {epsilon}: {e}")); + + let sensitivity: f64 = sensitivity + .parse() + .unwrap_or_else(|e| panic!("Expected an f64, got {sensitivity}: {e}")); + TestHybridRecord::TestConversion { + match_key, + value: number, + key_id, + helper_origin: helper_origin.to_string(), + conversion_site_domain: conversion_site_domain.to_string(), + timestamp, + epsilon, + sensitivity, + } + } else { + panic!("{s} is not a valid {}", type_name::()) + } + } + _ => panic!( + "{}", + format!( "Invalid input. Rows should start with 'i' or 'c'. Did not expect {event_type}" ) - ), - } - } else { - panic!("{s} is not a valid {}", type_name::()) + ), } } } diff --git a/ipa-core/src/protocol/hybrid/agg.rs b/ipa-core/src/protocol/hybrid/agg.rs index 85f59f58a..d756b0748 100644 --- a/ipa-core/src/protocol/hybrid/agg.rs +++ b/ipa-core/src/protocol/hybrid/agg.rs @@ -193,65 +193,128 @@ pub mod test { const SHARD1_MKS: [u64; 7] = [12345, 12345, 34567, 34567, 78901, 78901, 78901]; const SHARD2_MKS: [u64; 7] = [23456, 23456, 45678, 56789, 67890, 67890, 67890]; + #[allow(clippy::too_many_lines)] fn get_records() -> Vec { + let helper_origin = "HELPER_ORIGIN".to_string(); + let conversion_site_domain = "meta.com".to_string(); let shard1_records = [ TestHybridRecord::TestImpression { match_key: SHARD1_MKS[0], breakdown_key: 45, + key_id: 0, + helper_origin: helper_origin.clone(), }, TestHybridRecord::TestConversion { match_key: SHARD1_MKS[1], value: 1, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 102, + epsilon: 0.0, + sensitivity: 0.0, }, // attributed TestHybridRecord::TestConversion { match_key: SHARD1_MKS[2], value: 3, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 103, + epsilon: 0.0, + sensitivity: 0.0, }, TestHybridRecord::TestConversion { match_key: SHARD1_MKS[3], value: 4, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 104, + epsilon: 0.0, + sensitivity: 0.0, }, // not attibuted, but duplicated conversion. will land in breakdown_key 0 TestHybridRecord::TestImpression { match_key: SHARD1_MKS[4], breakdown_key: 1, + key_id: 0, + helper_origin: helper_origin.clone(), }, // duplicated impression with same match_key TestHybridRecord::TestImpression { match_key: SHARD1_MKS[4], breakdown_key: 2, + key_id: 0, + helper_origin: helper_origin.clone(), }, // duplicated impression with same match_key TestHybridRecord::TestConversion { match_key: SHARD1_MKS[5], value: 7, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 105, + epsilon: 0.0, + sensitivity: 0.0, }, // removed ]; let shard2_records = [ TestHybridRecord::TestImpression { match_key: SHARD2_MKS[0], breakdown_key: 56, + key_id: 0, + helper_origin: helper_origin.clone(), }, TestHybridRecord::TestConversion { match_key: SHARD2_MKS[1], value: 2, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 100, + epsilon: 0.0, + sensitivity: 0.0, }, // attributed TestHybridRecord::TestImpression { match_key: SHARD2_MKS[2], breakdown_key: 78, + key_id: 0, + helper_origin: helper_origin.clone(), }, // NOT attributed TestHybridRecord::TestConversion { match_key: SHARD2_MKS[3], value: 5, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 101, + epsilon: 0.0, + sensitivity: 0.0, }, // NOT attributed TestHybridRecord::TestImpression { match_key: SHARD2_MKS[4], breakdown_key: 90, + key_id: 0, + helper_origin: helper_origin.clone(), }, // attributed twice, removed TestHybridRecord::TestConversion { match_key: SHARD2_MKS[5], value: 6, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 102, + epsilon: 0.0, + sensitivity: 0.0, }, // attributed twice, removed TestHybridRecord::TestConversion { match_key: SHARD2_MKS[6], value: 7, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 103, + epsilon: 0.0, + sensitivity: 0.0, }, // attributed twice, removed ]; diff --git a/ipa-core/src/protocol/hybrid/oprf.rs b/ipa-core/src/protocol/hybrid/oprf.rs index 63f2c7b17..7ededb91e 100644 --- a/ipa-core/src/protocol/hybrid/oprf.rs +++ b/ipa-core/src/protocol/hybrid/oprf.rs @@ -216,6 +216,7 @@ mod test { }; #[test] + #[allow(clippy::too_many_lines)] fn hybrid_oprf() { run(|| async { const SHARDS: usize = 2; @@ -229,26 +230,50 @@ mod test { TestHybridRecord::TestImpression { match_key: 12345, breakdown_key: 2, + key_id: 0, + helper_origin: "HELPER_ORIGIN".to_string(), }, TestHybridRecord::TestImpression { match_key: 68362, breakdown_key: 1, + key_id: 0, + helper_origin: "HELPER_ORIGIN".to_string(), }, TestHybridRecord::TestConversion { match_key: 12345, value: 5, + key_id: 0, + helper_origin: "HELPER_ORIGIN".to_string(), + conversion_site_domain: "meta.com".to_string(), + timestamp: 100, + epsilon: 0.0, + sensitivity: 0.0, }, TestHybridRecord::TestConversion { match_key: 68362, value: 2, + key_id: 0, + helper_origin: "HELPER_ORIGIN".to_string(), + conversion_site_domain: "meta.com".to_string(), + timestamp: 102, + epsilon: 0.0, + sensitivity: 0.0, }, TestHybridRecord::TestImpression { match_key: 68362, breakdown_key: 1, + key_id: 0, + helper_origin: "HELPER_ORIGIN".to_string(), }, TestHybridRecord::TestConversion { match_key: 68362, value: 7, + key_id: 0, + helper_origin: "HELPER_ORIGIN".to_string(), + conversion_site_domain: "meta.com".to_string(), + timestamp: 104, + epsilon: 0.0, + sensitivity: 0.0, }, ]; diff --git a/ipa-core/src/query/runner/hybrid.rs b/ipa-core/src/query/runner/hybrid.rs index 0947f99a5..199ee385f 100644 --- a/ipa-core/src/query/runner/hybrid.rs +++ b/ipa-core/src/query/runner/hybrid.rs @@ -35,11 +35,8 @@ use crate::{ step::ProtocolStep::Hybrid, }, query::runner::reshard_tag::reshard_aad, - report::{ - hybrid::{ - EncryptedHybridReport, IndistinguishableHybridReport, UniqueTag, UniqueTagValidator, - }, - hybrid_info::HybridInfo, + report::hybrid::{ + EncryptedHybridReport, IndistinguishableHybridReport, UniqueTag, UniqueTagValidator, }, secret_sharing::{ replicated::semi_honest::AdditiveShare as Replicated, BitDecomposed, TransposeFrom, @@ -48,30 +45,24 @@ use crate::{ }; #[allow(dead_code)] -pub struct Query<'a, C, HV, R: PrivateKeyRegistry> { +pub struct Query { config: HybridQueryParams, key_registry: Arc, - hybrid_info: HybridInfo<'a>, phantom_data: PhantomData<(C, HV)>, } #[allow(dead_code)] -impl<'a, C, HV, R: PrivateKeyRegistry> Query<'a, C, HV, R> { - pub fn new( - query_params: HybridQueryParams, - key_registry: Arc, - hybrid_info: HybridInfo<'a>, - ) -> Self { +impl Query { + pub fn new(query_params: HybridQueryParams, key_registry: Arc) -> Self { Self { config: query_params, key_registry, - hybrid_info, phantom_data: PhantomData, } } } -impl<'a, C, HV, R> Query<'a, C, HV, R> +impl Query where C: UpgradableContext + Shuffle @@ -107,7 +98,6 @@ where let Self { config, key_registry, - hybrid_info, phantom_data: _, } = self; @@ -127,7 +117,7 @@ where iter(enc_reports.into_iter().map({ |enc_report| { let dec_report = enc_report - .decrypt(key_registry.as_ref(), &hybrid_info) + .decrypt(key_registry.as_ref()) .map_err(Into::::into); let unique_tag = UniqueTag::from_unique_bytes(&enc_report); dec_report.map(|dec_report1| (dec_report1, unique_tag)) @@ -196,7 +186,7 @@ mod tests { }, hpke::{KeyPair, KeyRegistry}, query::runner::hybrid::Query as HybridQuery, - report::{hybrid::HybridReport, hybrid_info::HybridInfo, DEFAULT_KEY_ID}, + report::{hybrid::HybridReport, DEFAULT_KEY_ID}, secret_sharing::IntoShares, test_executor::run, test_fixture::{ @@ -212,11 +202,7 @@ mod tests { query_sizes: Vec, } - fn build_buffers_from_records( - records: &[TestHybridRecord], - s: usize, - info: &HybridInfo, - ) -> BufferAndKeyRegistry { + fn build_buffers_from_records(records: &[TestHybridRecord], s: usize) -> BufferAndKeyRegistry { let mut rng = StdRng::seed_from_u64(42); let key_id = DEFAULT_KEY_ID; let key_registry = Arc::new(KeyRegistry::::random(1, &mut rng)); @@ -226,13 +212,7 @@ mod tests { for (buf, shares) in zip(&mut buffers, shares) { for (i, share) in shares.into_iter().enumerate() { share - .delimited_encrypt_to( - key_id, - key_registry.as_ref(), - info, - &mut rng, - &mut buf[i % s], - ) + .delimited_encrypt_to(key_id, key_registry.as_ref(), &mut rng, &mut buf[i % s]) .unwrap(); } } @@ -276,14 +256,11 @@ mod tests { _ => {} } - let hybrid_info = - HybridInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1).unwrap(); - let BufferAndKeyRegistry { buffers, key_registry, query_sizes, - } = build_buffers_from_records(&test_hybrid_records, SHARDS, &hybrid_info); + } = build_buffers_from_records(&test_hybrid_records, SHARDS); let world = TestWorld::>::with_shards(TestWorldConfig::default()); let contexts = world.malicious_contexts(); @@ -305,7 +282,6 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), - hybrid_info.clone(), ) .execute(ctx, query_size, input) }) @@ -343,14 +319,11 @@ mod tests { const SHARDS: usize = 2; let (test_hybrid_records, _expected) = build_hybrid_records_and_expectation(); - let hybrid_info = - HybridInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1).unwrap(); - let BufferAndKeyRegistry { mut buffers, key_registry, query_sizes, - } = build_buffers_from_records(&test_hybrid_records, SHARDS, &hybrid_info); + } = build_buffers_from_records(&test_hybrid_records, SHARDS); // this is double, since we duplicate the data below let query_sizes = query_sizes @@ -392,7 +365,6 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), - hybrid_info.clone(), ) .execute(ctx, query_size, input) }) @@ -412,14 +384,11 @@ mod tests { const SHARDS: usize = 2; let (test_hybrid_records, _expected) = build_hybrid_records_and_expectation(); - let hybrid_info = - HybridInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1).unwrap(); - let BufferAndKeyRegistry { buffers, key_registry, query_sizes, - } = build_buffers_from_records(&test_hybrid_records, SHARDS, &hybrid_info); + } = build_buffers_from_records(&test_hybrid_records, SHARDS); let world: TestWorld> = TestWorld::with_shards(TestWorldConfig::default()); @@ -442,7 +411,6 @@ mod tests { HybridQuery::<_, BA16, KeyRegistry>::new( query_params, Arc::clone(&key_registry), - hybrid_info.clone(), ) .execute(ctx, query_size, input) }) diff --git a/ipa-core/src/report/hybrid.rs b/ipa-core/src/report/hybrid.rs index 511a0418b..6b01e3bde 100644 --- a/ipa-core/src/report/hybrid.rs +++ b/ipa-core/src/report/hybrid.rs @@ -49,7 +49,7 @@ use crate::{ PublicKeyRegistry, TagSize, }, protocol::ipa_prf::{boolean_ops::expand_shared_array_in_place, shuffle::Shuffleable}, - report::hybrid_info::{HybridConversionInfo, HybridImpressionInfo, HybridInfo}, + report::hybrid_info::{HybridConversionInfo, HybridImpressionInfo}, secret_sharing::{ replicated::{semi_honest::AdditiveShare as Replicated, ReplicatedSecretSharing}, SharedValue, @@ -114,36 +114,40 @@ impl TryFrom for HybridEventType { } /// Reports for impression events are represented here. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct HybridImpressionReport where BK: SharedValue, { pub match_key: Replicated, pub breakdown_key: Replicated, + pub info: HybridImpressionInfo, } -impl Serializable for HybridImpressionReport +impl HybridImpressionReport where BK: SharedValue, Replicated: Serializable, as Serializable>::Size: Add, < as Serializable>::Size as Add< as Serializable>::Size>>:: Output: ArrayLength, { - type Size = < as Serializable>::Size as Add< as Serializable>::Size>>:: Output; - type DeserializationError = InvalidHybridReportError; - - fn serialize(&self, buf: &mut GenericArray) { + pub fn serialize(&self, buf: &mut B) { let mk_sz = as Serializable>::Size::USIZE; let bk_sz = as Serializable>::Size::USIZE; - self.match_key - .serialize(GenericArray::from_mut_slice(&mut buf[..mk_sz])); + let mut plaintext_mk = vec![0u8; mk_sz]; + self.match_key.serialize(GenericArray::from_mut_slice(&mut plaintext_mk)); + let mut plaintext_bk = vec![0u8; bk_sz]; + self.breakdown_key.serialize(GenericArray::from_mut_slice(&mut plaintext_bk)); - self.breakdown_key - .serialize(GenericArray::from_mut_slice(&mut buf[mk_sz..mk_sz + bk_sz])); + buf.put_slice(&plaintext_mk); + buf.put_slice(&plaintext_bk); + buf.put_slice(&self.info.to_bytes()); } - fn deserialize(buf: &GenericArray) -> Result { + + /// # Errors + /// If there is a problem deserializing the report. + pub fn deserialize(buf: &Bytes) -> Result { let mk_sz = as Serializable>::Size::USIZE; let bk_sz = as Serializable>::Size::USIZE; let match_key = @@ -151,7 +155,14 @@ where let breakdown_key = Replicated::::deserialize(GenericArray::from_slice(&buf[mk_sz..mk_sz + bk_sz])) .map_err(|e| InvalidHybridReportError::DeserializationError("breakdown_key", e.into()))?; - Ok(Self { match_key, breakdown_key }) + let info = HybridImpressionInfo::from_bytes(&buf[mk_sz + bk_sz..])?; + + Ok(Self { match_key, breakdown_key, info }) + } + + #[must_use] + pub fn serialized_len() -> usize { + Replicated::::size() + Replicated::::size() } } @@ -166,23 +177,29 @@ where /// # Panics /// If report length does not fit in `u16`. - pub fn encrypted_len(&self) -> u16 { - let len = EncryptedHybridImpressionReport::::SITE_DOMAIN_OFFSET; + pub fn ciphertext_len(&self) -> u16 { + let len = EncryptedHybridImpressionReport::::INFO_OFFSET; len.try_into().unwrap() } + /// # Panics + /// If report length does not fit in `u16`. + pub fn encrypted_len(&self) -> u16 { + // Todo: get this more efficiently + self.ciphertext_len() + u16::try_from(self.info.to_bytes().len()).unwrap() + } + /// # Errors /// If there is a problem encrypting the report. pub fn delimited_encrypt_to( &self, key_id: KeyIdentifier, key_registry: &impl PublicKeyRegistry, - info: &HybridImpressionInfo, rng: &mut R, out: &mut B, ) -> Result<(), InvalidHybridReportError> { out.put_u16_le(self.encrypted_len()); - self.encrypt_to(key_id, key_registry, info, rng, out) + self.encrypt_to(key_id, key_registry, rng, out) } /// # Errors @@ -191,11 +208,10 @@ where &self, key_id: KeyIdentifier, key_registry: &impl PublicKeyRegistry, - info: &HybridImpressionInfo, rng: &mut R, ) -> Result, InvalidHybridReportError> { let mut out = Vec::with_capacity(usize::from(self.encrypted_len())); - self.encrypt_to(key_id, key_registry, info, rng, &mut out)?; + self.encrypt_to(key_id, key_registry, rng, &mut out)?; debug_assert_eq!(out.len(), usize::from(self.encrypted_len())); Ok(out) } @@ -206,7 +222,6 @@ where &self, key_id: KeyIdentifier, key_registry: &impl PublicKeyRegistry, - info: &HybridImpressionInfo, rng: &mut R, out: &mut B, ) -> Result<(), InvalidHybridReportError> { @@ -218,18 +233,19 @@ where .serialize(GenericArray::from_mut_slice(&mut plaintext_btt[..])); let pk = key_registry.public_key(key_id).ok_or(CryptError::NoSuchKey(key_id))?; + let info_bytes = self.info.to_bytes(); let (encap_key_mk, ciphertext_mk, tag_mk) = seal_in_place( pk, plaintext_mk.as_mut(), - &info.to_bytes(), + &info_bytes, rng, )?; let (encap_key_btt, ciphertext_btt, tag_btt) = seal_in_place( pk, plaintext_btt.as_mut(), - &info.to_bytes(), + &info_bytes, rng, )?; @@ -240,51 +256,61 @@ where out.put_slice(ciphertext_btt); out.put_slice(&tag_btt.to_bytes()); out.put_slice(&[key_id]); + out.put_slice(&info_bytes); Ok(()) } } /// Reports for conversion events are represented here. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub struct HybridConversionReport where V: SharedValue, { pub match_key: Replicated, pub value: Replicated, + pub info: HybridConversionInfo, } -impl Serializable for HybridConversionReport +impl HybridConversionReport where V: SharedValue, Replicated: Serializable, as Serializable>::Size: Add, < as Serializable>::Size as Add< as Serializable>::Size>>:: Output: ArrayLength, { - type Size = < as Serializable>::Size as Add< as Serializable>::Size>>:: Output; - type DeserializationError = InvalidHybridReportError; - - fn serialize(&self, buf: &mut GenericArray) { + pub fn serialize(&self, buf: &mut B) { let mk_sz = as Serializable>::Size::USIZE; let v_sz = as Serializable>::Size::USIZE; - self.match_key - .serialize(GenericArray::from_mut_slice(&mut buf[..mk_sz])); + let mut plaintext_mk = vec![0u8; mk_sz]; + self.match_key.serialize(GenericArray::from_mut_slice(&mut plaintext_mk)); + let mut plaintext_v = vec![0u8; v_sz]; + self.value.serialize(GenericArray::from_mut_slice(&mut plaintext_v)); - self.value - .serialize(GenericArray::from_mut_slice(&mut buf[mk_sz..mk_sz + v_sz])); + buf.put_slice(&plaintext_mk); + buf.put_slice(&plaintext_v); + buf.put_slice(&self.info.to_bytes()); } - fn deserialize(buf: &GenericArray) -> Result { + + /// # Errors + /// If there is a problem deserializing the report. + pub fn deserialize(buf: &Bytes) -> Result { let mk_sz = as Serializable>::Size::USIZE; let v_sz = as Serializable>::Size::USIZE; let match_key = - Replicated::::deserialize(GenericArray::from_slice(&buf[..mk_sz])) - .map_err(|e| InvalidHybridReportError::DeserializationError("match_key", e.into()))?; + Replicated::::deserialize_infallible(GenericArray::from_slice(&buf[..mk_sz])); let value = Replicated::::deserialize(GenericArray::from_slice(&buf[mk_sz..mk_sz + v_sz])) .map_err(|e| InvalidHybridReportError::DeserializationError("breakdown_key", e.into()))?; - Ok(Self { match_key, value }) + let info = HybridConversionInfo::from_bytes(&buf[mk_sz + v_sz..])?; + Ok(Self { match_key, value, info }) + } + + #[must_use] + pub fn serialized_len() -> usize { + Replicated::::size() + Replicated::::size() } } @@ -299,23 +325,29 @@ where /// # Panics /// If report length does not fit in `u16`. - pub fn encrypted_len(&self) -> u16 { - let len = EncryptedHybridConversionReport::::SITE_DOMAIN_OFFSET; + pub fn ciphertext_len(&self) -> u16 { + let len = EncryptedHybridConversionReport::::INFO_OFFSET; len.try_into().unwrap() } + /// # Panics + /// If report length does not fit in `u16`. + pub fn encrypted_len(&self) -> u16 { + // Todo: get this more efficiently + self.ciphertext_len() + u16::try_from(self.info.to_bytes().len()).unwrap() + } + /// # Errors /// If there is a problem encrypting the report. pub fn delimited_encrypt_to( &self, key_id: KeyIdentifier, key_registry: &impl PublicKeyRegistry, - info: &HybridConversionInfo, rng: &mut R, out: &mut B, ) -> Result<(), InvalidHybridReportError> { out.put_u16_le(self.encrypted_len()); - self.encrypt_to(key_id, key_registry, info, rng, out) + self.encrypt_to(key_id, key_registry, rng, out) } /// # Errors @@ -324,11 +356,10 @@ where &self, key_id: KeyIdentifier, key_registry: &impl PublicKeyRegistry, - info: &HybridConversionInfo, rng: &mut R, ) -> Result, InvalidHybridReportError> { let mut out = Vec::with_capacity(usize::from(self.encrypted_len())); - self.encrypt_to(key_id, key_registry, info, rng, &mut out)?; + self.encrypt_to(key_id, key_registry, rng, &mut out)?; debug_assert_eq!(out.len(), usize::from(self.encrypted_len())); Ok(out) } @@ -339,7 +370,6 @@ where &self, key_id: KeyIdentifier, key_registry: &impl PublicKeyRegistry, - info: &HybridConversionInfo, rng: &mut R, out: &mut B, ) -> Result<(), InvalidHybridReportError> { @@ -352,18 +382,19 @@ where .serialize(GenericArray::from_mut_slice(&mut plaintext_btt[..])); let pk = key_registry.public_key(key_id).ok_or(CryptError::NoSuchKey(key_id))?; + let info_bytes = self.info.to_bytes(); let (encap_key_mk, ciphertext_mk, tag_mk) = seal_in_place( pk, plaintext_mk.as_mut(), - &info.to_bytes(), + &info_bytes, rng, )?; let (encap_key_btt, ciphertext_btt, tag_btt) = seal_in_place( pk, plaintext_btt.as_mut(), - &info.to_bytes(), + &info_bytes, rng, )?; @@ -373,14 +404,15 @@ where out.put_slice(&encap_key_btt.to_bytes()); out.put_slice(ciphertext_btt); out.put_slice(&tag_btt.to_bytes()); - out.put_slice(&[key_id]); + out.put_slice(&[key_id]); //todo: this is also in the info + out.put_slice(&info_bytes); Ok(()) } } /// This enum contains both report types, impression and conversion. -#[derive(Clone, Debug, Eq, PartialEq)] +#[derive(Clone, Debug, PartialEq)] pub enum HybridReport where BK: SharedValue, @@ -420,7 +452,6 @@ where &self, key_id: KeyIdentifier, key_registry: &impl PublicKeyRegistry, - info: &HybridInfo, rng: &mut R, out: &mut B, ) -> Result<(), InvalidHybridReportError> { @@ -428,12 +459,12 @@ where HybridReport::Impression(impression_report) => { out.put_u16_le(self.encrypted_len()); out.put_u8(HybridEventType::Impression as u8); - impression_report.encrypt_to(key_id, key_registry, &info.impression, rng, out) + impression_report.encrypt_to(key_id, key_registry, rng, out) }, HybridReport::Conversion(conversion_report) => { out.put_u16_le(self.encrypted_len()); out.put_u8(HybridEventType::Conversion as u8); - conversion_report.encrypt_to(key_id, key_registry, &info.conversion, rng, out) + conversion_report.encrypt_to(key_id, key_registry, rng, out) }, } } @@ -444,15 +475,14 @@ where &self, key_id: KeyIdentifier, key_registry: &impl PublicKeyRegistry, - info: &HybridInfo, rng: &mut R, ) -> Result, InvalidHybridReportError> { match self { HybridReport::Impression(impression_report) => { - impression_report.encrypt(key_id, key_registry, &info.impression, rng).map(|v| once(HybridEventType::Impression as u8).chain(v).collect()) + impression_report.encrypt(key_id, key_registry, rng).map(|v| once(HybridEventType::Impression as u8).chain(v).collect()) }, HybridReport::Conversion(conversion_report) => { - conversion_report.encrypt(key_id, key_registry, &info.conversion, rng).map(|v| once(HybridEventType::Conversion as u8).chain(v).collect()) + conversion_report.encrypt(key_id, key_registry, rng).map(|v| once(HybridEventType::Conversion as u8).chain(v).collect()) }, } } @@ -463,18 +493,17 @@ where &self, key_id: KeyIdentifier, key_registry: &impl PublicKeyRegistry, - info: &HybridInfo, rng: &mut R, out: &mut B, ) -> Result<(), InvalidHybridReportError> { match self { HybridReport::Impression(impression_report) =>{ out.put_u8(HybridEventType::Impression as u8); - impression_report.encrypt_to(key_id, key_registry, &info.impression, rng, out) + impression_report.encrypt_to(key_id, key_registry, rng, out) }, HybridReport::Conversion(conversion_report) => { out.put_u8(HybridEventType::Conversion as u8); - conversion_report.encrypt_to(key_id, key_registry, &info.conversion, rng, out) + conversion_report.encrypt_to(key_id, key_registry, rng, out) }, } } @@ -507,7 +536,7 @@ where const KEY_IDENTIFIER_OFFSET: usize = (Self::CIPHERTEXT_BTT_OFFSET + TagSize::USIZE + Replicated::::size()); - const SITE_DOMAIN_OFFSET: usize = Self::KEY_IDENTIFIER_OFFSET + 1; + const INFO_OFFSET: usize = Self::KEY_IDENTIFIER_OFFSET + 1; pub fn encap_key_mk(&self) -> &[u8] { &self.data[Self::ENCAP_KEY_MK_OFFSET..Self::CIPHERTEXT_MK_OFFSET] @@ -532,10 +561,10 @@ where /// ## Errors /// If the report contents are invalid. pub fn from_bytes(bytes: Bytes) -> Result { - if bytes.len() < Self::SITE_DOMAIN_OFFSET { + if bytes.len() < Self::INFO_OFFSET { return Err(InvalidHybridReportError::Length( bytes.len(), - Self::SITE_DOMAIN_OFFSET, + Self::INFO_OFFSET, )); } Ok(Self { @@ -553,7 +582,6 @@ where pub fn decrypt( &self, key_registry: &P, - info: &HybridImpressionInfo, ) -> Result, InvalidHybridReportError> { type CTMKLength = Sum< as Serializable>::Size, TagSize>; type CTBTTLength = < as Serializable>::Size as Add>::Output; @@ -563,6 +591,10 @@ where let sk = key_registry .private_key(self.key_id()) .ok_or(CryptError::NoSuchKey(self.key_id()))?; + let info = + HybridImpressionInfo::from_bytes(&self.data[Self::INFO_OFFSET..]).map_err(|e| { + InvalidHybridReportError::DeserializationError("HybridImpressionInfo", e.into()) + })?; let plaintext_mk = open_in_place(sk, self.encap_key_mk(), &mut ct_mk, &info.to_bytes())?; let mut ct_btt: GenericArray> = GenericArray::from_slice(self.btt_ciphertext()).clone(); @@ -577,6 +609,7 @@ where .map_err(|e| { InvalidHybridReportError::DeserializationError("is_trigger", e.into()) })?, + info, }) } } @@ -605,7 +638,8 @@ where const KEY_IDENTIFIER_OFFSET: usize = (Self::CIPHERTEXT_BTT_OFFSET + TagSize::USIZE + Replicated::::size()); - const SITE_DOMAIN_OFFSET: usize = Self::KEY_IDENTIFIER_OFFSET + 1; + // Todo: determine a minimum size for Info which can be used for debugging + const INFO_OFFSET: usize = Self::KEY_IDENTIFIER_OFFSET + 1; pub fn encap_key_mk(&self) -> &[u8] { &self.data[Self::ENCAP_KEY_MK_OFFSET..Self::CIPHERTEXT_MK_OFFSET] @@ -630,10 +664,10 @@ where /// ## Errors /// If the report contents are invalid. pub fn from_bytes(bytes: Bytes) -> Result { - if bytes.len() < Self::SITE_DOMAIN_OFFSET { + if bytes.len() < Self::INFO_OFFSET { return Err(InvalidHybridReportError::Length( bytes.len(), - Self::SITE_DOMAIN_OFFSET, + Self::INFO_OFFSET, )); } Ok(Self { @@ -651,7 +685,6 @@ where pub fn decrypt( &self, key_registry: &P, - info: &HybridConversionInfo, ) -> Result, InvalidHybridReportError> { type CTMKLength = Sum< as Serializable>::Size, TagSize>; type CTBTTLength = < as Serializable>::Size as Add>::Output; @@ -661,10 +694,14 @@ where let sk = key_registry .private_key(self.key_id()) .ok_or(CryptError::NoSuchKey(self.key_id()))?; + let info = + HybridConversionInfo::from_bytes(&self.data[Self::INFO_OFFSET..]).map_err(|e| { + InvalidHybridReportError::DeserializationError("HybridConversionInfo", e.into()) + })?; + let plaintext_mk = open_in_place(sk, self.encap_key_mk(), &mut ct_mk, &info.to_bytes())?; let mut ct_btt: GenericArray> = GenericArray::from_slice(self.btt_ciphertext()).clone(); - let plaintext_btt = open_in_place(sk, self.encap_key_btt(), &mut ct_btt, &info.to_bytes())?; Ok(HybridConversionReport:: { @@ -674,6 +711,7 @@ where value: Replicated::::deserialize(GenericArray::from_slice(plaintext_btt)).map_err( |e| InvalidHybridReportError::DeserializationError("trigger_value", e.into()), )?, + info, }) } } @@ -1049,14 +1087,13 @@ where pub fn decrypt( &self, key_registry: &P, - info: &HybridInfo, ) -> Result, InvalidHybridReportError> { match self { EncryptedHybridReport::Impression(impression_report) => Ok(HybridReport::Impression( - impression_report.decrypt(key_registry, &info.impression)?, + impression_report.decrypt(key_registry)?, )), EncryptedHybridReport::Conversion(conversion_report) => Ok(HybridReport::Conversion( - conversion_report.decrypt(key_registry, &info.conversion)?, + conversion_report.decrypt(key_registry)?, )), } } @@ -1197,8 +1234,8 @@ impl UniqueTagValidator { #[cfg(all(test, unit_test))] mod test { + use bytes::Bytes; use rand::Rng; - use typenum::Unsigned; use super::{ EncryptedHybridImpressionReport, EncryptedHybridReport, GenericArray, @@ -1214,8 +1251,8 @@ mod test { }, hpke::{KeyPair, KeyRegistry}, report::{ - hybrid::{EncryptedHybridConversionReport, HybridEventType, NonAsciiStringError, BA64}, - hybrid_info::{HybridConversionInfo, HybridImpressionInfo, HybridInfo}, + hybrid::{EncryptedHybridConversionReport, HybridEventType, NonAsciiStringError}, + hybrid_info::{HybridConversionInfo, HybridImpressionInfo}, }, secret_sharing::replicated::{ semi_honest::{AdditiveShare as Replicated, AdditiveShare}, @@ -1233,12 +1270,22 @@ mod test { HybridReport::Impression(HybridImpressionReport:: { match_key: AdditiveShare::new(rng.gen(), rng.gen()), breakdown_key: AdditiveShare::new(rng.gen(), rng.gen()), + info: HybridImpressionInfo::new(0, "HelperOrigin").unwrap(), }) } HybridEventType::Conversion => { HybridReport::Conversion(HybridConversionReport:: { match_key: AdditiveShare::new(rng.gen(), rng.gen()), value: AdditiveShare::new(rng.gen(), rng.gen()), + info: HybridConversionInfo::new( + 0, + "HelperOrigin", + "https://www.example2.com", + rng.gen(), + 0.0, + 0.0, + ) + .unwrap(), }) } } @@ -1265,6 +1312,15 @@ mod test { let conversion_report = HybridConversionReport:: { match_key: AdditiveShare::new(rng.gen(), rng.gen()), value: AdditiveShare::new(rng.gen(), rng.gen()), + info: HybridConversionInfo::new( + 0, + "HelperOrigin", + "https://www.example2.com", + 1_234_567, + 0.0, + 0.0, + ) + .unwrap(), }; let indistinguishable_report: IndistinguishableHybridReport = conversion_report.clone().into(); @@ -1294,6 +1350,7 @@ mod test { let impression_report = HybridImpressionReport:: { match_key: AdditiveShare::new(rng.gen(), rng.gen()), breakdown_key: AdditiveShare::new(rng.gen(), rng.gen()), + info: HybridImpressionInfo::new(0, "HelperOrigin").unwrap(), }; let indistinguishable_report: IndistinguishableHybridReport = impression_report.clone().into(); @@ -1343,14 +1400,13 @@ mod test { let hybrid_impression_report = HybridImpressionReport:: { match_key: AdditiveShare::new(rng.gen(), rng.gen()), breakdown_key: AdditiveShare::new(rng.gen(), rng.gen()), + info: HybridImpressionInfo::new(0, "HelperOrigin").unwrap(), }; let mut hybrid_impression_report_bytes = - [0u8; as Serializable>::Size::USIZE]; - hybrid_impression_report.serialize(GenericArray::from_mut_slice( - &mut hybrid_impression_report_bytes[..], - )); + Vec::with_capacity(HybridImpressionReport::::serialized_len()); + hybrid_impression_report.serialize(&mut hybrid_impression_report_bytes); let hybrid_impression_report2 = HybridImpressionReport::::deserialize( - GenericArray::from_mut_slice(&mut hybrid_impression_report_bytes[..]), + &Bytes::copy_from_slice(&hybrid_impression_report_bytes[..]), ) .unwrap(); assert_eq!(hybrid_impression_report, hybrid_impression_report2); @@ -1363,111 +1419,48 @@ mod test { let hybrid_conversion_report = HybridConversionReport:: { match_key: AdditiveShare::new(rng.gen(), rng.gen()), value: AdditiveShare::new(rng.gen(), rng.gen()), + info: HybridConversionInfo::new( + 0, + "HelperOrigin", + "https://www.example2.com", + 1_234_567, + 0.0, + 0.0, + ) + .unwrap(), }; let mut hybrid_conversion_report_bytes = - [0u8; as Serializable>::Size::USIZE]; - hybrid_conversion_report.serialize(GenericArray::from_mut_slice( - &mut hybrid_conversion_report_bytes[..], - )); + Vec::with_capacity(HybridImpressionReport::::serialized_len()); + hybrid_conversion_report.serialize(&mut hybrid_conversion_report_bytes); let hybrid_conversion_report2 = HybridConversionReport::::deserialize( - GenericArray::from_mut_slice(&mut hybrid_conversion_report_bytes[..]), + &Bytes::copy_from_slice(&hybrid_conversion_report_bytes[..]), ) .unwrap(); assert_eq!(hybrid_conversion_report, hybrid_conversion_report2); }); } - #[test] - fn constant_serialization_hybrid_impression() { - let hybrid_report = HybridImpressionReport::::deserialize(GenericArray::from_slice( - &hex::decode("4123a6e38ef1d6d9785c948797cb744d38f4").unwrap(), - )) - .unwrap(); - - let match_key = AdditiveShare::::deserialize(GenericArray::from_slice( - &hex::decode("4123a6e38ef1d6d9785c948797cb744d").unwrap(), - )) - .unwrap(); - let breakdown_key = AdditiveShare::::deserialize(GenericArray::from_slice( - &hex::decode("38f4").unwrap(), - )) - .unwrap(); - - assert_eq!( - hybrid_report, - HybridImpressionReport:: { - match_key, - breakdown_key - } - ); - - let mut hybrid_impression_report_bytes = - [0u8; as Serializable>::Size::USIZE]; - hybrid_report.serialize(GenericArray::from_mut_slice( - &mut hybrid_impression_report_bytes[..], - )); - - assert_eq!( - hybrid_impression_report_bytes.to_vec(), - hex::decode("4123a6e38ef1d6d9785c948797cb744d38f4").unwrap() - ); - } - - #[test] - fn constant_serialization_hybrid_conversion() { - let hybrid_report = HybridConversionReport::::deserialize(GenericArray::from_slice( - &hex::decode("4123a6e38ef1d6d9785c948797cb744d0203").unwrap(), - )) - .unwrap(); - - let match_key = AdditiveShare::::deserialize(GenericArray::from_slice( - &hex::decode("4123a6e38ef1d6d9785c948797cb744d").unwrap(), - )) - .unwrap(); - let value = AdditiveShare::::deserialize(GenericArray::from_slice( - &hex::decode("0203").unwrap(), - )) - .unwrap(); - - assert_eq!( - hybrid_report, - HybridConversionReport:: { match_key, value } - ); - - let mut hybrid_conversion_report_bytes = - [0u8; as Serializable>::Size::USIZE]; - hybrid_report.serialize(GenericArray::from_mut_slice( - &mut hybrid_conversion_report_bytes[..], - )); - - assert_eq!( - hybrid_conversion_report_bytes.to_vec(), - hex::decode("4123a6e38ef1d6d9785c948797cb744d0203").unwrap() - ); - } - #[test] fn enc_dec_roundtrip_hybrid_impression() { run_random(|mut rng| async move { + let key_registry = KeyRegistry::::random(1, &mut rng); + let key_id = 0; + let hybrid_impression_report = HybridImpressionReport:: { match_key: AdditiveShare::new(rng.gen(), rng.gen()), breakdown_key: AdditiveShare::new(rng.gen(), rng.gen()), + info: HybridImpressionInfo::new(key_id, HELPER_ORIGIN).unwrap(), }; - let key_registry = KeyRegistry::::random(1, &mut rng); - let key_id = 0; - - let info = HybridImpressionInfo::new(key_id, HELPER_ORIGIN).unwrap(); - let enc_report_bytes = hybrid_impression_report - .encrypt(key_id, &key_registry, &info, &mut rng) + .encrypt(key_id, &key_registry, &mut rng) .unwrap(); let enc_report = EncryptedHybridImpressionReport::::from_bytes(enc_report_bytes.into()) .unwrap(); let dec_report: HybridImpressionReport = - enc_report.decrypt(&key_registry, &info).unwrap(); + enc_report.decrypt(&key_registry).unwrap(); assert_eq!(dec_report, hybrid_impression_report); }); @@ -1479,30 +1472,29 @@ mod test { let hybrid_conversion_report = HybridConversionReport:: { match_key: AdditiveShare::new(rng.gen(), rng.gen()), value: AdditiveShare::new(rng.gen(), rng.gen()), + info: HybridConversionInfo::new( + 0, + HELPER_ORIGIN, + "meta.com", + 1_729_707_432, + 5.0, + 1.1, + ) + .unwrap(), }; let key_registry = KeyRegistry::::random(1, &mut rng); let key_id = 0; - let info = HybridConversionInfo::new( - key_id, - HELPER_ORIGIN, - "meta.com", - 1_729_707_432, - 5.0, - 1.1, - ) - .unwrap(); - let enc_report_bytes = hybrid_conversion_report - .encrypt(key_id, &key_registry, &info, &mut rng) + .encrypt(key_id, &key_registry, &mut rng) .unwrap(); let enc_report = EncryptedHybridConversionReport::::from_bytes(enc_report_bytes.into()) .unwrap(); let dec_report: HybridConversionReport = - enc_report.decrypt(&key_registry, &info).unwrap(); + enc_report.decrypt(&key_registry).unwrap(); assert_eq!(dec_report, hybrid_conversion_report); }); @@ -1517,17 +1509,13 @@ mod test { let key_registry = KeyRegistry::::random(1, &mut rng); let key_id = 0; - let info = HybridInfo::new(key_id, HELPER_ORIGIN, "meta.com", 1_729_707_432, 5.0, 1.1) - .unwrap(); - let enc_report_bytes = hybrid_report - .encrypt(key_id, &key_registry, &info, &mut rng) + .encrypt(key_id, &key_registry, &mut rng) .unwrap(); let enc_report = EncryptedHybridReport::::from_bytes(enc_report_bytes.into()).unwrap(); - let dec_report: HybridReport = - enc_report.decrypt(&key_registry, &info).unwrap(); + let dec_report: HybridReport = enc_report.decrypt(&key_registry).unwrap(); assert_eq!(dec_report, hybrid_report); }); @@ -1539,16 +1527,22 @@ mod test { let hybrid_conversion_report = HybridConversionReport:: { match_key: AdditiveShare::new(rng.gen(), rng.gen()), value: AdditiveShare::new(rng.gen(), rng.gen()), + info: HybridConversionInfo::new( + 0, + "HELPER_ORIGIN", + "meta.com", + 1_729_707_432, + 5.0, + 1.1, + ) + .unwrap(), }; let key_registry = KeyRegistry::::random(1, &mut rng); let key_id = 0; - let info = - HybridInfo::new(0, "HELPER_ORIGIN", "meta.com", 1_729_707_432, 5.0, 1.1).unwrap(); - let enc_report_bytes = hybrid_conversion_report - .encrypt(key_id, &key_registry, &info.conversion, &mut rng) + .encrypt(key_id, &key_registry, &mut rng) .unwrap(); let mut enc_report_bytes2 = enc_report_bytes.clone(); @@ -1557,7 +1551,7 @@ mod test { EncryptedHybridConversionReport::::from_bytes(enc_report_bytes.into()) .unwrap(); let dec_report: HybridConversionReport = - enc_report.decrypt(&key_registry, &info.conversion).unwrap(); + enc_report.decrypt(&key_registry).unwrap(); assert_eq!(dec_report, hybrid_conversion_report); // Prepend a byte to the ciphertext to mark it as a ConversionReport @@ -1571,14 +1565,13 @@ mod test { match enc_report2 { EncryptedHybridReport::Impression(_) => panic!("Expected conversion report"), EncryptedHybridReport::Conversion(enc_report_conv) => { - let dec_report2: HybridConversionReport = enc_report_conv - .decrypt(&key_registry, &info.conversion) - .unwrap(); + let dec_report2: HybridConversionReport = + enc_report_conv.decrypt(&key_registry).unwrap(); assert_eq!(dec_report2, hybrid_conversion_report); } } // Case 2: Decrypt directly - let dec_report3 = enc_report3.decrypt(&key_registry, &info).unwrap(); + let dec_report3 = enc_report3.decrypt(&key_registry).unwrap(); assert_eq!( dec_report3, HybridReport::Conversion(hybrid_conversion_report) diff --git a/ipa-core/src/report/hybrid_info.rs b/ipa-core/src/report/hybrid_info.rs index 5dc2950f2..1fdee599d 100644 --- a/ipa-core/src/report/hybrid_info.rs +++ b/ipa-core/src/report/hybrid_info.rs @@ -1,11 +1,14 @@ -use crate::report::{hybrid::NonAsciiStringError, KeyIdentifier}; +use crate::report::{ + hybrid::{InvalidHybridReportError, NonAsciiStringError}, + KeyIdentifier, +}; const DOMAIN: &str = "private-attribution"; -#[derive(Clone, Debug)] +#[derive(Clone, Debug, PartialEq)] pub struct HybridImpressionInfo { pub key_id: KeyIdentifier, - pub helper_origin: &'static str, + pub helper_origin: String, } impl HybridImpressionInfo { @@ -13,10 +16,7 @@ impl HybridImpressionInfo { /// /// ## Errors /// if helper or site origin is not a valid ASCII string. - pub fn new( - key_id: KeyIdentifier, - helper_origin: &'static str, - ) -> Result { + pub fn new(key_id: KeyIdentifier, helper_origin: &str) -> Result { // If the types of errors returned from this function change, then the validation in // `EncryptedReport::from_bytes` may need to change as well. if !helper_origin.is_ascii() { @@ -25,13 +25,14 @@ impl HybridImpressionInfo { Ok(Self { key_id, - helper_origin, + helper_origin: helper_origin.to_string(), }) } // Converts this instance into an owned byte slice that can further be used to create HPKE // sender or receiver context. - pub(super) fn to_bytes(&self) -> Box<[u8]> { + #[must_use] + pub fn to_bytes(&self) -> Box<[u8]> { let info_len = DOMAIN.len() + self.helper_origin.len() + 2 // delimiters(?) @@ -49,27 +50,66 @@ impl HybridImpressionInfo { r.into_boxed_slice() } + + /// ## Errors + /// If deserialization fails. + /// ## Panics + /// If not enough delimiters are found in the input bytes. + pub fn from_bytes(bytes: &[u8]) -> Result { + let mut pos = 0; + + let domain = std::str::from_utf8(&bytes[pos..pos + DOMAIN.len()]).map_err(|e| { + InvalidHybridReportError::DeserializationError("HybridImpressionInfo: domain", e.into()) + })?; + assert!( + domain == DOMAIN, + "HPKE Info domain does not match hardcoded domain" + ); + pos += DOMAIN.len() + 1; + + let delimiter_pos = bytes[pos..] + .iter() + .position(|&b| b == 0) + .unwrap_or_else(|| panic!("not enough delimiters for HybridImpressionInfo")); + let helper_origin = + String::from_utf8(bytes[pos..pos + delimiter_pos].to_vec()).map_err(|e| { + InvalidHybridReportError::DeserializationError( + "HybridImpressionInfo: helper_origin", + e.into(), + ) + })?; + pos += delimiter_pos; + debug_assert!(pos + 2 == bytes.len(), "{}", format!("bytes for HybridImpressionInfo::from_bytes has incorrect length. Expected: {}, Actual: {}", pos + 2, bytes.len()).to_string()); + pos += 1; + + let key_id = bytes[pos]; + + Ok(Self { + key_id, + helper_origin, + }) + } } -#[derive(Clone, Debug)] -pub struct HybridConversionInfo<'a> { +#[derive(Clone, Debug, PartialEq)] +pub struct HybridConversionInfo { pub key_id: KeyIdentifier, - pub helper_origin: &'static str, - pub conversion_site_domain: &'a str, + pub helper_origin: String, + pub conversion_site_domain: String, pub timestamp: u64, pub epsilon: f64, pub sensitivity: f64, } -impl<'a> HybridConversionInfo<'a> { +impl HybridConversionInfo { /// Creates a new instance. /// /// ## Errors /// if helper or site origin is not a valid ASCII string. pub fn new( key_id: KeyIdentifier, - helper_origin: &'static str, - conversion_site_domain: &'a str, + helper_origin: &str, + conversion_site_domain: &str, timestamp: u64, epsilon: f64, sensitivity: f64, @@ -86,8 +126,8 @@ impl<'a> HybridConversionInfo<'a> { Ok(Self { key_id, - helper_origin, - conversion_site_domain, + helper_origin: helper_origin.to_string(), + conversion_site_domain: conversion_site_domain.to_string(), timestamp, epsilon, sensitivity, @@ -96,7 +136,8 @@ impl<'a> HybridConversionInfo<'a> { // Converts this instance into an owned byte slice that can further be used to create HPKE // sender or receiver context. - pub(super) fn to_bytes(&self) -> Box<[u8]> { + #[must_use] + pub fn to_bytes(&self) -> Box<[u8]> { let info_len = DOMAIN.len() + self.helper_origin.len() + self.conversion_site_domain.len() @@ -123,22 +164,83 @@ impl<'a> HybridConversionInfo<'a> { r.into_boxed_slice() } + + /// ## Errors + /// If deserialization fails. + /// ## Panics + /// If not enough delimiters are found in the input bytes. + pub fn from_bytes(bytes: &[u8]) -> Result { + let mut pos = 0; + + let domain = std::str::from_utf8(&bytes[pos..pos + DOMAIN.len()]).map_err(|e| { + InvalidHybridReportError::DeserializationError("HybridConversionInfo: domain", e.into()) + })?; + assert!( + domain == DOMAIN, + "HPKE Info domain does not match hardcoded domain" + ); + pos += DOMAIN.len() + 1; + + let mut delimiter_pos = bytes[pos..] + .iter() + .position(|&b| b == 0) + .unwrap_or_else(|| panic!("not enough delimiters for HybridConversionInfo")); + let helper_origin = + String::from_utf8(bytes[pos..pos + delimiter_pos].to_vec()).map_err(|e| { + InvalidHybridReportError::DeserializationError( + "HybridConversionInfo: helper_origin", + e.into(), + ) + })?; + pos += delimiter_pos + 1; + + delimiter_pos = bytes[pos..] + .iter() + .position(|&b| b == 0) + .unwrap_or_else(|| panic!("not enough delimiters for HybridConversionInfo")); + let conversion_site_domain = String::from_utf8(bytes[pos..pos + delimiter_pos].to_vec()) + .map_err(|e| { + InvalidHybridReportError::DeserializationError( + "HybridConversionInfo: conversion_site_domain", + e.into(), + ) + })?; + pos += delimiter_pos + 1; + debug_assert!(pos + 25 == bytes.len(), "{}", format!("bytes for HybridConversionInfo::from_bytes has incorrect length. Expected: {}, Actual: {}", pos + 25, bytes.len()).to_string()); + + let key_id = bytes[pos]; + pos += 1; + let timestamp = u64::from_be_bytes(bytes[pos..pos + 8].try_into().unwrap()); + pos += 8; + let epsilon = f64::from_be_bytes(bytes[pos..pos + 8].try_into().unwrap()); + pos += 8; + let sensitivity = f64::from_be_bytes(bytes[pos..pos + 8].try_into().unwrap()); + + Ok(Self { + key_id, + helper_origin, + conversion_site_domain, + timestamp, + epsilon, + sensitivity, + }) + } } #[derive(Clone, Debug)] -pub struct HybridInfo<'a> { +pub struct HybridInfo { pub impression: HybridImpressionInfo, - pub conversion: HybridConversionInfo<'a>, + pub conversion: HybridConversionInfo, } -impl HybridInfo<'_> { +impl HybridInfo { /// Creates a new instance. /// ## Errors /// if helper or site origin is not a valid ASCII string. pub fn new( key_id: KeyIdentifier, - helper_origin: &'static str, - conversion_site_domain: &'static str, + helper_origin: &str, + conversion_site_domain: &str, timestamp: u64, epsilon: f64, sensitivity: f64, @@ -157,4 +259,75 @@ impl HybridInfo<'_> { conversion, }) } + + #[must_use] + pub fn to_bytes(&self) -> Box<[u8]> { + self.conversion.to_bytes() + } + + /// ## Errors + /// If deserialization fails. + pub fn from_bytes(bytes: &[u8]) -> Result { + let conversion = HybridConversionInfo::from_bytes(bytes)?; + let impression = HybridImpressionInfo { + key_id: conversion.key_id, + helper_origin: conversion.helper_origin.clone(), + }; + Ok(Self { + impression, + conversion, + }) + } +} + +#[cfg(all(test, unit_test))] +mod test { + use super::*; + + #[test] + fn test_hybrid_impression_serialization() { + let info = HybridImpressionInfo::new(0, "https://www.example.com").unwrap(); + let bytes = info.to_bytes(); + let info2 = HybridImpressionInfo::from_bytes(&bytes).unwrap(); + assert_eq!(info.to_bytes(), info2.to_bytes()); + } + + #[test] + #[allow(clippy::float_cmp)] + fn test_hybrid_conversion_serialization() { + let info = HybridConversionInfo::new( + 0, + "https://www.example.com", + "https://www.example2.com", + 1_234_567, + 1.151, + 0.95, + ) + .unwrap(); + let bytes = info.to_bytes(); + let info2 = HybridConversionInfo::from_bytes(&bytes).unwrap(); + assert_eq!(info2.key_id, 0); + assert_eq!(info2.helper_origin, "https://www.example.com"); + assert_eq!(info2.conversion_site_domain, "https://www.example2.com"); + assert_eq!(info2.timestamp, 1_234_567); + assert_eq!(info2.epsilon, 1.151); + assert_eq!(info2.sensitivity, 0.95); + assert_eq!(info.to_bytes(), info2.to_bytes()); + } + + #[test] + fn test_hybrid_info_serialization() { + let info = HybridInfo::new( + 0, + "https://www.example.com", + "https://www.example2.com", + 1_234_567, + 1.151, + 0.95, + ) + .unwrap(); + let bytes = info.to_bytes(); + let info2 = HybridInfo::from_bytes(&bytes).unwrap(); + assert_eq!(info.to_bytes(), info2.to_bytes()); + } } diff --git a/ipa-core/src/test_fixture/hybrid.rs b/ipa-core/src/test_fixture/hybrid.rs index 680c60c80..80f5abc07 100644 --- a/ipa-core/src/test_fixture/hybrid.rs +++ b/ipa-core/src/test_fixture/hybrid.rs @@ -6,18 +6,35 @@ use crate::{ U128Conversions, }, rand::Rng, - report::hybrid::{ - AggregateableHybridReport, HybridConversionReport, HybridImpressionReport, HybridReport, - IndistinguishableHybridReport, + report::{ + hybrid::{ + AggregateableHybridReport, HybridConversionReport, HybridImpressionReport, + HybridReport, IndistinguishableHybridReport, KeyIdentifier, + }, + hybrid_info::{HybridConversionInfo, HybridImpressionInfo}, }, secret_sharing::{replicated::semi_honest::AdditiveShare as Replicated, IntoShares}, test_fixture::sharing::Reconstruct, }; -#[derive(Debug, Clone, PartialEq, PartialOrd, Eq)] +#[derive(Debug, Clone, PartialEq, PartialOrd)] pub enum TestHybridRecord { - TestImpression { match_key: u64, breakdown_key: u32 }, - TestConversion { match_key: u64, value: u32 }, + TestImpression { + match_key: u64, + breakdown_key: u32, + key_id: KeyIdentifier, + helper_origin: String, + }, + TestConversion { + match_key: u64, + value: u32, + key_id: KeyIdentifier, + helper_origin: String, + conversion_site_domain: String, + timestamp: u64, + epsilon: f64, + sensitivity: f64, + }, } #[derive(Clone, PartialEq, Eq, Debug)] @@ -118,6 +135,8 @@ where TestHybridRecord::TestImpression { match_key, breakdown_key, + key_id, + helper_origin, } => { let ba_match_key = BA64::try_from(u128::from(match_key)) .unwrap() @@ -130,13 +149,23 @@ where HybridReport::Impression::(HybridImpressionReport { match_key: match_key_share, breakdown_key: breakdown_key_share, + info: HybridImpressionInfo::new(key_id, &helper_origin).unwrap(), }) }) .collect::>() .try_into() .unwrap() } - TestHybridRecord::TestConversion { match_key, value } => { + TestHybridRecord::TestConversion { + match_key, + value, + key_id, + helper_origin, + conversion_site_domain, + timestamp, + epsilon, + sensitivity, + } => { let ba_match_key = BA64::try_from(u128::from(match_key)) .unwrap() .share_with(rng); @@ -146,6 +175,15 @@ where HybridReport::Conversion::(HybridConversionReport { match_key: match_key_share, value: value_share, + info: HybridConversionInfo::new( + key_id, + &helper_origin, + &conversion_site_domain, + timestamp, + epsilon, + sensitivity, + ) + .unwrap(), }) }) .collect::>() @@ -225,63 +263,126 @@ pub fn hybrid_in_the_clear(input_rows: &[TestHybridRecord], max_breakdown: usize } #[must_use] +#[allow(clippy::too_many_lines)] pub fn build_hybrid_records_and_expectation() -> (Vec, Vec) { + let helper_origin = "HELPER_ORIGIN".to_string(); + let conversion_site_domain = "meta.com".to_string(); let test_hybrid_records = vec![ TestHybridRecord::TestConversion { match_key: 12345, value: 2, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 100, + epsilon: 0.0, + sensitivity: 0.0, }, // malicious client attributed to breakdown 0 TestHybridRecord::TestConversion { match_key: 12345, value: 5, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 101, + epsilon: 0.0, + sensitivity: 0.0, }, // malicious client attributed to breakdown 0 TestHybridRecord::TestImpression { match_key: 23456, breakdown_key: 4, + key_id: 0, + helper_origin: helper_origin.clone(), }, // attributed TestHybridRecord::TestConversion { match_key: 23456, value: 7, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 102, + epsilon: 0.0, + sensitivity: 0.0, }, // attributed TestHybridRecord::TestImpression { match_key: 34567, breakdown_key: 1, + key_id: 0, + helper_origin: helper_origin.clone(), }, // no conversion TestHybridRecord::TestImpression { match_key: 45678, breakdown_key: 3, + key_id: 0, + helper_origin: helper_origin.clone(), }, // attributed TestHybridRecord::TestConversion { match_key: 45678, value: 5, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 103, + epsilon: 0.0, + sensitivity: 0.0, }, // attributed TestHybridRecord::TestImpression { match_key: 56789, breakdown_key: 5, + key_id: 0, + helper_origin: helper_origin.clone(), }, // no conversion TestHybridRecord::TestConversion { match_key: 67890, value: 2, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 104, + epsilon: 0.0, + sensitivity: 0.0, }, // NOT attributed TestHybridRecord::TestImpression { match_key: 78901, breakdown_key: 2, + key_id: 0, + helper_origin: helper_origin.clone(), }, // too many reports TestHybridRecord::TestConversion { match_key: 78901, value: 3, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 105, + epsilon: 0.0, + sensitivity: 0.0, }, // not attributed, too many reports TestHybridRecord::TestConversion { match_key: 78901, value: 4, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 103, + epsilon: 0.0, + sensitivity: 0.0, }, // not attributed, too many reports TestHybridRecord::TestImpression { match_key: 89012, breakdown_key: 4, + key_id: 0, + helper_origin: helper_origin.clone(), }, // attributed TestHybridRecord::TestConversion { match_key: 89012, value: 6, + key_id: 0, + helper_origin: helper_origin.clone(), + conversion_site_domain: conversion_site_domain.clone(), + timestamp: 103, + epsilon: 0.0, + sensitivity: 0.0, }, // attributed ]; diff --git a/ipa-core/src/test_fixture/hybrid_event_gen.rs b/ipa-core/src/test_fixture/hybrid_event_gen.rs index da9dd76d8..f1e1893e8 100644 --- a/ipa-core/src/test_fixture/hybrid_event_gen.rs +++ b/ipa-core/src/test_fixture/hybrid_event_gen.rs @@ -111,6 +111,12 @@ impl EventGenerator { value: self .rng .gen_range(1..self.config.max_conversion_value.get()), + key_id: 0, + helper_origin: "HELPER_ORIGIN".to_string(), + conversion_site_domain: "meta.com".to_string(), + timestamp: self.rng.gen_range(0..1000), + epsilon: 0.0, + sensitivity: 0.0, } } @@ -118,6 +124,8 @@ impl EventGenerator { TestHybridRecord::TestImpression { match_key, breakdown_key: self.rng.gen_range(0..self.config.max_breakdown_key.get()), + key_id: 0, + helper_origin: "HELPER_ORIGIN".to_string(), } } } @@ -231,6 +239,7 @@ mod tests { TestHybridRecord::TestImpression { match_key, breakdown_key, + .. } => { assert!(breakdown_key <= MAX_BREAKDOWN_KEY); match_keys.insert(match_key); @@ -254,7 +263,9 @@ mod tests { let mut match_keys = HashSet::new(); for event in gen.take(NUM_EVENTS) { match event { - TestHybridRecord::TestConversion { match_key, value } => { + TestHybridRecord::TestConversion { + match_key, value, .. + } => { assert!(value <= MAX_VALUE); match_keys.insert(match_key); }