feat(protocol): Add scraping_attempts field (#2575)

getsentry/sentry#57461 introduced a new
top-level field on the event that is only written by sentry. To prevent
errors when the event passes through normalization after writing
`scraping_attempts`, make sure that the new field is part of the schema
known by (lib)relay.

CHANGELOG.md

@@ -26,6 +26,7 @@
 - Add `profile_id` to spans. ([#2569](https://github.com/getsentry/relay/pull/2569))
 - Introduce a dedicated usage metric for transactions that replaces the duration metric. ([#2571](https://github.com/getsentry/relay/pull/2571))
 - Restore the profiling killswitch. ([#2573](https://github.com/getsentry/relay/pull/2573))
+- Add `scraping_attempts` field to the event schema. ([#2575](https://github.com/getsentry/relay/pull/2575))
 
 ## 23.9.1
 

py/CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## Unreleased
+
+- Add `scraping_attempts` field to the event schema. ([#2575](https://github.com/getsentry/relay/pull/2575))
+
 ## 0.8.31
 
 - Add `Reservoir` variant to `SamplingRule`. ([#2550](https://github.com/getsentry/relay/pull/2550))

relay-event-normalization/src/remove_other.rs

@@ -76,7 +76,7 @@ impl Processor for RemoveOtherProcessor {
 mod tests {
     use relay_event_schema::processor::process_value;
     use relay_event_schema::protocol::{Context, Contexts, OsContext, User, Values};
-    use relay_protocol::get_value;
+    use relay_protocol::{get_value, FromValue};
     use similar_asserts::assert_eq;
 
     use super::*;
@@ -228,4 +228,23 @@ mod tests {
             Annotated::from_error(ErrorKind::InvalidAttribute, None)
         );
     }
+
+    #[test]
+    fn test_scrape_attempts() {
+        let json = serde_json::json!({
+            "scraping_attempts": [
+                {"status": "not_attempted", "url": "http://example.com/embedded.js"},
+                {"status": "not_attempted", "url": "http://example.com/embedded.js.map"},
+            ]
+        });
+
+        let mut event = Event::from_value(json.into());
+        process_value(
+            &mut event,
+            &mut RemoveOtherProcessor,
+            ProcessingState::root(),
+        )
+        .unwrap();
+        assert!(event.value().unwrap().other.is_empty());
+    }
 }

relay-event-schema/src/protocol/event.rs

@@ -480,6 +480,11 @@ pub struct Event {
     #[metastructure(omit_from_schema)] // we only document error events for now
     pub breakdowns: Annotated<Breakdowns>,
 
+    /// Information about attempts to scrape a JS source or sourcemap file from the web.
+    /// This field is populated by sentry.
+    #[metastructure(omit_from_schema)] // not part of external schema
+    pub scraping_attempts: Annotated<Value>,
+
     /// Internal ingestion and processing metrics.
     ///
     /// This value should not be ingested and will be overwritten by the store normalizer.
@@ -969,6 +974,19 @@ mod tests {
         assert_eq!(None, event.extra_at("c.f"));
     }
 
+    #[test]
+    fn test_scrape_attempts() {
+        let json = serde_json::json!({
+            "scraping_attempts": [
+                {"status": "not_attempted", "url": "http://example.com/embedded.js"},
+                {"status": "not_attempted", "url": "http://example.com/embedded.js.map"},
+            ]
+        });
+
+        let event = Event::from_value(json.into());
+        assert!(!event.value().unwrap().scraping_attempts.meta().has_errors());
+    }
+
     #[test]
     fn test_field_value_provider_event_filled() {
         let event = Event {

relay-pii/src/snapshots/relay_pii__processor__tests__does_not_scrub_if_no_graphql.snap

@@ -1,5 +1,5 @@
 ---
-source: relay-general/src/pii/processor.rs
+source: relay-pii/src/processor.rs
 expression: "&data"
 ---
 Event {
@@ -105,6 +105,7 @@ Event {
     spans: ~,
     measurements: ~,
     breakdowns: ~,
+    scraping_attempts: ~,
     _metrics: ~,
     other: {},
 }

relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_with_variables.snap

@@ -1,5 +1,5 @@
 ---
-source: relay-general/src/pii/processor.rs
+source: relay-pii/src/processor.rs
 expression: "&data"
 ---
 Event {
@@ -105,6 +105,7 @@ Event {
     spans: ~,
     measurements: ~,
     breakdowns: ~,
+    scraping_attempts: ~,
     _metrics: ~,
     other: {},
 }

relay-pii/src/snapshots/relay_pii__processor__tests__scrub_graphql_response_data_without_variables.snap

@@ -1,5 +1,5 @@
 ---
-source: relay-general/src/pii/processor.rs
+source: relay-pii/src/processor.rs
 expression: "&data"
 ---
 Event {
@@ -86,6 +86,7 @@ Event {
     spans: ~,
     measurements: ~,
     breakdowns: ~,
+    scraping_attempts: ~,
     _metrics: ~,
     other: {},
 }

relay-pii/src/snapshots/relay_pii__processor__tests__scrub_original_value.snap

@@ -1,5 +1,5 @@
 ---
-source: relay-general/src/pii/processor.rs
+source: relay-pii/src/processor.rs
 expression: "&data"
 ---
 Event {
@@ -116,6 +116,7 @@ Event {
     spans: ~,
     measurements: ~,
     breakdowns: ~,
+    scraping_attempts: ~,
     _metrics: ~,
     other: {},
 }