diff --git a/Cargo.lock b/Cargo.lock index 6657c2656..2072578db 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4222,6 +4222,7 @@ dependencies = [ "symbolic", "symbolicator-crash", "symbolicator-js", + "symbolicator-native", "symbolicator-service", "symbolicator-sources", "symbolicator-test", @@ -4274,20 +4275,47 @@ dependencies = [ ] [[package]] -name = "symbolicator-service" +name = "symbolicator-native" version = "23.10.0" dependencies = [ "anyhow", "apple-crash-report-parser", "async-trait", + "chrono", + "futures", + "insta", + "lazy_static", + "minidump", + "minidump-processor", + "minidump-unwind", + "moka", + "regex", + "sentry", + "serde", + "serde_json", + "symbolic", + "symbolicator-service", + "symbolicator-sources", + "symbolicator-test", + "tempfile", + "test-assembler", + "thiserror", + "tokio", + "tracing", + "url", +] + +[[package]] +name = "symbolicator-service" +version = "23.10.0" +dependencies = [ + "anyhow", "aws-config", "aws-credential-types", "aws-sdk-s3", "aws-types", - "backtrace", "cadence", "chrono", - "data-url", "filetime", "flate2", "futures", @@ -4295,18 +4323,13 @@ dependencies = [ "humantime", "humantime-serde", "idna 0.4.0", - "insta", "ipnetwork", "jsonwebtoken", "lazy_static", - "minidump", - "minidump-processor", - "minidump-unwind", "moka", "once_cell", "parking_lot 0.12.1", "rand", - "regex", "reqwest", "sentry", "serde", @@ -4318,7 +4341,6 @@ dependencies = [ "symbolicator-sources", "symbolicator-test", "tempfile", - "test-assembler", "thiserror", "tokio", "tokio-util", @@ -4358,6 +4380,7 @@ dependencies = [ "serde_json", "serde_yaml", "symbolicator-js", + "symbolicator-native", "symbolicator-service", "symbolicator-test", "tempfile", @@ -4397,6 +4420,7 @@ dependencies = [ "serde_yaml", "symbolic", "symbolicator-js", + "symbolicator-native", "symbolicator-service", "symbolicator-sources", "tempfile", diff --git a/crates/symbolicator-js/src/api_lookup.rs b/crates/symbolicator-js/src/api_lookup.rs index 122e83bda..f72f54257 100644 --- a/crates/symbolicator-js/src/api_lookup.rs +++ b/crates/symbolicator-js/src/api_lookup.rs @@ -6,9 +6,9 @@ use std::time::Duration; use sentry::types::DebugId; use sentry::SentryFutureExt; use serde::Deserialize; +use symbolicator_service::download::retry; +use symbolicator_service::download::sentry::{SearchQuery, SentryDownloader}; use symbolicator_service::metric; -use symbolicator_service::services::download::retry; -use symbolicator_service::services::download::sentry::{SearchQuery, SentryDownloader}; use url::Url; use symbolicator_service::caching::{CacheEntry, CacheError}; diff --git a/crates/symbolicator-js/src/bundle_index_cache.rs b/crates/symbolicator-js/src/bundle_index_cache.rs index e6b88695e..460b6ae2b 100644 --- a/crates/symbolicator-js/src/bundle_index_cache.rs +++ b/crates/symbolicator-js/src/bundle_index_cache.rs @@ -1,12 +1,11 @@ use std::sync::Arc; use symbolic::common::ByteView; +use symbolicator_service::caches::versions::BUNDLE_INDEX_CACHE_VERSIONS; use symbolicator_service::caching::{ Cache, CacheEntry, CacheItemRequest, CacheKey, CacheVersions, Cacher, SharedCacheRef, }; -use symbolicator_service::services::caches::versions::BUNDLE_INDEX_CACHE_VERSIONS; -use symbolicator_service::services::download::DownloadService; -use symbolicator_service::services::fetch_file; +use symbolicator_service::download::{fetch_file, DownloadService}; use symbolicator_service::types::Scope; use symbolicator_sources::RemoteFile; diff --git a/crates/symbolicator-js/src/interface.rs b/crates/symbolicator-js/src/interface.rs index 60a3a0b90..3edfa011e 100644 --- a/crates/symbolicator-js/src/interface.rs +++ b/crates/symbolicator-js/src/interface.rs @@ -6,8 +6,7 @@ use reqwest::Url; use serde::{Deserialize, Serialize}; use symbolicator_service::caching::CacheError; -use symbolicator_service::services::ScrapingConfig; -use symbolicator_service::types::{RawObjectInfo, Scope}; +use symbolicator_service::types::{RawObjectInfo, Scope, ScrapingConfig}; use symbolicator_sources::{SentryFileId, SentrySourceConfig}; #[derive(Debug, Clone)] diff --git a/crates/symbolicator-js/src/lookup.rs b/crates/symbolicator-js/src/lookup.rs index 606fc5d35..65fcd0a09 100644 --- a/crates/symbolicator-js/src/lookup.rs +++ b/crates/symbolicator-js/src/lookup.rs @@ -46,15 +46,14 @@ use symbolicator_sources::{ }; use tempfile::NamedTempFile; +use symbolicator_service::caches::versions::SOURCEMAP_CACHE_VERSIONS; +use symbolicator_service::caches::{ByteViewString, SourceFilesCache}; use symbolicator_service::caching::{ CacheEntry, CacheError, CacheItemRequest, CacheKey, CacheKeyBuilder, CacheVersions, Cacher, }; -use symbolicator_service::services::caches::versions::SOURCEMAP_CACHE_VERSIONS; -use symbolicator_service::services::caches::{ByteViewString, SourceFilesCache}; -use symbolicator_service::services::download::DownloadService; -use symbolicator_service::services::objects::{ObjectHandle, ObjectMetaHandle, ObjectsActor}; -use symbolicator_service::services::symbolication::ScrapingConfig; -use symbolicator_service::types::Scope; +use symbolicator_service::download::DownloadService; +use symbolicator_service::objects::{ObjectHandle, ObjectMetaHandle, ObjectsActor}; +use symbolicator_service::types::{Scope, ScrapingConfig}; use symbolicator_service::utils::http::is_valid_origin; use crate::api_lookup::{ArtifactHeaders, JsLookupResult, SentryLookupApi}; diff --git a/crates/symbolicator-js/src/service.rs b/crates/symbolicator-js/src/service.rs index 40f330038..fcc3fdf80 100644 --- a/crates/symbolicator-js/src/service.rs +++ b/crates/symbolicator-js/src/service.rs @@ -2,10 +2,10 @@ use std::sync::Arc; +use symbolicator_service::caches::SourceFilesCache; use symbolicator_service::caching::Cacher; -use symbolicator_service::services::caches::SourceFilesCache; -use symbolicator_service::services::download::DownloadService; -use symbolicator_service::services::objects::ObjectsActor; +use symbolicator_service::download::DownloadService; +use symbolicator_service::objects::ObjectsActor; use symbolicator_service::services::SharedServices; use crate::api_lookup::SentryLookupApi; diff --git a/crates/symbolicator-js/src/symbolication.rs b/crates/symbolicator-js/src/symbolication.rs index 55e64cb87..af3e79029 100644 --- a/crates/symbolicator-js/src/symbolication.rs +++ b/crates/symbolicator-js/src/symbolication.rs @@ -3,7 +3,7 @@ use std::collections::BTreeSet; use symbolic::sourcemapcache::{ScopeLookupResult, SourcePosition}; use symbolicator_service::caching::CacheError; use symbolicator_service::metric; -use symbolicator_service::services::symbolication::source_context::get_context_lines; +use symbolicator_service::source_context::get_context_lines; use crate::interface::{ CompletedJsSymbolicationResponse, JsFrame, JsModuleError, JsModuleErrorKind, JsStacktrace, diff --git a/crates/symbolicator-js/tests/integration/sourcemap.rs b/crates/symbolicator-js/tests/integration/sourcemap.rs index 428f0616c..290b09efa 100644 --- a/crates/symbolicator-js/tests/integration/sourcemap.rs +++ b/crates/symbolicator-js/tests/integration/sourcemap.rs @@ -4,8 +4,7 @@ use std::sync::Arc; use reqwest::Url; use serde_json::json; use symbolicator_js::interface::{JsFrame, JsStacktrace, SymbolicateJsStacktraces}; -use symbolicator_service::services::ScrapingConfig; -use symbolicator_service::types::{RawObjectInfo, Scope}; +use symbolicator_service::types::{RawObjectInfo, Scope, ScrapingConfig}; use symbolicator_sources::{SentrySourceConfig, SourceId}; use crate::{assert_snapshot, setup_service}; diff --git a/crates/symbolicator-native/Cargo.toml b/crates/symbolicator-native/Cargo.toml new file mode 100644 index 000000000..a5ca718b3 --- /dev/null +++ b/crates/symbolicator-native/Cargo.toml @@ -0,0 +1,36 @@ +[package] +name = "symbolicator-native" +publish = false +version = "23.10.0" +authors = ["Sentry "] +edition = "2021" +license = "MIT" + +[dependencies] +anyhow = "1.0.57" +apple-crash-report-parser = "0.5.1" +async-trait = "0.1.53" +chrono = { version = "0.4.19", features = ["serde"] } +futures = "0.3.12" +lazy_static = "1.4.0" +minidump = "0.18.0" +minidump-processor = "0.18.0" +minidump-unwind = "0.18.0" +moka = { version = "0.12.1", features = ["future", "sync"] } +regex = "1.5.5" +sentry = { version = "0.31.7", features = ["tracing"] } +serde = { version = "1.0.137", features = ["derive", "rc"] } +symbolic = { version = "12.4.0", features = ["cfi", "common-serde", "debuginfo", "demangle", "symcache", "il2cpp", "ppdb"] } +symbolicator-service = { path = "../symbolicator-service" } +symbolicator-sources = { path = "../symbolicator-sources" } +tempfile = "3.2.0" +thiserror = "1.0.31" +tracing = "0.1.34" +url = { version = "2.2.0", features = ["serde"] } + +[dev-dependencies] +insta = { version = "1.18.0", features = ["redactions", "yaml"] } +serde_json = "1.0.81" +symbolicator-test = { path = "../symbolicator-test" } +test-assembler = "0.1.5" +tokio = { version = "1.24.2", features = ["rt", "macros", "fs"] } diff --git a/crates/symbolicator-service/src/services/bitcode.rs b/crates/symbolicator-native/src/caches/bitcode.rs similarity index 96% rename from crates/symbolicator-service/src/services/bitcode.rs rename to crates/symbolicator-native/src/caches/bitcode.rs index 53ff3b738..8ebb0504e 100644 --- a/crates/symbolicator-service/src/services/bitcode.rs +++ b/crates/symbolicator-native/src/caches/bitcode.rs @@ -12,18 +12,16 @@ use sentry::{Hub, SentryFutureExt}; use symbolic::common::{ByteView, DebugId}; use symbolic::debuginfo::macho::{BcSymbolMap, UuidMapping}; -use symbolicator_sources::{FileType, RemoteFile, SourceConfig}; -use tempfile::NamedTempFile; - -use crate::caching::{ +use symbolicator_service::caches::versions::BITCODE_CACHE_VERSIONS; +use symbolicator_service::caching::{ Cache, CacheEntry, CacheError, CacheItemRequest, CacheKey, CacheVersions, Cacher, SharedCacheRef, }; -use crate::services::download::DownloadService; -use crate::types::Scope; - -use super::caches::versions::BITCODE_CACHE_VERSIONS; -use super::fetch_file; +use symbolicator_service::download::{fetch_file, DownloadService}; +use symbolicator_service::metric; +use symbolicator_service::types::Scope; +use symbolicator_sources::{FileType, RemoteFile, SourceConfig}; +use tempfile::NamedTempFile; /// Handle to a valid BCSymbolMap. /// diff --git a/crates/symbolicator-service/src/services/cficaches.rs b/crates/symbolicator-native/src/caches/cficaches.rs similarity index 94% rename from crates/symbolicator-service/src/services/cficaches.rs rename to crates/symbolicator-native/src/caches/cficaches.rs index 5de1a9ed0..03a80b049 100644 --- a/crates/symbolicator-service/src/services/cficaches.rs +++ b/crates/symbolicator-native/src/caches/cficaches.rs @@ -10,18 +10,17 @@ use tempfile::NamedTempFile; use symbolic::cfi::CfiCache; use symbolic::common::ByteView; use symbolic::debuginfo::breakpad::BreakpadModuleRecord; -use symbolicator_sources::{FileType, ObjectId, ObjectType, SourceConfig}; - -use crate::caching::{ +use symbolicator_service::caches::versions::CFICACHE_VERSIONS; +use symbolicator_service::caching::{ Cache, CacheEntry, CacheError, CacheItemRequest, CacheVersions, Cacher, SharedCacheRef, }; -use crate::services::objects::{ - FindObject, ObjectHandle, ObjectMetaHandle, ObjectPurpose, ObjectsActor, +use symbolicator_service::objects::{ + CandidateStatus, FindObject, ObjectHandle, ObjectMetaHandle, ObjectPurpose, ObjectsActor, }; -use crate::types::{CandidateStatus, Scope}; -use crate::utils::sentry::ConfigureScope; +use symbolicator_service::types::Scope; +use symbolicator_service::utils::sentry::ConfigureScope; +use symbolicator_sources::{FileType, ObjectId, ObjectType, SourceConfig}; -use super::caches::versions::CFICACHE_VERSIONS; use super::derived::{derive_from_object_handle, DerivedCache}; type CfiItem = Option)>>; diff --git a/crates/symbolicator-service/src/services/derived.rs b/crates/symbolicator-native/src/caches/derived.rs similarity index 93% rename from crates/symbolicator-service/src/services/derived.rs rename to crates/symbolicator-native/src/caches/derived.rs index 120bceb1f..fed8c37be 100644 --- a/crates/symbolicator-service/src/services/derived.rs +++ b/crates/symbolicator-native/src/caches/derived.rs @@ -1,8 +1,10 @@ use std::sync::Arc; -use crate::caching::{CacheEntry, CacheError}; -use crate::services::objects::{FindResult, ObjectMetaHandle}; -use crate::types::{AllObjectCandidates, CandidateStatus, ObjectFeatures, ObjectUseInfo}; +use symbolicator_service::caching::{CacheEntry, CacheError}; +use symbolicator_service::objects::{ + AllObjectCandidates, CandidateStatus, FindResult, ObjectFeatures, ObjectMetaHandle, + ObjectUseInfo, +}; /// This is the result of fetching a derived cache file. /// diff --git a/crates/symbolicator-service/src/services/il2cpp.rs b/crates/symbolicator-native/src/caches/il2cpp.rs similarity index 95% rename from crates/symbolicator-service/src/services/il2cpp.rs rename to crates/symbolicator-native/src/caches/il2cpp.rs index ecdc9fbb9..3095a4783 100644 --- a/crates/symbolicator-service/src/services/il2cpp.rs +++ b/crates/symbolicator-native/src/caches/il2cpp.rs @@ -10,18 +10,16 @@ use sentry::{Hub, SentryFutureExt}; use symbolic::common::{ByteView, DebugId}; use symbolic::il2cpp::LineMapping; -use symbolicator_sources::{FileType, ObjectId, RemoteFile, SourceConfig}; -use tempfile::NamedTempFile; - -use crate::caching::{ +use symbolicator_service::caches::versions::IL2CPP_CACHE_VERSIONS; +use symbolicator_service::caching::{ Cache, CacheEntry, CacheError, CacheItemRequest, CacheKey, CacheVersions, Cacher, SharedCacheRef, }; -use crate::services::download::DownloadService; -use crate::types::Scope; - -use super::caches::versions::IL2CPP_CACHE_VERSIONS; -use super::fetch_file; +use symbolicator_service::download::{fetch_file, DownloadService}; +use symbolicator_service::metric; +use symbolicator_service::types::Scope; +use symbolicator_sources::{FileType, ObjectId, RemoteFile, SourceConfig}; +use tempfile::NamedTempFile; /// Handle to a valid [`LineMapping`]. /// diff --git a/crates/symbolicator-native/src/caches/mod.rs b/crates/symbolicator-native/src/caches/mod.rs new file mode 100644 index 000000000..a870a04cb --- /dev/null +++ b/crates/symbolicator-native/src/caches/mod.rs @@ -0,0 +1,6 @@ +pub mod bitcode; +pub mod cficaches; +pub mod derived; +pub mod il2cpp; +pub mod ppdb_caches; +pub mod symcaches; diff --git a/crates/symbolicator-service/src/services/ppdb_caches.rs b/crates/symbolicator-native/src/caches/ppdb_caches.rs similarity index 93% rename from crates/symbolicator-service/src/services/ppdb_caches.rs rename to crates/symbolicator-native/src/caches/ppdb_caches.rs index 2156a1a86..44a3c7135 100644 --- a/crates/symbolicator-service/src/services/ppdb_caches.rs +++ b/crates/symbolicator-native/src/caches/ppdb_caches.rs @@ -8,17 +8,18 @@ use tempfile::NamedTempFile; use symbolic::common::{ByteView, SelfCell}; use symbolic::debuginfo::Object; use symbolic::ppdb::{PortablePdbCache, PortablePdbCacheConverter}; -use symbolicator_sources::{FileType, ObjectId, SourceConfig}; - -use crate::caching::{ +use symbolicator_service::caches::versions::PPDB_CACHE_VERSIONS; +use symbolicator_service::caching::{ Cache, CacheEntry, CacheError, CacheItemRequest, CacheVersions, Cacher, SharedCacheRef, }; -use crate::types::{CandidateStatus, Scope}; -use crate::utils::sentry::ConfigureScope; +use symbolicator_service::objects::{ + CandidateStatus, FindObject, ObjectHandle, ObjectMetaHandle, ObjectPurpose, ObjectsActor, +}; +use symbolicator_service::types::Scope; +use symbolicator_service::utils::sentry::ConfigureScope; +use symbolicator_sources::{FileType, ObjectId, SourceConfig}; -use super::caches::versions::PPDB_CACHE_VERSIONS; use super::derived::{derive_from_object_handle, DerivedCache}; -use super::objects::{FindObject, ObjectHandle, ObjectMetaHandle, ObjectPurpose, ObjectsActor}; pub type OwnedPortablePdbCache = SelfCell, PortablePdbCache<'static>>; diff --git a/crates/symbolicator-service/src/services/symcaches.rs b/crates/symbolicator-native/src/caches/symcaches.rs similarity index 96% rename from crates/symbolicator-service/src/services/symcaches.rs rename to crates/symbolicator-native/src/caches/symcaches.rs index 6e2565c1e..2601e04ad 100644 --- a/crates/symbolicator-service/src/services/symcaches.rs +++ b/crates/symbolicator-native/src/caches/symcaches.rs @@ -9,20 +9,18 @@ use tempfile::NamedTempFile; use symbolic::common::{ByteView, SelfCell}; use symbolic::symcache::{SymCache, SymCacheConverter}; -use symbolicator_sources::{FileType, ObjectId, ObjectType, SourceConfig}; - -use crate::caching::{ +use symbolicator_service::caches::versions::SYMCACHE_VERSIONS; +use symbolicator_service::caching::{ Cache, CacheEntry, CacheError, CacheItemRequest, CacheVersions, Cacher, SharedCacheRef, }; -use crate::services::bitcode::BitcodeService; -use crate::services::objects::{ - FindObject, ObjectHandle, ObjectMetaHandle, ObjectPurpose, ObjectsActor, +use symbolicator_service::objects::{ + CandidateStatus, FindObject, ObjectHandle, ObjectMetaHandle, ObjectPurpose, ObjectsActor, }; -use crate::types::{CandidateStatus, Scope}; -use crate::utils::sentry::ConfigureScope; +use symbolicator_service::types::Scope; +use symbolicator_service::utils::sentry::ConfigureScope; +use symbolicator_sources::{FileType, ObjectId, ObjectType, SourceConfig}; -use super::bitcode::BcSymbolMapHandle; -use super::caches::versions::SYMCACHE_VERSIONS; +use super::bitcode::{BcSymbolMapHandle, BitcodeService}; use super::derived::{derive_from_object_handle, DerivedCache}; use super::il2cpp::{Il2cppHandle, Il2cppService}; @@ -283,14 +281,13 @@ mod tests { use symbolic::common::{DebugId, Uuid}; use super::*; - use crate::caching::Caches; - use crate::config::{CacheConfigs, Config}; - use crate::services::bitcode::BitcodeService; - use crate::services::DownloadService; - use crate::test::{self, fixture}; + use symbolicator_service::caching::Caches; + use symbolicator_service::config::{CacheConfigs, Config}; + use symbolicator_service::download::DownloadService; use symbolicator_sources::{ CommonSourceConfig, DirectoryLayoutType, FilesystemSourceConfig, SourceConfig, SourceId, }; + use symbolicator_test::{self as test, fixture}; /// Creates a `SymCacheActor` with the given cache directory /// and timeout for download cache misses. diff --git a/crates/symbolicator-service/src/types/mod.rs b/crates/symbolicator-native/src/interface.rs similarity index 76% rename from crates/symbolicator-service/src/types/mod.rs rename to crates/symbolicator-native/src/interface.rs index 2c5b73a81..927bf3e3e 100644 --- a/crates/symbolicator-service/src/types/mod.rs +++ b/crates/symbolicator-native/src/interface.rs @@ -1,63 +1,178 @@ -//! Types for the Symbolicator API. -//! -//! This module contains some types which (de)serialise to/from JSON to make up the public -//! HTTP API. Its messy and things probably need a better place and different way to signal -//! they are part of the public API. - +use std::borrow::Cow; use std::collections::BTreeMap; use std::fmt; +use std::str::FromStr; use std::sync::Arc; use chrono::{DateTime, Utc}; +use serde::de::{self, Deserializer}; +use serde::ser::Serializer; use serde::{Deserialize, Serialize}; use symbolic::common::{Arch, CodeId, DebugId, Language}; -use symbolicator_sources::ObjectType; +use symbolicator_service::objects::{AllObjectCandidates, ObjectFeatures}; +use symbolicator_service::types::{ObjectFileStatus, RawObjectInfo, Scope, ScrapingConfig}; +use symbolicator_service::utils::hex::HexValue; +use symbolicator_sources::SourceConfig; +use thiserror::Error; + +pub use crate::metrics::StacktraceOrigin; + +#[derive(Debug, Clone)] +/// A request for symbolication of multiple stack traces. +pub struct SymbolicateStacktraces { + /// The scope of this request which determines access to cached files. + pub scope: Scope, + + /// The signal thrown on certain operating systems. + /// + /// Signal handlers sometimes mess with the runtime stack. This is used to determine whether + /// the top frame should be fixed or not. + pub signal: Option, -use crate::utils::addr::AddrMode; -use crate::utils::hex::HexValue; + /// A list of external sources to load debug files. + pub sources: Arc<[SourceConfig]>, + + /// Where the stacktraces originated from. + pub origin: StacktraceOrigin, + + /// A list of threads containing stack traces. + pub stacktraces: Vec, + + /// A list of images that were loaded into the process. + /// + /// This list must cover the instruction addresses of the frames in + /// [`stacktraces`](Self::stacktraces). If a frame is not covered by any image, the frame cannot + /// be symbolicated as it is not clear which debug file to load. + pub modules: Vec, -mod objects; + /// Whether to apply source context for the stack frames. + pub apply_source_context: bool, -pub use objects::{ - AllObjectCandidates, CandidateStatus, ObjectCandidate, ObjectDownloadInfo, ObjectUseInfo, -}; + /// Scraping configuration controling authenticated requests. + pub scraping: ScrapingConfig, +} + +/// The symbolicated crash data. +/// +/// It contains the symbolicated stack frames, module information as well as other +/// meta-information about the crash. +/// +/// It is publicly documented at . +#[derive(Debug, Default, Clone, Deserialize, Serialize)] +pub struct CompletedSymbolicationResponse { + /// When the crash occurred. + #[serde( + default, + skip_serializing_if = "Option::is_none", + with = "chrono::serde::ts_seconds_option" + )] + pub timestamp: Option>, + + /// The signal that caused this crash. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signal: Option, + + /// Information about the operating system. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub system_info: Option, + + /// True if the process crashed, false if the dump was produced outside of an exception + /// handler. Only set for minidumps. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub crashed: Option, + + /// If the process crashed, the type of crash. OS- and possibly CPU- specific. For + /// example, "EXCEPTION_ACCESS_VIOLATION" (Windows), "EXC_BAD_ACCESS / + /// KERN_INVALID_ADDRESS" (Mac OS X), "SIGSEGV" (other Unix). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub crash_reason: Option, + + /// A detailed explanation of the crash, potentially in human readable form. This may + /// include a string representation of the crash reason or application-specific info. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub crash_details: Option, + + /// If there was an assertion that was hit, a textual representation of that assertion, + /// possibly including the file and line at which it occurred. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub assertion: Option, + + /// The threads containing symbolicated stack frames. + pub stacktraces: Vec, + + /// A list of images, extended with status information. + pub modules: Vec, +} /// OS-specific crash signal value. // TODO(markus): Also accept POSIX signal name as defined in signal.h #[derive(Debug, Clone, Copy, Deserialize, Serialize, Eq, PartialEq)] pub struct Signal(pub u32); -/// The scope of a source or debug file. -/// -/// Based on scopes, access to debug files that have been cached is determined. If a file comes from -/// a public source, it can be used for any symbolication request. Otherwise, the symbolication -/// request must match the scope of a file. -#[derive(Debug, Clone, Deserialize, Serialize, Eq, Ord, PartialEq, PartialOrd, Hash)] -#[serde(untagged)] +/// Information on the symbolication status of this frame. +#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize, Serialize)] +#[serde(rename_all = "snake_case")] #[derive(Default)] -pub enum Scope { - #[serde(rename = "global")] +pub enum FrameStatus { + /// The frame was symbolicated successfully. #[default] - Global, - Scoped(Arc), + Symbolicated, + /// The symbol (i.e. function) was not found within the debug file. + MissingSymbol, + /// No debug image is specified for the address of the frame. + UnknownImage, + /// The debug file could not be retrieved from any of the sources. + Missing, + /// The retrieved debug file could not be processed. + Malformed, } -impl AsRef for Scope { - fn as_ref(&self) -> &str { - match *self { - Scope::Global => "global", - Scope::Scoped(ref s) => s, - } - } +/// A potentially symbolicated frame in the symbolication response. +#[derive(Debug, Default, Clone, Deserialize, Serialize)] +pub struct SymbolicatedFrame { + /// Symbolication status of this frame. + pub status: FrameStatus, + + /// The index of this frame in the request. + /// + /// This is relevant for two reasons: + /// 1. Frames might disappear if the symbolicator determines them as a false-positive from + /// stackwalking without CFI. + /// 2. Frames might expand to multiple inline frames at the same instruction address. However, + /// this might occur within recursion, so the instruction address is not a good + pub original_index: Option, + + #[serde(flatten)] + pub raw: RawFrame, } -impl fmt::Display for Scope { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - Scope::Global => f.write_str("global"), - Scope::Scoped(ref scope) => f.write_str(scope), - } - } +/// A symbolicated stacktrace. +/// +/// Frames in this request may or may not be symbolicated. The status field contains information on +/// the individual success for each frame. +#[derive(Debug, Default, Clone, Deserialize, Serialize)] +pub struct CompleteStacktrace { + /// ID of thread that had this stacktrace. Returned when a minidump was processed. + #[serde(skip_serializing_if = "Option::is_none")] + pub thread_id: Option, + + #[serde(skip_serializing_if = "Option::is_none")] + pub thread_name: Option, + + /// If a dump was produced as a result of a crash, this will point to the thread that crashed. + /// If the dump was produced by user code without crashing, and the dump contains extended + /// Breakpad information, this will point to the thread that requested the dump. + /// + /// Currently only `Some` for minidumps. + #[serde(skip_serializing_if = "Option::is_none")] + pub is_requesting: Option, + + /// Registers, only useful when returning a processed minidump. + #[serde(default, skip_serializing_if = "Registers::is_empty")] + pub registers: Registers, + + /// Frames of this stack trace. + pub frames: Vec, } /// A map of register values. @@ -234,177 +349,6 @@ pub struct RawStacktrace { /// The first entry in the list is the active frame, with its callers below. pub frames: Vec, } - -/// Specification of a module loaded into the process. -#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] -pub struct RawObjectInfo { - /// Platform image file type (container format). - #[serde(rename = "type")] - pub ty: ObjectType, - - /// Identifier of the code file. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub code_id: Option, - - /// Name of the code file. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub code_file: Option, - - /// Identifier of the debug file. - #[serde(skip_serializing_if = "Option::is_none")] - pub debug_id: Option, - - /// Name of the debug file. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub debug_file: Option, - - /// Checksum of the file's contents. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub debug_checksum: Option, - - /// Absolute address at which the image was mounted into virtual memory. - /// - /// We do allow the `image_addr` to be skipped if it is zero. This is because systems like WASM - /// do not require modules to be mounted at a specific absolute address. Per definition, a - /// module mounted at `0` does not support absolute addressing. - #[serde(default)] - pub image_addr: HexValue, - - /// Size of the image in virtual memory. - /// - /// The size is infered from the module list if not specified. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub image_size: Option, -} - -/// Information on the symbolication status of this frame. -#[derive(Debug, Clone, Copy, Eq, PartialEq, Deserialize, Serialize)] -#[serde(rename_all = "snake_case")] -#[derive(Default)] -pub enum FrameStatus { - /// The frame was symbolicated successfully. - #[default] - Symbolicated, - /// The symbol (i.e. function) was not found within the debug file. - MissingSymbol, - /// No debug image is specified for the address of the frame. - UnknownImage, - /// The debug file could not be retrieved from any of the sources. - Missing, - /// The retrieved debug file could not be processed. - Malformed, -} - -/// A potentially symbolicated frame in the symbolication response. -#[derive(Debug, Default, Clone, Deserialize, Serialize)] -pub struct SymbolicatedFrame { - /// Symbolication status of this frame. - pub status: FrameStatus, - - /// The index of this frame in the request. - /// - /// This is relevant for two reasons: - /// 1. Frames might disappear if the symbolicator determines them as a false-positive from - /// stackwalking without CFI. - /// 2. Frames might expand to multiple inline frames at the same instruction address. However, - /// this might occur within recursion, so the instruction address is not a good - pub original_index: Option, - - #[serde(flatten)] - pub raw: RawFrame, -} - -/// A symbolicated stacktrace. -/// -/// Frames in this request may or may not be symbolicated. The status field contains information on -/// the individual success for each frame. -#[derive(Debug, Default, Clone, Deserialize, Serialize)] -pub struct CompleteStacktrace { - /// ID of thread that had this stacktrace. Returned when a minidump was processed. - #[serde(skip_serializing_if = "Option::is_none")] - pub thread_id: Option, - - #[serde(skip_serializing_if = "Option::is_none")] - pub thread_name: Option, - - /// If a dump was produced as a result of a crash, this will point to the thread that crashed. - /// If the dump was produced by user code without crashing, and the dump contains extended - /// Breakpad information, this will point to the thread that requested the dump. - /// - /// Currently only `Some` for minidumps. - #[serde(skip_serializing_if = "Option::is_none")] - pub is_requesting: Option, - - /// Registers, only useful when returning a processed minidump. - #[serde(default, skip_serializing_if = "Registers::is_empty")] - pub registers: Registers, - - /// Frames of this stack trace. - pub frames: Vec, -} - -/// Information on a debug information file. -#[derive(Debug, Clone, Copy, Serialize, Deserialize, Eq, PartialEq)] -#[serde(rename_all = "snake_case")] -#[derive(Default)] -pub enum ObjectFileStatus { - /// The file was found and successfully processed. - Found, - /// The image was not referenced in the stack trace and not further handled. - #[default] - Unused, - /// The file could not be found in any of the specified sources. - Missing, - /// The file failed to process. - Malformed, - /// The file could not be downloaded. - FetchingFailed, - /// Downloading or processing the file took too long. - Timeout, - /// An internal error while handling this image. - Other, -} - -impl ObjectFileStatus { - pub fn name(self) -> &'static str { - // used for metrics - match self { - ObjectFileStatus::Found => "found", - ObjectFileStatus::Unused => "unused", - ObjectFileStatus::Missing => "missing", - ObjectFileStatus::Malformed => "malformed", - ObjectFileStatus::FetchingFailed => "fetching_failed", - ObjectFileStatus::Timeout => "timeout", - ObjectFileStatus::Other => "other", - } - } -} - -#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] -pub struct ObjectFeatures { - /// The object file contains full debug info. - pub has_debug_info: bool, - - /// The object file contains unwind info. - pub has_unwind_info: bool, - - /// The object file contains a symbol table. - pub has_symbols: bool, - - /// The object file had sources available. - #[serde(default)] - pub has_sources: bool, -} - -impl ObjectFeatures { - pub fn merge(&mut self, other: ObjectFeatures) { - self.has_debug_info |= other.has_debug_info; - self.has_unwind_info |= other.has_unwind_info; - self.has_symbols |= other.has_symbols; - self.has_sources |= other.has_sources; - } -} - /// Normalized [`RawObjectInfo`] with status attached. /// /// This describes an object in the modules list of a response to a symbolication request. @@ -500,58 +444,6 @@ impl From for CompleteObjectInfo { } } -/// The symbolicated crash data. -/// -/// It contains the symbolicated stack frames, module information as well as other -/// meta-information about the crash. -/// -/// It is publicly documented at . -#[derive(Debug, Default, Clone, Deserialize, Serialize)] -pub struct CompletedSymbolicationResponse { - /// When the crash occurred. - #[serde( - default, - skip_serializing_if = "Option::is_none", - with = "chrono::serde::ts_seconds_option" - )] - pub timestamp: Option>, - - /// The signal that caused this crash. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub signal: Option, - - /// Information about the operating system. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub system_info: Option, - - /// True if the process crashed, false if the dump was produced outside of an exception - /// handler. Only set for minidumps. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub crashed: Option, - - /// If the process crashed, the type of crash. OS- and possibly CPU- specific. For - /// example, "EXCEPTION_ACCESS_VIOLATION" (Windows), "EXC_BAD_ACCESS / - /// KERN_INVALID_ADDRESS" (Mac OS X), "SIGSEGV" (other Unix). - #[serde(default, skip_serializing_if = "Option::is_none")] - pub crash_reason: Option, - - /// A detailed explanation of the crash, potentially in human readable form. This may - /// include a string representation of the crash reason or application-specific info. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub crash_details: Option, - - /// If there was an assertion that was hit, a textual representation of that assertion, - /// possibly including the file and line at which it occurred. - #[serde(default, skip_serializing_if = "Option::is_none")] - pub assertion: Option, - - /// The threads containing symbolicated stack frames. - pub stacktraces: Vec, - - /// A list of images, extended with status information. - pub modules: Vec, -} - /// Information about the operating system. #[derive(Clone, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] pub struct SystemInfo { @@ -570,3 +462,112 @@ pub struct SystemInfo { /// Device model name pub device_model: String, } + +/// Whether a frame's instruction address needs to be "adjusted" by subtracting a word. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AdjustInstructionAddr { + /// The frame's address definitely needs to be adjusted. + Yes, + /// The frame's address definitely does not need to be adjusted. + No, + /// The frame's address might need to be adjusted. + /// + /// This defers to the heuristic in + /// [InstructionInfo::caller_address](symbolic::common::InstructionInfo::caller_address). + Auto, +} + +impl AdjustInstructionAddr { + /// Returns the adjustment strategy for the given frame. + /// + /// If the frame has the + /// [`adjust_instruction_addr`](RawFrame::adjust_instruction_addr) + /// field set, this will be [`Yes`](Self::Yes) or [`No`](Self::No) accordingly, otherwise + /// the given default is used. + pub fn for_frame(frame: &RawFrame, default: Self) -> Self { + match frame.adjust_instruction_addr { + Some(true) => Self::Yes, + Some(false) => Self::No, + None => default, + } + } + + /// Returns the default adjustment strategy for the given thread. + /// + /// This will be [`Yes`](Self::Yes) if any frame in the thread has the + /// [`adjust_instruction_addr`](RawFrame::adjust_instruction_addr) + /// field set, otherwise it will be [`Auto`](Self::Auto). + pub fn default_for_thread(thread: &RawStacktrace) -> Self { + if thread + .frames + .iter() + .any(|frame| frame.adjust_instruction_addr.is_some()) + { + AdjustInstructionAddr::Yes + } else { + AdjustInstructionAddr::Auto + } + } +} + +/// Defines the addressing mode. +#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Default)] +pub enum AddrMode { + /// Declares addresses to be absolute with a shared memory space. + #[default] + Abs, + /// Declares an address to be relative to an indexed module. + Rel(usize), +} + +impl fmt::Display for AddrMode { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + AddrMode::Abs => write!(f, "abs"), + AddrMode::Rel(idx) => write!(f, "rel:{idx}"), + } + } +} + +impl Serialize for AddrMode { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(&self.to_string()) + } +} + +#[derive(Debug, Error)] +#[error("invalid address mode")] +pub struct ParseAddrModeError; + +impl FromStr for AddrMode { + type Err = ParseAddrModeError; + + fn from_str(s: &str) -> Result { + if s == "abs" { + return Ok(AddrMode::Abs); + } + let mut iter = s.splitn(2, ':'); + let kind = iter.next().ok_or(ParseAddrModeError)?; + let index = iter + .next() + .and_then(|x| x.parse().ok()) + .ok_or(ParseAddrModeError)?; + match kind { + "rel" => Ok(AddrMode::Rel(index)), + _ => Err(ParseAddrModeError), + } + } +} + +impl<'de> Deserialize<'de> for AddrMode { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let s = Cow::::deserialize(deserializer).map_err(de::Error::custom)?; + AddrMode::from_str(&s).map_err(de::Error::custom) + } +} diff --git a/crates/symbolicator-native/src/lib.rs b/crates/symbolicator-native/src/lib.rs new file mode 100644 index 000000000..53879f079 --- /dev/null +++ b/crates/symbolicator-native/src/lib.rs @@ -0,0 +1,6 @@ +mod caches; +pub mod interface; +mod metrics; +mod symbolication; + +pub use symbolication::symbolicate::SymbolicationActor; diff --git a/crates/symbolicator-native/src/metrics.rs b/crates/symbolicator-native/src/metrics.rs new file mode 100644 index 000000000..e46dd7c7d --- /dev/null +++ b/crates/symbolicator-native/src/metrics.rs @@ -0,0 +1,196 @@ +use symbolic::common::DebugId; +use symbolicator_service::metric; +use symbolicator_service::types::ObjectFileStatus; +use symbolicator_sources::ObjectType; + +use crate::interface::{CompleteObjectInfo, CompleteStacktrace}; + +#[derive(Debug, Copy, Clone)] +/// Where the Stack Traces in the [`SymbolicateStacktraces`](crate::interface::SymbolicateStacktraces) +/// originated from. +pub enum StacktraceOrigin { + /// The stack traces came from a direct request to symbolicate. + Symbolicate, + /// The stack traces were extracted from a minidump. + Minidump, + /// The stack traces came from an Apple Crash Report. + AppleCrashReport, +} + +impl std::fmt::Display for StacktraceOrigin { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(match self { + StacktraceOrigin::Symbolicate => "symbolicate", + StacktraceOrigin::Minidump => "minidump", + StacktraceOrigin::AppleCrashReport => "applecrashreport", + }) + } +} + +/// Stacktrace related Metrics +/// +/// This gives some metrics about the quality of the stack traces included +/// in a symbolication request. See the individual members for more information. +/// +/// These numbers are being accumulated across one symbolication request, and are emitted +/// as a histogram. +#[derive(Default)] +pub struct StacktraceMetrics { + /// A truncated stack trace is one that does not end in a + /// well known thread base. + pub truncated_traces: u64, + + /// We classify a short stacktrace as one that has less that 5 frames. + pub short_traces: u64, + + /// This indicated a stack trace that has at least one bad frame + /// from the below categories. + pub bad_traces: u64, + + /// Frames that were scanned. + /// + /// These are frequently wrong and lead to bad and incomplete stack traces. + /// We can improve (lower) these numbers by having more usable CFI info. + pub scanned_frames: u64, + + /// Unsymbolicated Frames. + /// + /// These may be the result of unavailable or broken debug info. + /// We can improve (lower) these numbers by having more usable debug info. + pub unsymbolicated_frames: u64, + + /// Unsymbolicated Context Frames. + /// + /// This is an indication of broken contexts, or failure to extract it from minidumps. + pub unsymbolicated_context_frames: u64, + + /// Unsymbolicated Frames found by scanning. + pub unsymbolicated_scanned_frames: u64, + + /// Unsymbolicated Frames found by CFI. + /// + /// These are the result of the *previous* frame being wrongly scanned. + pub unsymbolicated_cfi_frames: u64, + + /// Frames referencing unmapped memory regions. + /// + /// These may be the result of issues in the client-side module finder, or + /// broken debug-id information. + /// + /// We can improve this by fixing client-side implementations and having + /// proper debug-ids. + pub unmapped_frames: u64, +} + +pub fn record_symbolication_metrics( + origin: StacktraceOrigin, + metrics: StacktraceMetrics, + modules: &[CompleteObjectInfo], + stacktraces: &[CompleteStacktrace], +) { + let origin = origin.to_string(); + + let platform = modules + .iter() + .find_map(|m| { + if m.raw.ty == ObjectType::Unknown { + None + } else { + Some(m.raw.ty) + } + }) + .unwrap_or(ObjectType::Unknown) + .to_string(); + + // Unusable modules that don’t have any kind of ID to look them up with + let mut unusable_modules = 0; + // Modules that failed parsing + let mut unparsable_modules = 0; + + for m in modules { + metric!( + counter("symbolication.debug_status") += 1, + "status" => m.debug_status.name() + ); + + let usable_code_id = !matches!(m.raw.code_id.as_deref(), None | Some("")); + + // NOTE: this is a closure as a way to short-circuit the computation because + // it is expensive + let usable_debug_id = || match m.raw.debug_id.as_deref() { + None | Some("") => false, + Some(string) => string.parse::().is_ok(), + }; + + if !usable_code_id && !usable_debug_id() { + unusable_modules += 1; + } + + if m.debug_status == ObjectFileStatus::Malformed { + unparsable_modules += 1; + } + } + + metric!( + time_raw("symbolication.num_modules") = modules.len() as u64, + "platform" => &platform, "origin" => &origin + ); + metric!( + time_raw("symbolication.unusable_modules") = unusable_modules, + "platform" => &platform, "origin" => &origin + ); + metric!( + time_raw("symbolication.unparsable_modules") = unparsable_modules, + "platform" => &platform, "origin" => &origin + ); + + metric!( + time_raw("symbolication.num_stacktraces") = stacktraces.len() as u64, + "platform" => &platform, "origin" => &origin + ); + metric!( + time_raw("symbolication.short_stacktraces") = metrics.short_traces, + "platform" => &platform, "origin" => &origin + ); + metric!( + time_raw("symbolication.truncated_stacktraces") = metrics.truncated_traces, + "platform" => &platform, "origin" => &origin + ); + metric!( + time_raw("symbolication.bad_stacktraces") = metrics.bad_traces, + "platform" => &platform, "origin" => &origin + ); + + metric!( + time_raw("symbolication.num_frames") = + stacktraces.iter().map(|s| s.frames.len() as u64).sum::(), + "platform" => &platform, "origin" => &origin + ); + metric!( + time_raw("symbolication.scanned_frames") = metrics.scanned_frames, + "platform" => &platform, "origin" => &origin + ); + metric!( + time_raw("symbolication.unsymbolicated_frames") = metrics.unsymbolicated_frames, + "platform" => &platform, "origin" => &origin + ); + metric!( + time_raw("symbolication.unsymbolicated_context_frames") = + metrics.unsymbolicated_context_frames, + "platform" => &platform, "origin" => &origin + ); + metric!( + time_raw("symbolication.unsymbolicated_cfi_frames") = + metrics.unsymbolicated_cfi_frames, + "platform" => &platform, "origin" => &origin + ); + metric!( + time_raw("symbolication.unsymbolicated_scanned_frames") = + metrics.unsymbolicated_scanned_frames, + "platform" => &platform, "origin" => &origin + ); + metric!( + time_raw("symbolication.unmapped_frames") = metrics.unmapped_frames, + "platform" => &platform, "origin" => &origin + ); +} diff --git a/crates/symbolicator-service/src/services/symbolication/apple.rs b/crates/symbolicator-native/src/symbolication/apple.rs similarity index 95% rename from crates/symbolicator-service/src/services/symbolication/apple.rs rename to crates/symbolicator-native/src/symbolication/apple.rs index 917ac87f4..38a66e18c 100644 --- a/crates/symbolicator-service/src/services/symbolication/apple.rs +++ b/crates/symbolicator-native/src/symbolication/apple.rs @@ -5,17 +5,18 @@ use anyhow::{Context, Result}; use apple_crash_report_parser::AppleCrashReport; use chrono::{DateTime, Utc}; use regex::Regex; - use symbolic::common::{Arch, CodeId, DebugId}; +use symbolicator_service::types::{RawObjectInfo, Scope, ScrapingConfig}; +use symbolicator_service::utils::hex::HexValue; use symbolicator_sources::{ObjectType, SourceConfig}; -use crate::types::{ - CompleteObjectInfo, CompletedSymbolicationResponse, RawFrame, RawObjectInfo, RawStacktrace, - Scope, SystemInfo, +use crate::interface::{ + CompleteObjectInfo, CompletedSymbolicationResponse, RawFrame, RawStacktrace, + SymbolicateStacktraces, SystemInfo, }; -use crate::utils::hex::HexValue; +use crate::metrics::StacktraceOrigin; -use super::{ScrapingConfig, StacktraceOrigin, SymbolicateStacktraces, SymbolicationActor}; +use super::symbolicate::SymbolicationActor; impl SymbolicationActor { #[tracing::instrument(skip_all)] diff --git a/crates/symbolicator-native/src/symbolication/demangle.rs b/crates/symbolicator-native/src/symbolication/demangle.rs new file mode 100644 index 000000000..aa2c46b64 --- /dev/null +++ b/crates/symbolicator-native/src/symbolication/demangle.rs @@ -0,0 +1,45 @@ +use symbolic::common::{Language, Name}; +use symbolic::demangle::{Demangle, DemangleOptions}; +use symbolic::symcache::Function; + +/// Options for demangling all symbols. +pub const DEMANGLE_OPTIONS: DemangleOptions = DemangleOptions::complete().return_type(false); + +/// A cache for demangled symbols +pub type DemangleCache = moka::sync::Cache<(String, Language), String>; + +/// Demangles the name of the given [`Function`]. +pub fn demangle_symbol(cache: &DemangleCache, func: &Function) -> (String, String) { + let symbol = func.name(); + let key = (symbol.to_string(), func.language()); + + let init = || { + // Detect the language from the bare name, ignoring any pre-set language. There are a few + // languages that we should always be able to demangle. Only complain about those that we + // detect explicitly, but silently ignore the rest. For instance, there are C-identifiers + // reported as C++, which are expected not to demangle. + let detected_language = Name::from(symbol).detect_language(); + let should_demangle = match (func.language(), detected_language) { + (_, Language::Unknown) => false, // can't demangle what we cannot detect + (Language::ObjCpp, Language::Cpp) => true, // C++ demangles even if it was in ObjC++ + (Language::Unknown, _) => true, // if there was no language, then rely on detection + (lang, detected) => lang == detected, // avoid false-positive detections + }; + + let demangled_opt = func.name_for_demangling().demangle(DEMANGLE_OPTIONS); + if should_demangle && demangled_opt.is_none() { + sentry::with_scope( + |scope| scope.set_extra("identifier", symbol.to_string().into()), + || { + let message = format!("Failed to demangle {} identifier", func.language()); + sentry::capture_message(&message, sentry::Level::Error); + }, + ); + } + demangled_opt.unwrap_or_else(|| symbol.to_string()) + }; + + let entry = cache.entry_by_ref(&key).or_insert_with(init); + + (key.0, entry.into_value()) +} diff --git a/crates/symbolicator-native/src/symbolication/dotnet.rs b/crates/symbolicator-native/src/symbolication/dotnet.rs new file mode 100644 index 000000000..500596124 --- /dev/null +++ b/crates/symbolicator-native/src/symbolication/dotnet.rs @@ -0,0 +1,34 @@ +use symbolic::common::split_path; +use symbolic::ppdb::PortablePdbCache; + +use crate::interface::{FrameStatus, RawFrame, SymbolicatedFrame}; + +pub fn symbolicate_dotnet_frame( + ppdbcache: &PortablePdbCache, + frame: &RawFrame, + index: usize, +) -> Result, FrameStatus> { + // TODO: Add a new error variant for this? + let function_idx = frame.function_id.ok_or(FrameStatus::MissingSymbol)?.0 as u32; + let il_offset = frame.instruction_addr.0 as u32; + + let line_info = ppdbcache + .lookup(function_idx, il_offset) + .ok_or(FrameStatus::MissingSymbol)?; + + let abs_path = line_info.file_name; + let filename = split_path(abs_path).1; + let result = SymbolicatedFrame { + status: FrameStatus::Symbolicated, + original_index: Some(index), + raw: RawFrame { + lang: Some(line_info.file_lang), + filename: Some(filename.to_string()), + abs_path: Some(abs_path.to_string()), + lineno: Some(line_info.line), + ..frame.clone() + }, + }; + + Ok(vec![result]) +} diff --git a/crates/symbolicator-service/src/services/minidump.rs b/crates/symbolicator-native/src/symbolication/minidump_stacktraces.rs similarity index 99% rename from crates/symbolicator-service/src/services/minidump.rs rename to crates/symbolicator-native/src/symbolication/minidump_stacktraces.rs index 8b61de751..68933f9df 100644 --- a/crates/symbolicator-service/src/services/minidump.rs +++ b/crates/symbolicator-native/src/symbolication/minidump_stacktraces.rs @@ -97,10 +97,10 @@ use std::convert::TryFrom; use std::fmt; use symbolic::common::ByteView; +use symbolicator_service::utils::hex; use thiserror::Error; -use crate::types::{self, FrameTrust}; -use crate::utils::hex; +use crate::interface::{self as types, FrameTrust}; const MINIDUMP_EXTENSION_TYPE: u32 = u32::from_be_bytes([b'S', b'y', 0, 1]); const MINIDUMP_FORMAT_VERSION: u32 = 1; diff --git a/crates/symbolicator-native/src/symbolication/mod.rs b/crates/symbolicator-native/src/symbolication/mod.rs new file mode 100644 index 000000000..1a4b84eda --- /dev/null +++ b/crates/symbolicator-native/src/symbolication/mod.rs @@ -0,0 +1,9 @@ +mod apple; +mod demangle; +mod dotnet; +mod minidump_stacktraces; +mod module_lookup; +mod native; +mod process_minidump; +mod source_context; +pub mod symbolicate; diff --git a/crates/symbolicator-service/src/services/module_lookup.rs b/crates/symbolicator-native/src/symbolication/module_lookup.rs similarity index 96% rename from crates/symbolicator-service/src/services/module_lookup.rs rename to crates/symbolicator-native/src/symbolication/module_lookup.rs index e2f24f4da..f90f9da1d 100644 --- a/crates/symbolicator-service/src/services/module_lookup.rs +++ b/crates/symbolicator-native/src/symbolication/module_lookup.rs @@ -5,22 +5,21 @@ use futures::future; use sentry::{Hub, SentryFutureExt}; use symbolic::debuginfo::ObjectDebugSession; +use symbolicator_service::caching::{CacheEntry, CacheError}; +use symbolicator_service::objects::{ + AllObjectCandidates, FindObject, FindResult, ObjectFeatures, ObjectHandle, ObjectPurpose, + ObjectsActor, +}; +use symbolicator_service::source_context::get_context_lines; +use symbolicator_service::types::{ObjectFileStatus, RawObjectInfo, Scope}; use symbolicator_sources::{FileType, ObjectId, ObjectType, SourceConfig}; -use crate::caching::{CacheEntry, CacheError}; -use crate::services::derived::DerivedCache; -use crate::services::objects::{FindObject, FindResult, ObjectHandle, ObjectPurpose, ObjectsActor}; -use crate::services::ppdb_caches::{ +use crate::caches::derived::DerivedCache; +use crate::caches::ppdb_caches::{ FetchPortablePdbCache, OwnedPortablePdbCache, PortablePdbCacheActor, }; -use crate::services::symcaches::{FetchSymCache, OwnedSymCache, SymCacheActor}; -use crate::types::{ - AllObjectCandidates, CompleteObjectInfo, CompleteStacktrace, ObjectFeatures, ObjectFileStatus, - RawFrame, RawObjectInfo, RawStacktrace, Scope, -}; -use crate::utils::addr::AddrMode; - -use super::symbolication::source_context::get_context_lines; +use crate::caches::symcaches::{FetchSymCache, OwnedSymCache, SymCacheActor}; +use crate::interface::{AddrMode, CompleteObjectInfo, CompleteStacktrace, RawFrame, RawStacktrace}; fn object_id_from_object_info(object_info: &RawObjectInfo) -> ObjectId { ObjectId { @@ -477,8 +476,8 @@ impl ModuleLookup { #[cfg(test)] mod tests { - use crate::types::RawObjectInfo; - use crate::utils::hex::HexValue; + use symbolicator_service::types::RawObjectInfo; + use symbolicator_service::utils::hex::HexValue; use super::*; diff --git a/crates/symbolicator-native/src/symbolication/native.rs b/crates/symbolicator-native/src/symbolication/native.rs new file mode 100644 index 000000000..596ac9e68 --- /dev/null +++ b/crates/symbolicator-native/src/symbolication/native.rs @@ -0,0 +1,147 @@ +use symbolic::common::{split_path, InstructionInfo, Language}; +use symbolic::symcache::SymCache; +use symbolicator_service::metric; +use symbolicator_service::utils::hex::HexValue; + +use crate::interface::{ + AdjustInstructionAddr, FrameStatus, RawFrame, Registers, Signal, SymbolicatedFrame, +}; + +use super::demangle::{demangle_symbol, DemangleCache}; +use super::module_lookup::CacheLookupResult; + +pub fn symbolicate_native_frame( + demangle_cache: &DemangleCache, + symcache: &SymCache, + lookup_result: CacheLookupResult, + relative_addr: u64, + frame: &RawFrame, + index: usize, +) -> Result, FrameStatus> { + tracing::trace!("Symbolicating {:#x}", relative_addr); + let mut rv = vec![]; + + // The symbol addr only makes sense for the outermost top-level function, and not its inlinees. + // We keep track of it while iterating and only set it for the last frame, + // which is the top-level function. + let mut sym_addr = None; + let instruction_addr = HexValue(lookup_result.expose_preferred_addr(relative_addr)); + + for source_location in symcache.lookup(relative_addr) { + let abs_path = source_location + .file() + .map(|f| f.full_path()) + .unwrap_or_default(); + let filename = split_path(&abs_path).1; + + let func = source_location.function(); + let (symbol, function) = demangle_symbol(demangle_cache, &func); + + sym_addr = Some(HexValue( + lookup_result.expose_preferred_addr(func.entry_pc() as u64), + )); + let filename = if !filename.is_empty() { + Some(filename.to_string()) + } else { + frame.filename.clone() + }; + rv.push(SymbolicatedFrame { + status: FrameStatus::Symbolicated, + original_index: Some(index), + raw: RawFrame { + package: lookup_result.object_info.raw.code_file.clone(), + addr_mode: lookup_result.preferred_addr_mode(), + instruction_addr, + adjust_instruction_addr: frame.adjust_instruction_addr, + function_id: frame.function_id, + symbol: Some(symbol), + abs_path: if !abs_path.is_empty() { + Some(abs_path) + } else { + frame.abs_path.clone() + }, + function: Some(function), + filename, + lineno: Some(source_location.line()), + pre_context: vec![], + context_line: None, + post_context: vec![], + source_link: None, + sym_addr: None, + lang: match func.language() { + Language::Unknown => None, + language => Some(language), + }, + in_app: None, + trust: frame.trust, + }, + }); + } + + if let Some(last_frame) = rv.last_mut() { + last_frame.raw.sym_addr = sym_addr; + } + + if rv.is_empty() { + return Err(FrameStatus::MissingSymbol); + } + + Ok(rv) +} + +pub fn get_relative_caller_addr( + symcache: &SymCache, + lookup_result: &CacheLookupResult, + registers: &Registers, + signal: Option, + index: usize, + adjustment: AdjustInstructionAddr, +) -> Result { + if let Some(addr) = lookup_result.relative_addr { + // heuristics currently are only supported when we can work with absolute addresses. + // In cases where this is not possible we skip this part entirely and use the relative + // address calculated by the lookup result as lookup address in the module. + if let Some(absolute_addr) = lookup_result.object_info.rel_to_abs_addr(addr) { + let is_crashing_frame = index == 0; + let ip_register_value = if is_crashing_frame { + symcache + .arch() + .cpu_family() + .ip_register_name() + .and_then(|ip_reg_name| registers.get(ip_reg_name)) + .map(|x| x.0) + } else { + None + }; + + let mut instruction_info = InstructionInfo::new(symcache.arch(), absolute_addr); + let instruction_info = instruction_info + .is_crashing_frame(is_crashing_frame) + .signal(signal.map(|signal| signal.0)) + .ip_register_value(ip_register_value); + + let absolute_caller_addr = match adjustment { + AdjustInstructionAddr::Yes => instruction_info.previous_address(), + AdjustInstructionAddr::No => instruction_info.aligned_address(), + AdjustInstructionAddr::Auto => instruction_info.caller_address(), + }; + + lookup_result + .object_info + .abs_to_rel_addr(absolute_caller_addr) + .ok_or_else(|| { + tracing::warn!( + "Underflow when trying to subtract image start addr from caller address after heuristics" + ); + metric!(counter("relative_addr.underflow") += 1); + FrameStatus::MissingSymbol + }) + } else { + Ok(addr) + } + } else { + tracing::warn!("Underflow when trying to subtract image start addr from caller address before heuristics"); + metric!(counter("relative_addr.underflow") += 1); + Err(FrameStatus::MissingSymbol) + } +} diff --git a/crates/symbolicator-service/src/services/symbolication/process_minidump.rs b/crates/symbolicator-native/src/symbolication/process_minidump.rs similarity index 97% rename from crates/symbolicator-service/src/services/symbolication/process_minidump.rs rename to crates/symbolicator-native/src/symbolication/process_minidump.rs index d1e6ea104..1798cff55 100644 --- a/crates/symbolicator-service/src/services/symbolication/process_minidump.rs +++ b/crates/symbolicator-native/src/symbolication/process_minidump.rs @@ -15,21 +15,23 @@ use minidump_unwind::{ }; use sentry::{Hub, SentryFutureExt}; use serde::{Deserialize, Serialize}; -use tempfile::TempPath; - use symbolic::common::{Arch, ByteView, CodeId, DebugId}; +use symbolicator_service::metric; +use symbolicator_service::types::{ObjectFileStatus, RawObjectInfo, Scope, ScrapingConfig}; +use symbolicator_service::utils::hex::HexValue; use symbolicator_sources::{ObjectId, ObjectType, SourceConfig}; +use tempfile::TempPath; -use crate::services::cficaches::{CfiCacheActor, CfiModuleInfo, FetchCfiCache, FetchedCfiCache}; -use crate::services::minidump::parse_stacktraces_from_minidump; -use crate::services::module_lookup::object_file_status_from_cache_entry; -use crate::types::{ - CompleteObjectInfo, CompletedSymbolicationResponse, ObjectFileStatus, RawFrame, RawObjectInfo, - RawStacktrace, Registers, Scope, SystemInfo, +use crate::caches::cficaches::{CfiCacheActor, CfiModuleInfo, FetchCfiCache, FetchedCfiCache}; +use crate::interface::{ + CompleteObjectInfo, CompletedSymbolicationResponse, RawFrame, RawStacktrace, Registers, + SymbolicateStacktraces, SystemInfo, }; -use crate::utils::hex::HexValue; +use crate::metrics::StacktraceOrigin; -use super::{ScrapingConfig, StacktraceOrigin, SymbolicateStacktraces, SymbolicationActor}; +use super::minidump_stacktraces::parse_stacktraces_from_minidump; +use super::module_lookup::object_file_status_from_cache_entry; +use super::symbolicate::SymbolicationActor; type Minidump = minidump::Minidump<'static, ByteView<'static>>; diff --git a/crates/symbolicator-native/src/symbolication/source_context.rs b/crates/symbolicator-native/src/symbolication/source_context.rs new file mode 100644 index 000000000..6922edde8 --- /dev/null +++ b/crates/symbolicator-native/src/symbolication/source_context.rs @@ -0,0 +1,75 @@ +use std::collections::HashMap; + +use futures::future; +use symbolicator_service::types::{Scope, ScrapingConfig}; +use symbolicator_service::utils::http::is_valid_origin; +use symbolicator_sources::HttpRemoteFile; + +use crate::interface::{CompleteStacktrace, RawFrame}; + +use super::module_lookup::ModuleLookup; +use super::symbolicate::SymbolicationActor; + +impl SymbolicationActor { + pub async fn apply_source_context( + &self, + module_lookup: &mut ModuleLookup, + stacktraces: &mut [CompleteStacktrace], + scraping: &ScrapingConfig, + ) { + module_lookup + .fetch_sources(self.objects.clone(), stacktraces) + .await; + + // Map collected source contexts to frames and collect URLs for remote source links. + let mut remote_sources: HashMap<(Scope, url::Url), Vec<&mut RawFrame>> = HashMap::new(); + { + let debug_sessions = module_lookup.prepare_debug_sessions(); + + for trace in stacktraces { + for frame in &mut trace.frames { + let Some(url) = + module_lookup.try_set_source_context(&debug_sessions, &mut frame.raw) + else { + continue; + }; + if let Some(vec) = remote_sources.get_mut(&url) { + vec.push(&mut frame.raw) + } else { + remote_sources.insert(url, vec![&mut frame.raw]); + } + } + } + } + + // Download remote sources and update contexts. + if !remote_sources.is_empty() { + let cache = self.sourcefiles_cache.as_ref(); + let futures = + remote_sources + .into_iter() + .map(|((source_scope, url), frames)| async move { + let mut remote_file = HttpRemoteFile::from_url(url.clone()); + + if scraping.enabled && is_valid_origin(&url, &scraping.allowed_origins) { + remote_file.headers.extend( + scraping + .headers + .iter() + .map(|(key, value)| (key.clone(), value.clone())), + ); + } + + if let Ok(source) = cache + .fetch_file(&source_scope, remote_file.into(), true) + .await + { + for frame in frames { + ModuleLookup::set_source_context(&source, frame); + } + } + }); + future::join_all(futures).await; + } + } +} diff --git a/crates/symbolicator-native/src/symbolication/symbolicate.rs b/crates/symbolicator-native/src/symbolication/symbolicate.rs new file mode 100644 index 000000000..0f6384615 --- /dev/null +++ b/crates/symbolicator-native/src/symbolication/symbolicate.rs @@ -0,0 +1,351 @@ +use std::sync::Arc; + +use symbolic::common::Name; +use symbolic::demangle::Demangle; +use symbolicator_service::caches::SourceFilesCache; +use symbolicator_service::caching::{Cache, CacheError}; +use symbolicator_service::objects::ObjectsActor; +use symbolicator_service::services::SharedServices; + +use crate::caches::bitcode::BitcodeService; +use crate::caches::cficaches::CfiCacheActor; +use crate::caches::il2cpp::Il2cppService; +use crate::caches::ppdb_caches::PortablePdbCacheActor; +use crate::caches::symcaches::SymCacheActor; +use crate::interface::{ + AdjustInstructionAddr, CompleteStacktrace, CompletedSymbolicationResponse, FrameStatus, + FrameTrust, RawFrame, RawStacktrace, Registers, Signal, SymbolicateStacktraces, + SymbolicatedFrame, +}; +use crate::metrics::{record_symbolication_metrics, StacktraceMetrics}; + +use super::demangle::{DemangleCache, DEMANGLE_OPTIONS}; +use super::dotnet::symbolicate_dotnet_frame; +use super::module_lookup::{CacheFileEntry, ModuleLookup}; +use super::native::{get_relative_caller_addr, symbolicate_native_frame}; + +// we should really rename this here to the `SymbolicatorService`, as it does a lot more +// than just symbolication ;-) +#[derive(Clone, Debug)] +pub struct SymbolicationActor { + demangle_cache: DemangleCache, + pub(crate) objects: ObjectsActor, + symcaches: SymCacheActor, + pub(crate) cficaches: CfiCacheActor, + ppdb_caches: PortablePdbCacheActor, + pub(crate) diagnostics_cache: Cache, + pub(crate) sourcefiles_cache: Arc, +} + +impl SymbolicationActor { + pub fn new(services: &SharedServices) -> Self { + let caches = &services.caches; + let shared_cache = services.shared_cache.clone(); + let objects = services.objects.clone(); + let download_svc = services.download_svc.clone(); + let sourcefiles_cache = services.sourcefiles_cache.clone(); + + let bitcode = BitcodeService::new( + caches.auxdifs.clone(), + shared_cache.clone(), + download_svc.clone(), + ); + + let il2cpp = Il2cppService::new(caches.il2cpp.clone(), shared_cache.clone(), download_svc); + + let symcaches = SymCacheActor::new( + caches.symcaches.clone(), + shared_cache.clone(), + objects.clone(), + bitcode, + il2cpp, + ); + + let cficaches = CfiCacheActor::new( + caches.cficaches.clone(), + shared_cache.clone(), + objects.clone(), + ); + + let ppdb_caches = + PortablePdbCacheActor::new(caches.ppdb_caches.clone(), shared_cache, objects.clone()); + + let demangle_cache = DemangleCache::builder() + .max_capacity(10 * 1024 * 1024) // 10 MiB, considering key and value: + .weigher(|k, v| (k.0.len() + v.len()).try_into().unwrap_or(u32::MAX)) + .build(); + + SymbolicationActor { + demangle_cache, + objects, + symcaches, + cficaches, + ppdb_caches, + diagnostics_cache: caches.diagnostics.clone(), + sourcefiles_cache, + } + } + + #[tracing::instrument(skip_all)] + pub async fn symbolicate( + &self, + request: SymbolicateStacktraces, + ) -> anyhow::Result { + let SymbolicateStacktraces { + stacktraces, + sources, + scope, + signal, + origin, + modules, + apply_source_context, + scraping, + } = request; + + let mut module_lookup = ModuleLookup::new(scope.clone(), sources, modules); + module_lookup + .fetch_caches( + self.symcaches.clone(), + self.ppdb_caches.clone(), + &stacktraces, + ) + .await; + + let mut metrics = StacktraceMetrics::default(); + let mut stacktraces: Vec<_> = stacktraces + .into_iter() + .map(|trace| { + symbolicate_stacktrace( + &self.demangle_cache, + trace, + &module_lookup, + &mut metrics, + signal, + ) + }) + .collect(); + + if apply_source_context { + self.apply_source_context(&mut module_lookup, &mut stacktraces, &scraping) + .await + } + + // bring modules back into the original order + let modules = module_lookup.into_inner(); + record_symbolication_metrics(origin, metrics, &modules, &stacktraces); + + Ok(CompletedSymbolicationResponse { + signal, + stacktraces, + modules, + ..Default::default() + }) + } +} + +fn symbolicate_stacktrace( + demangle_cache: &DemangleCache, + thread: RawStacktrace, + caches: &ModuleLookup, + metrics: &mut StacktraceMetrics, + signal: Option, +) -> CompleteStacktrace { + let default_adjustment = AdjustInstructionAddr::default_for_thread(&thread); + let mut symbolicated_frames = vec![]; + let mut unsymbolicated_frames_iter = thread.frames.into_iter().enumerate().peekable(); + + while let Some((index, mut frame)) = unsymbolicated_frames_iter.next() { + let adjustment = AdjustInstructionAddr::for_frame(&frame, default_adjustment); + match symbolicate_frame( + demangle_cache, + caches, + &thread.registers, + signal, + &mut frame, + index, + adjustment, + ) { + Ok(frames) => { + if matches!(frame.trust, FrameTrust::Scan) { + metrics.scanned_frames += 1; + } + symbolicated_frames.extend(frames) + } + Err(status) => { + // Since symbolication failed, the function name was not demangled. In case there is + // either one of `function` or `symbol`, treat that as mangled name and try to + // demangle it. If that succeeds, write the demangled name back. + let mangled = frame.function.as_deref().xor(frame.symbol.as_deref()); + let demangled = mangled.and_then(|m| Name::from(m).demangle(DEMANGLE_OPTIONS)); + if let Some(demangled) = demangled { + if let Some(old_mangled) = frame.function.replace(demangled) { + frame.symbol = Some(old_mangled); + } + } + + // Temporary workaround: Skip false-positive frames from stack scanning after the + // fact. + // + // Usually, the stack scanner would skip all scanned frames when it *knows* that + // they cannot be symbolized. However, in our case we supply breakpad symbols + // without function records. This breaks its original heuristic, since it would now + // *always* skip scan frames. Our patch in breakpad omits this check. + // + // Here, we fix this after the fact. + // + // - MissingSymbol: If symbolication failed for a scanned frame where we *know* we + // have a debug info, but the lookup inside that file failed. + // - UnknownImage: If symbolication failed because the stackscanner found an + // instruction_addr that is not in any image *we* consider valid. We discard + // images which do not have a debug id, while the stackscanner considers them + // perfectly fine. + if frame.trust == FrameTrust::Scan + && (status == FrameStatus::MissingSymbol || status == FrameStatus::UnknownImage) + { + continue; + } + + // Glibc inserts an explicit `DW_CFA_undefined: RIP` DWARF rule to say that `_start` + // has no return address. + // See https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/x86_64/start.S;h=1b3e36826b8a477474cee24d1c931429fbdf6d8f;hb=HEAD#l59 + // We do not support this due to lack of breakpad support, and will thus use the + // previous rule for RIP, which says to look it up the value on the stack, + // resulting in an unmapped garbage frame. We work around this by trimming the + // trailing garbage frame on the following conditions: + // * it is unmapped (UnknownImage) + // * this is the last frame to symbolicate (via peek) + // * the previous symbolicated frame is `_start` + let is_start = + |frame: &SymbolicatedFrame| frame.raw.function.as_deref() == Some("_start"); + if status == FrameStatus::UnknownImage + && unsymbolicated_frames_iter.peek().is_none() + && symbolicated_frames.last().map_or(false, is_start) + { + continue; + } + + metrics.unsymbolicated_frames += 1; + match frame.trust { + FrameTrust::Scan => { + metrics.scanned_frames += 1; + metrics.unsymbolicated_scanned_frames += 1; + } + FrameTrust::Cfi => metrics.unsymbolicated_cfi_frames += 1, + FrameTrust::Context => metrics.unsymbolicated_context_frames += 1, + _ => {} + } + if status == FrameStatus::UnknownImage { + metrics.unmapped_frames += 1; + } + + symbolicated_frames.push(SymbolicatedFrame { + status, + original_index: Some(index), + raw: frame, + }); + } + } + } + + // we try to find a base frame among the bottom 5 + if !symbolicated_frames + .iter() + .rev() + .take(5) + .any(is_likely_base_frame) + { + metrics.truncated_traces += 1; + } + // macOS has some extremely short but perfectly fine stacks, such as: + // `__workq_kernreturn` > `_pthread_wqthread` > `start_wqthread` + if symbolicated_frames.len() < 3 { + metrics.short_traces += 1; + } + + if metrics.scanned_frames > 0 || metrics.unsymbolicated_frames > 0 { + metrics.bad_traces += 1; + } + + CompleteStacktrace { + thread_id: thread.thread_id, + thread_name: thread.thread_name, + is_requesting: thread.is_requesting, + registers: thread.registers, + frames: symbolicated_frames, + } +} + +fn symbolicate_frame( + demangle_cache: &DemangleCache, + caches: &ModuleLookup, + registers: &Registers, + signal: Option, + frame: &mut RawFrame, + index: usize, + adjustment: AdjustInstructionAddr, +) -> Result, FrameStatus> { + let lookup_result = caches + .lookup_cache(frame.instruction_addr.0, frame.addr_mode) + .ok_or(FrameStatus::UnknownImage)?; + + frame.package = lookup_result.object_info.raw.code_file.clone(); + + match lookup_result.cache { + Ok(CacheFileEntry::SymCache(symcache)) => { + let symcache = symcache.get(); + let relative_addr = get_relative_caller_addr( + symcache, + &lookup_result, + registers, + signal, + index, + adjustment, + )?; + symbolicate_native_frame( + demangle_cache, + symcache, + lookup_result, + relative_addr, + frame, + index, + ) + } + Ok(CacheFileEntry::PortablePdbCache(ppdb_cache)) => { + symbolicate_dotnet_frame(ppdb_cache.get(), frame, index) + } + Err(CacheError::Malformed(_)) => Err(FrameStatus::Malformed), + _ => Err(FrameStatus::Missing), + } +} + +/// Determine if the [`SymbolicatedFrame`] is likely to be a thread base. +/// +/// This is just a heuristic that matches the function to well known thread entry points. +fn is_likely_base_frame(frame: &SymbolicatedFrame) -> bool { + let function = match frame + .raw + .function + .as_deref() + .or(frame.raw.symbol.as_deref()) + { + Some(f) => f, + None => return false, + }; + + // C start/main + if matches!(function, "main" | "start" | "_start") { + return true; + } + + // Windows and posix thread base. These often have prefixes depending on the OS and Version, so + // we use a substring match here. + if function.contains("UserThreadStart") + || function.contains("thread_start") + || function.contains("start_thread") + || function.contains("start_wqthread") + { + return true; + } + + false +} diff --git a/crates/symbolicator-service/tests/integration/e2e.rs b/crates/symbolicator-native/tests/integration/e2e.rs similarity index 99% rename from crates/symbolicator-service/tests/integration/e2e.rs rename to crates/symbolicator-native/tests/integration/e2e.rs index 5d803d6f1..92b25a4d1 100644 --- a/crates/symbolicator-service/tests/integration/e2e.rs +++ b/crates/symbolicator-native/tests/integration/e2e.rs @@ -1,8 +1,9 @@ use std::path::Path; use std::sync::Arc; -use symbolicator_service::services::symbolication::SymbolicateStacktraces; -use symbolicator_service::types::{FrameStatus, ObjectDownloadInfo, ObjectFileStatus, Scope}; +use symbolicator_native::interface::{FrameStatus, SymbolicateStacktraces}; +use symbolicator_service::objects::ObjectDownloadInfo; +use symbolicator_service::types::{ObjectFileStatus, Scope}; use symbolicator_sources::{ DirectoryLayoutType, FileType, FilesystemSourceConfig, HttpSourceConfig, RemoteFileUri, SentrySourceConfig, SourceConfig, SourceId, diff --git a/crates/symbolicator-service/tests/integration/main.rs b/crates/symbolicator-native/tests/integration/main.rs similarity index 100% rename from crates/symbolicator-service/tests/integration/main.rs rename to crates/symbolicator-native/tests/integration/main.rs diff --git a/crates/symbolicator-service/tests/integration/process_minidump.rs b/crates/symbolicator-native/tests/integration/process_minidump.rs similarity index 100% rename from crates/symbolicator-service/tests/integration/process_minidump.rs rename to crates/symbolicator-native/tests/integration/process_minidump.rs diff --git a/crates/symbolicator-service/tests/integration/public_sources.rs b/crates/symbolicator-native/tests/integration/public_sources.rs similarity index 100% rename from crates/symbolicator-service/tests/integration/public_sources.rs rename to crates/symbolicator-native/tests/integration/public_sources.rs diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__e2e__no_sources.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__e2e__no_sources.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__e2e__no_sources.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__e2e__no_sources.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__e2e__sources_filetypes.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__e2e__sources_filetypes.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__e2e__sources_filetypes.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__e2e__sources_filetypes.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__process_minidump__minidump_linux.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__process_minidump__minidump_linux.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__process_minidump__minidump_linux.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__process_minidump__minidump_linux.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__process_minidump__minidump_macos.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__process_minidump__minidump_macos.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__process_minidump__minidump_macos.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__process_minidump__minidump_macos.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__process_minidump__minidump_windows.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__process_minidump__minidump_windows.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__process_minidump__minidump_windows.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__process_minidump__minidump_windows.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__public_sources__nuget_source.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__public_sources__nuget_source.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__public_sources__nuget_source.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__public_sources__nuget_source.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__public_sources__ubuntu_source.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__public_sources__ubuntu_source.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__public_sources__ubuntu_source.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__public_sources__ubuntu_source.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__add_bucket-2.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__add_bucket-2.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__add_bucket-2.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__add_bucket-2.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__add_bucket.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__add_bucket.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__add_bucket.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__add_bucket.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__apple_crash_report.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__apple_crash_report.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__apple_crash_report.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__apple_crash_report.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__dotnet_embedded_sources.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__dotnet_embedded_sources.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__dotnet_embedded_sources.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__dotnet_embedded_sources.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__dotnet_integration.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__dotnet_integration.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__dotnet_integration.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__dotnet_integration.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__dotnet_only_source_links.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__dotnet_only_source_links.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__dotnet_only_source_links.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__dotnet_only_source_links.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__dotnet_source_links.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__dotnet_source_links.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__dotnet_source_links.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__dotnet_source_links.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__remove_bucket-2.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__remove_bucket-2.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__remove_bucket-2.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__remove_bucket-2.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__remove_bucket.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__remove_bucket.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__remove_bucket.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__remove_bucket.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__source_candidates.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__source_candidates.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__source_candidates.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__source_candidates.snap diff --git a/crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__wasm_payload.snap b/crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__wasm_payload.snap similarity index 100% rename from crates/symbolicator-service/tests/integration/snapshots/integration__symbolication__wasm_payload.snap rename to crates/symbolicator-native/tests/integration/snapshots/integration__symbolication__wasm_payload.snap diff --git a/crates/symbolicator-service/tests/integration/source_errors.rs b/crates/symbolicator-native/tests/integration/source_errors.rs similarity index 97% rename from crates/symbolicator-service/tests/integration/source_errors.rs rename to crates/symbolicator-native/tests/integration/source_errors.rs index 47dc87258..1b58e49fb 100644 --- a/crates/symbolicator-service/tests/integration/source_errors.rs +++ b/crates/symbolicator-native/tests/integration/source_errors.rs @@ -1,9 +1,8 @@ use std::time::Duration; -use symbolicator_service::types::{ - CompletedSymbolicationResponse, FrameStatus, ObjectDownloadInfo, ObjectFileStatus, - ObjectUseInfo, -}; +use symbolicator_native::interface::{CompletedSymbolicationResponse, FrameStatus}; +use symbolicator_service::objects::{ObjectDownloadInfo, ObjectUseInfo}; +use symbolicator_service::types::ObjectFileStatus; use crate::{example_request, setup_service, Server}; diff --git a/crates/symbolicator-service/tests/integration/symbolication.rs b/crates/symbolicator-native/tests/integration/symbolication.rs similarity index 100% rename from crates/symbolicator-service/tests/integration/symbolication.rs rename to crates/symbolicator-native/tests/integration/symbolication.rs diff --git a/crates/symbolicator-service/tests/integration/utils.rs b/crates/symbolicator-native/tests/integration/utils.rs similarity index 95% rename from crates/symbolicator-service/tests/integration/utils.rs rename to crates/symbolicator-native/tests/integration/utils.rs index 161752377..858deed9a 100644 --- a/crates/symbolicator-service/tests/integration/utils.rs +++ b/crates/symbolicator-native/tests/integration/utils.rs @@ -1,9 +1,8 @@ use std::sync::Arc; +use symbolicator_native::interface::{StacktraceOrigin, SymbolicateStacktraces}; +use symbolicator_native::SymbolicationActor; use symbolicator_service::config::Config; -use symbolicator_service::services::symbolication::{ - StacktraceOrigin, SymbolicateStacktraces, SymbolicationActor, -}; use symbolicator_service::services::SharedServices; use symbolicator_service::types::RawObjectInfo; use symbolicator_sources::SourceConfig; diff --git a/crates/symbolicator-service/Cargo.toml b/crates/symbolicator-service/Cargo.toml index 688fc01a0..d2b859c86 100644 --- a/crates/symbolicator-service/Cargo.toml +++ b/crates/symbolicator-service/Cargo.toml @@ -10,16 +10,12 @@ https = [] [dependencies] anyhow = "1.0.57" -apple-crash-report-parser = "0.5.1" -async-trait = "0.1.53" aws-config = "0.56.0" aws-sdk-s3 = "0.31.1" aws-types = "0.56.0" aws-credential-types = { version = "0.56.0", features = ["hardcoded-credentials"] } -backtrace = "0.3.65" cadence = "0.29.0" chrono = { version = "0.4.19", features = ["serde"] } -data-url = "0.3.0" filetime = "0.2.16" flate2 = "1.0.23" futures = "0.3.12" @@ -30,21 +26,17 @@ idna = "0.4.0" ipnetwork = "0.20.0" jsonwebtoken = "8.1.0" lazy_static = "1.4.0" -minidump = "0.18.0" -minidump-processor = "0.18.0" -minidump-unwind = "0.18.0" moka = { version = "0.12.1", features = ["future", "sync"] } once_cell = "1.17.1" parking_lot = "0.12.0" rand = "0.8.5" -regex = "1.5.5" reqwest = { version = "0.11.0", features = ["gzip", "brotli", "deflate", "json", "stream", "trust-dns"] } sentry = { version = "0.31.7", features = ["tracing"] } serde = { version = "1.0.137", features = ["derive", "rc"] } serde_json = "1.0.81" serde_yaml = "0.9.14" sha2 = "0.10.6" -symbolic = { version = "12.4.0", features = ["cfi", "common-serde", "debuginfo", "demangle", "sourcemapcache", "symcache", "il2cpp", "ppdb"] } +symbolic = { version = "12.4.0", features = ["cfi", "common-serde", "debuginfo", "symcache"] } symbolicator-sources = { path = "../symbolicator-sources" } tempfile = "3.2.0" thiserror = "1.0.31" @@ -57,8 +49,5 @@ zip = { version = "0.6.4", default-features = false, features = ["deflate"] } zstd = "0.12.1" [dev-dependencies] -insta = { version = "1.18.0", features = ["redactions", "yaml"] } -reqwest = { version = "0.11.0", features = ["multipart"] } sha-1 = "0.10.0" symbolicator-test = { path = "../symbolicator-test" } -test-assembler = "0.1.5" diff --git a/crates/symbolicator-service/src/services/caches/mod.rs b/crates/symbolicator-service/src/caches/mod.rs similarity index 100% rename from crates/symbolicator-service/src/services/caches/mod.rs rename to crates/symbolicator-service/src/caches/mod.rs diff --git a/crates/symbolicator-service/src/services/caches/sourcefiles.rs b/crates/symbolicator-service/src/caches/sourcefiles.rs similarity index 97% rename from crates/symbolicator-service/src/services/caches/sourcefiles.rs rename to crates/symbolicator-service/src/caches/sourcefiles.rs index 49bf545a4..05aa964a1 100644 --- a/crates/symbolicator-service/src/services/caches/sourcefiles.rs +++ b/crates/symbolicator-service/src/caches/sourcefiles.rs @@ -10,8 +10,7 @@ use crate::caching::{ Cache, CacheEntry, CacheError, CacheItemRequest, CacheKey, CacheVersions, Cacher, SharedCacheRef, }; -use crate::services::download::DownloadService; -use crate::services::fetch_file; +use crate::download::{fetch_file, DownloadService}; use crate::types::Scope; use super::versions::SOURCEFILES_CACHE_VERSIONS; diff --git a/crates/symbolicator-service/src/services/caches/versions.rs b/crates/symbolicator-service/src/caches/versions.rs similarity index 100% rename from crates/symbolicator-service/src/services/caches/versions.rs rename to crates/symbolicator-service/src/caches/versions.rs diff --git a/crates/symbolicator-service/src/caching/mod.rs b/crates/symbolicator-service/src/caching/mod.rs index 0bd0c8576..7e26e6ebe 100644 --- a/crates/symbolicator-service/src/caching/mod.rs +++ b/crates/symbolicator-service/src/caching/mod.rs @@ -141,7 +141,7 @@ //! //! A [`CacheItemRequest`] also needs to specify [`CacheVersions`] which are used for cache fallback //! as explained in detail above. Newly added caches should start with version `1`, and all the -//! cache versions and their versioning history should be recorded in [`caches::versions`](crate::services::caches::versions). +//! cache versions and their versioning history should be recorded in [`caches::versions`](crate::caches::versions). //! //! A new cache item also needs a new [`CacheName`] and [`Cache`] configuration. This should be added //! to [`Caches`] down below as well, and to [`Caches::cleanup`] to properly clean up cache files. @@ -175,18 +175,17 @@ pub use memory::{CacheItemRequest, CacheVersions, Cacher}; pub use shared_cache::{CacheStoreReason, SharedCacheConfig, SharedCacheRef, SharedCacheService}; pub struct Caches { - /// Caches for object files, used by [`crate::services::objects::ObjectsActor`]. + /// Caches for object files. pub objects: Cache, - /// Caches for object metadata, used by [`crate::services::objects::ObjectsActor`]. + /// Caches for object metadata. pub object_meta: Cache, - /// Caches for auxiliary DIF files, used by [`crate::services::bitcode::BitcodeService`]. + /// Caches for auxiliary DIF files. pub auxdifs: Cache, - /// Caches for il2cpp line mapping files, used by [`crate::services::il2cpp::Il2cppService`]. + /// Caches for il2cpp line mapping files. pub il2cpp: Cache, - /// Caches for [`symbolic::symcache::SymCache`], used by - /// [`crate::services::symcaches::SymCacheActor`]. + /// Caches for [`symbolic::symcache::SymCache`], used by. pub symcaches: Cache, - /// Caches for breakpad CFI info, used by [`crate::services::cficaches::CfiCacheActor`]. + /// Caches for breakpad CFI info. pub cficaches: Cache, /// PortablePDB files. pub ppdb_caches: Cache, @@ -196,8 +195,7 @@ pub struct Caches { pub sourcefiles: Cache, /// Bundle Indexes. pub bundle_index: Cache, - /// Store for diagnostics data symbolicator failed to process, used by - /// [`crate::services::symbolication::SymbolicationActor`]. + /// Store for minidump data symbolicator failed to process, for diagnostics purposes pub diagnostics: Cache, } diff --git a/crates/symbolicator-service/src/caching/shared_cache/mod.rs b/crates/symbolicator-service/src/caching/shared_cache/mod.rs index 60b0c2e6c..55406edb2 100644 --- a/crates/symbolicator-service/src/caching/shared_cache/mod.rs +++ b/crates/symbolicator-service/src/caching/shared_cache/mod.rs @@ -25,7 +25,7 @@ use tokio::sync::{mpsc, oneshot, OnceCell}; use tokio_util::io::{ReaderStream, StreamReader}; use url::Url; -use crate::services::download::MeasureSourceDownloadGuard; +use crate::download::MeasureSourceDownloadGuard; use crate::utils::futures::CancelOnDrop; use crate::utils::gcs::{self, GcsError}; diff --git a/crates/symbolicator-service/src/utils/compression.rs b/crates/symbolicator-service/src/download/compression.rs similarity index 100% rename from crates/symbolicator-service/src/utils/compression.rs rename to crates/symbolicator-service/src/download/compression.rs diff --git a/crates/symbolicator-service/src/services/fetch_file.rs b/crates/symbolicator-service/src/download/fetch_file.rs similarity index 92% rename from crates/symbolicator-service/src/services/fetch_file.rs rename to crates/symbolicator-service/src/download/fetch_file.rs index f167f16fa..76bdade09 100644 --- a/crates/symbolicator-service/src/services/fetch_file.rs +++ b/crates/symbolicator-service/src/download/fetch_file.rs @@ -4,9 +4,9 @@ use std::sync::Arc; use symbolicator_sources::RemoteFile; use tempfile::NamedTempFile; +use super::compression::maybe_decompress_file; +use super::DownloadService; use crate::caching::{CacheEntry, CacheError}; -use crate::services::download::DownloadService; -use crate::utils::compression::maybe_decompress_file; /// Downloads the gives [`RemoteFile`] and decompresses it. /// diff --git a/crates/symbolicator-service/src/services/download/filesystem.rs b/crates/symbolicator-service/src/download/filesystem.rs similarity index 100% rename from crates/symbolicator-service/src/services/download/filesystem.rs rename to crates/symbolicator-service/src/download/filesystem.rs diff --git a/crates/symbolicator-service/src/services/download/gcs.rs b/crates/symbolicator-service/src/download/gcs.rs similarity index 100% rename from crates/symbolicator-service/src/services/download/gcs.rs rename to crates/symbolicator-service/src/download/gcs.rs diff --git a/crates/symbolicator-service/src/services/download/http.rs b/crates/symbolicator-service/src/download/http.rs similarity index 100% rename from crates/symbolicator-service/src/services/download/http.rs rename to crates/symbolicator-service/src/download/http.rs diff --git a/crates/symbolicator-service/src/services/download/mod.rs b/crates/symbolicator-service/src/download/mod.rs similarity index 99% rename from crates/symbolicator-service/src/services/download/mod.rs rename to crates/symbolicator-service/src/download/mod.rs index 4e05f4101..7d1059f80 100644 --- a/crates/symbolicator-service/src/services/download/mod.rs +++ b/crates/symbolicator-service/src/download/mod.rs @@ -31,12 +31,17 @@ use crate::utils::gcs::GcsError; use crate::utils::http::DownloadTimeouts; use crate::utils::sentry::ConfigureScope; +mod compression; +mod fetch_file; mod filesystem; mod gcs; mod http; mod s3; pub mod sentry; +pub use compression::tempfile_in_parent; +pub use fetch_file::fetch_file; + impl ConfigureScope for RemoteFile { fn to_scope(&self, scope: &mut ::sentry::Scope) { scope.set_tag("source.id", self.source_id()); diff --git a/crates/symbolicator-service/src/services/download/s3.rs b/crates/symbolicator-service/src/download/s3.rs similarity index 100% rename from crates/symbolicator-service/src/services/download/s3.rs rename to crates/symbolicator-service/src/download/s3.rs diff --git a/crates/symbolicator-service/src/services/download/sentry.rs b/crates/symbolicator-service/src/download/sentry.rs similarity index 100% rename from crates/symbolicator-service/src/services/download/sentry.rs rename to crates/symbolicator-service/src/download/sentry.rs diff --git a/crates/symbolicator-service/src/lib.rs b/crates/symbolicator-service/src/lib.rs index 4094fd5d4..0b58d4666 100644 --- a/crates/symbolicator-service/src/lib.rs +++ b/crates/symbolicator-service/src/lib.rs @@ -4,9 +4,13 @@ pub mod macros; #[macro_use] pub mod metrics; +pub mod caches; pub mod caching; pub mod config; +pub mod download; +pub mod objects; pub mod services; +pub mod source_context; pub mod types; pub mod utils; diff --git a/crates/symbolicator-service/src/types/objects.rs b/crates/symbolicator-service/src/objects/candidates.rs similarity index 93% rename from crates/symbolicator-service/src/types/objects.rs rename to crates/symbolicator-service/src/objects/candidates.rs index 6090f0881..1aa24cc3e 100644 --- a/crates/symbolicator-service/src/types/objects.rs +++ b/crates/symbolicator-service/src/objects/candidates.rs @@ -4,18 +4,40 @@ use serde::{Deserialize, Serialize}; use symbolicator_sources::{RemoteFileUri, SourceId}; -use super::ObjectFeatures; +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq, Serialize, Deserialize)] +pub struct ObjectFeatures { + /// The object file contains full debug info. + pub has_debug_info: bool, -/// Information about a Debug Information File in the [`CompleteObjectInfo`]. + /// The object file contains unwind info. + pub has_unwind_info: bool, + + /// The object file contains a symbol table. + pub has_symbols: bool, + + /// The object file had sources available. + #[serde(default)] + pub has_sources: bool, +} + +impl ObjectFeatures { + pub fn merge(&mut self, other: ObjectFeatures) { + self.has_debug_info |= other.has_debug_info; + self.has_unwind_info |= other.has_unwind_info; + self.has_symbols |= other.has_symbols; + self.has_sources |= other.has_sources; + } +} + +/// Information about a Debug Information File in the `CompleteObjectInfo`. /// -/// All DIFs are backed by an [`ObjectHandle`](crate::services::objects::ObjectHandle). But we +/// All DIFs are backed by an [`ObjectHandle`](crate::objects::ObjectHandle). But we /// may not have been able to get hold of this object file. We still want to describe the /// relevant DIF however. /// /// Currently has no [`ObjectId`] attached and the parent container is expected to know /// which ID this DIF info was for. /// -/// [`CompleteObjectInfo`]: crate::types::CompleteObjectInfo /// [`ObjectId`]: symbolicator_sources::ObjectId #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct ObjectCandidate { @@ -240,8 +262,6 @@ impl From> for AllObjectCandidates { #[cfg(test)] mod tests { - use crate::types::ObjectDownloadInfo; - use super::*; #[test] diff --git a/crates/symbolicator-service/src/services/objects/data_cache.rs b/crates/symbolicator-service/src/objects/data_cache.rs similarity index 97% rename from crates/symbolicator-service/src/services/objects/data_cache.rs rename to crates/symbolicator-service/src/objects/data_cache.rs index a8202f2b0..480d69df1 100644 --- a/crates/symbolicator-service/src/services/objects/data_cache.rs +++ b/crates/symbolicator-service/src/objects/data_cache.rs @@ -19,13 +19,11 @@ use symbolic::common::ByteView; use symbolic::debuginfo::{Archive, Object}; use symbolicator_sources::{ObjectId, RemoteFile}; +use crate::caches::versions::OBJECTS_CACHE_VERSIONS; use crate::caching::CacheVersions; use crate::caching::{CacheEntry, CacheError, CacheItemRequest, CacheKey}; -use crate::services::caches::versions::OBJECTS_CACHE_VERSIONS; -use crate::services::download::DownloadService; -use crate::services::fetch_file; +use crate::download::{fetch_file, tempfile_in_parent, DownloadService}; use crate::types::Scope; -use crate::utils::compression::tempfile_in_parent; use crate::utils::sentry::ConfigureScope; use super::meta_cache::FetchFileMetaRequest; @@ -249,9 +247,8 @@ mod tests { use super::*; use crate::caching::{Cache, CacheName}; use crate::config::{CacheConfig, CacheConfigs, Config}; - use crate::services::download::DownloadService; - use crate::services::objects::data_cache::Scope; - use crate::services::objects::{FindObject, ObjectPurpose, ObjectsActor}; + use crate::download::DownloadService; + use crate::objects::{FindObject, ObjectPurpose, ObjectsActor}; use crate::test::{self, tempdir}; use symbolic::common::DebugId; diff --git a/crates/symbolicator-service/src/services/objects/meta_cache.rs b/crates/symbolicator-service/src/objects/meta_cache.rs similarity index 96% rename from crates/symbolicator-service/src/services/objects/meta_cache.rs rename to crates/symbolicator-service/src/objects/meta_cache.rs index 75e83e86e..41575fd36 100644 --- a/crates/symbolicator-service/src/services/objects/meta_cache.rs +++ b/crates/symbolicator-service/src/objects/meta_cache.rs @@ -15,12 +15,14 @@ use symbolic::common::ByteView; use symbolicator_sources::{ObjectId, RemoteFile}; use tempfile::NamedTempFile; +use crate::caches::versions::META_CACHE_VERSIONS; use crate::caching::{ CacheEntry, CacheItemRequest, CacheKey, CacheKeyBuilder, CacheVersions, Cacher, }; -use crate::services::caches::versions::META_CACHE_VERSIONS; -use crate::types::{ObjectFeatures, Scope}; +use crate::download::DownloadService; +use crate::types::Scope; +use super::candidates::ObjectFeatures; use super::FetchFileDataRequest; /// This requests metadata of a single file at a specific path/url. @@ -36,7 +38,7 @@ pub(super) struct FetchFileMetaRequest { // `::compute`, e.g. make the Cacher hold arbitrary // state for computing. pub(super) data_cache: Arc>, - pub(super) download_svc: Arc, + pub(super) download_svc: Arc, } /// Handle to local metadata file of an object. diff --git a/crates/symbolicator-service/src/services/objects/mod.rs b/crates/symbolicator-service/src/objects/mod.rs similarity index 98% rename from crates/symbolicator-service/src/services/objects/mod.rs rename to crates/symbolicator-service/src/objects/mod.rs index fe9a2a211..01b1e4dda 100644 --- a/crates/symbolicator-service/src/services/objects/mod.rs +++ b/crates/symbolicator-service/src/objects/mod.rs @@ -7,18 +7,20 @@ use sentry::{Hub, SentryFutureExt}; use symbolicator_sources::{FileType, ObjectId, RemoteFile, RemoteFileUri, SourceConfig, SourceId}; use crate::caching::{Cache, CacheEntry, CacheError, CacheKey, Cacher, SharedCacheRef}; -use crate::services::download::DownloadService; -use crate::types::{AllObjectCandidates, ObjectCandidate, ObjectDownloadInfo, Scope}; +use crate::download::DownloadService; +use crate::types::Scope; + +mod candidates; +mod data_cache; +mod meta_cache; use data_cache::FetchFileDataRequest; use meta_cache::FetchFileMetaRequest; +pub use candidates::*; pub use data_cache::ObjectHandle; pub use meta_cache::ObjectMetaHandle; -mod data_cache; -mod meta_cache; - /// Wrapper around [`CacheError`] to also pass the file information along. /// /// Because of the requirement of [`CacheItemRequest`] to impl `From` it can not @@ -26,7 +28,7 @@ mod meta_cache; /// along some errors, so we use this wrapper. /// /// [`CacheItemRequest`]: crate::caching::CacheItemRequest -/// [`SourceLocation`]: crate::services::download::SourceLocation +/// [`SourceLocation`]: crate::download::SourceLocation #[derive(Clone, Debug)] pub struct CacheLookupError { /// The object file which was attempted to be fetched. diff --git a/crates/symbolicator-service/src/services/mod.rs b/crates/symbolicator-service/src/services.rs similarity index 82% rename from crates/symbolicator-service/src/services/mod.rs rename to crates/symbolicator-service/src/services.rs index 8928765b4..7efc8fa5e 100644 --- a/crates/symbolicator-service/src/services/mod.rs +++ b/crates/symbolicator-service/src/services.rs @@ -14,26 +14,9 @@ use anyhow::{Context, Result}; use crate::caching::{Caches, SharedCacheRef, SharedCacheService}; use crate::config::Config; -pub mod bitcode; -pub mod caches; -pub mod cficaches; -pub mod derived; -pub mod download; -mod fetch_file; -pub mod il2cpp; -mod minidump; -mod module_lookup; -pub mod objects; -pub mod ppdb_caches; -pub mod symbolication; -pub mod symcaches; - -use self::caches::SourceFilesCache; -use self::download::DownloadService; -use self::objects::ObjectsActor; - -pub use self::symbolication::ScrapingConfig; -pub use fetch_file::fetch_file; +use crate::caches::SourceFilesCache; +use crate::download::DownloadService; +use crate::objects::ObjectsActor; pub struct SharedServices { pub config: Config, diff --git a/crates/symbolicator-service/src/services/symbolication/mod.rs b/crates/symbolicator-service/src/services/symbolication/mod.rs deleted file mode 100644 index c6b1ba64e..000000000 --- a/crates/symbolicator-service/src/services/symbolication/mod.rs +++ /dev/null @@ -1,933 +0,0 @@ -use std::collections::{BTreeMap, HashMap}; -use std::sync::Arc; - -use serde::{Deserialize, Serialize}; - -use futures::future; -use symbolic::common::{split_path, DebugId, InstructionInfo, Language, Name}; -use symbolic::demangle::{Demangle, DemangleOptions}; -use symbolic::ppdb::PortablePdbCache; -use symbolic::symcache::{Function, SymCache}; -use symbolicator_sources::{HttpRemoteFile, ObjectType, SourceConfig}; - -use crate::caching::{Cache, CacheError}; -use crate::services::caches::SourceFilesCache; -use crate::services::cficaches::CfiCacheActor; -use crate::services::module_lookup::{CacheFileEntry, CacheLookupResult, ModuleLookup}; -use crate::services::objects::ObjectsActor; -use crate::services::ppdb_caches::PortablePdbCacheActor; -use crate::services::symcaches::SymCacheActor; -use crate::types::{ - CompleteObjectInfo, CompleteStacktrace, CompletedSymbolicationResponse, FrameStatus, - FrameTrust, ObjectFileStatus, RawFrame, RawStacktrace, Registers, Scope, Signal, - SymbolicatedFrame, -}; -use crate::utils::hex::HexValue; -use crate::utils::http::is_valid_origin; - -use super::bitcode::BitcodeService; -use super::il2cpp::Il2cppService; -use super::SharedServices; - -mod apple; -mod process_minidump; -pub mod source_context; - -/// Whether a frame's instruction address needs to be "adjusted" by subtracting a word. -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum AdjustInstructionAddr { - /// The frame's address definitely needs to be adjusted. - Yes, - /// The frame's address definitely does not need to be adjusted. - No, - /// The frame's address might need to be adjusted. - /// - /// This defers to the heuristic in [InstructionInfo::caller_address]. - Auto, -} - -impl AdjustInstructionAddr { - /// Returns the adjustment strategy for the given frame. - /// - /// If the frame has the - /// [`adjust_instruction_addr`](RawFrame::adjust_instruction_addr) - /// field set, this will be [`Yes`](Self::Yes) or [`No`](Self::No) accordingly, otherwise - /// the given default is used. - fn for_frame(frame: &RawFrame, default: Self) -> Self { - match frame.adjust_instruction_addr { - Some(true) => Self::Yes, - Some(false) => Self::No, - None => default, - } - } - - /// Returns the default adjustment strategy for the given thread. - /// - /// This will be [`Yes`](Self::Yes) if any frame in the thread has the - /// [`adjust_instruction_addr`](RawFrame::adjust_instruction_addr) - /// field set, otherwise it will be [`Auto`](Self::Auto). - fn default_for_thread(thread: &RawStacktrace) -> Self { - if thread - .frames - .iter() - .any(|frame| frame.adjust_instruction_addr.is_some()) - { - AdjustInstructionAddr::Yes - } else { - AdjustInstructionAddr::Auto - } - } -} - -// we should really rename this here to the `SymbolicatorService`, as it does a lot more -// than just symbolication ;-) -#[derive(Clone, Debug)] -pub struct SymbolicationActor { - demangle_cache: DemangleCache, - objects: ObjectsActor, - symcaches: SymCacheActor, - cficaches: CfiCacheActor, - ppdb_caches: PortablePdbCacheActor, - diagnostics_cache: Cache, - sourcefiles_cache: Arc, -} - -impl SymbolicationActor { - pub fn new(services: &SharedServices) -> Self { - let caches = &services.caches; - let shared_cache = services.shared_cache.clone(); - let objects = services.objects.clone(); - let download_svc = services.download_svc.clone(); - let sourcefiles_cache = services.sourcefiles_cache.clone(); - - let bitcode = BitcodeService::new( - caches.auxdifs.clone(), - shared_cache.clone(), - download_svc.clone(), - ); - - let il2cpp = Il2cppService::new(caches.il2cpp.clone(), shared_cache.clone(), download_svc); - - let symcaches = SymCacheActor::new( - caches.symcaches.clone(), - shared_cache.clone(), - objects.clone(), - bitcode, - il2cpp, - ); - - let cficaches = CfiCacheActor::new( - caches.cficaches.clone(), - shared_cache.clone(), - objects.clone(), - ); - - let ppdb_caches = - PortablePdbCacheActor::new(caches.ppdb_caches.clone(), shared_cache, objects.clone()); - - let demangle_cache = DemangleCache::builder() - .max_capacity(10 * 1024 * 1024) // 10 MiB, considering key and value: - .weigher(|k, v| (k.0.len() + v.len()).try_into().unwrap_or(u32::MAX)) - .build(); - - SymbolicationActor { - demangle_cache, - objects, - symcaches, - cficaches, - ppdb_caches, - diagnostics_cache: caches.diagnostics.clone(), - sourcefiles_cache, - } - } - - #[tracing::instrument(skip_all)] - pub async fn symbolicate( - &self, - request: SymbolicateStacktraces, - ) -> anyhow::Result { - let SymbolicateStacktraces { - stacktraces, - sources, - scope, - signal, - origin, - modules, - apply_source_context, - scraping, - } = request; - - let mut module_lookup = ModuleLookup::new(scope.clone(), sources, modules); - module_lookup - .fetch_caches( - self.symcaches.clone(), - self.ppdb_caches.clone(), - &stacktraces, - ) - .await; - - let mut metrics = StacktraceMetrics::default(); - let mut stacktraces: Vec<_> = stacktraces - .into_iter() - .map(|trace| { - symbolicate_stacktrace( - &self.demangle_cache, - trace, - &module_lookup, - &mut metrics, - signal, - ) - }) - .collect(); - - if apply_source_context { - self.apply_source_context(&mut module_lookup, &mut stacktraces, &scraping) - .await - } - - // bring modules back into the original order - let modules = module_lookup.into_inner(); - record_symbolication_metrics(origin, metrics, &modules, &stacktraces); - - Ok(CompletedSymbolicationResponse { - signal, - stacktraces, - modules, - ..Default::default() - }) - } - - async fn apply_source_context( - &self, - module_lookup: &mut ModuleLookup, - stacktraces: &mut [CompleteStacktrace], - scraping: &ScrapingConfig, - ) { - module_lookup - .fetch_sources(self.objects.clone(), stacktraces) - .await; - - // Map collected source contexts to frames and collect URLs for remote source links. - let mut remote_sources: HashMap<(Scope, url::Url), Vec<&mut RawFrame>> = HashMap::new(); - { - let debug_sessions = module_lookup.prepare_debug_sessions(); - - for trace in stacktraces { - for frame in &mut trace.frames { - let Some(url) = - module_lookup.try_set_source_context(&debug_sessions, &mut frame.raw) - else { - continue; - }; - if let Some(vec) = remote_sources.get_mut(&url) { - vec.push(&mut frame.raw) - } else { - remote_sources.insert(url, vec![&mut frame.raw]); - } - } - } - } - - // Download remote sources and update contexts. - if !remote_sources.is_empty() { - let cache = self.sourcefiles_cache.as_ref(); - let futures = - remote_sources - .into_iter() - .map(|((source_scope, url), frames)| async move { - let mut remote_file = HttpRemoteFile::from_url(url.clone()); - - if scraping.enabled && is_valid_origin(&url, &scraping.allowed_origins) { - remote_file.headers.extend( - scraping - .headers - .iter() - .map(|(key, value)| (key.clone(), value.clone())), - ); - } - - if let Ok(source) = cache - .fetch_file(&source_scope, remote_file.into(), true) - .await - { - for frame in frames { - ModuleLookup::set_source_context(&source, frame); - } - } - }); - future::join_all(futures).await; - } - } -} - -#[derive(Debug, Clone)] -/// A request for symbolication of multiple stack traces. -pub struct SymbolicateStacktraces { - /// The scope of this request which determines access to cached files. - pub scope: Scope, - - /// The signal thrown on certain operating systems. - /// - /// Signal handlers sometimes mess with the runtime stack. This is used to determine whether - /// the top frame should be fixed or not. - pub signal: Option, - - /// A list of external sources to load debug files. - pub sources: Arc<[SourceConfig]>, - - /// Where the stacktraces originated from. - pub origin: StacktraceOrigin, - - /// A list of threads containing stack traces. - pub stacktraces: Vec, - - /// A list of images that were loaded into the process. - /// - /// This list must cover the instruction addresses of the frames in - /// [`stacktraces`](Self::stacktraces). If a frame is not covered by any image, the frame cannot - /// be symbolicated as it is not clear which debug file to load. - pub modules: Vec, - - /// Whether to apply source context for the stack frames. - pub apply_source_context: bool, - - /// Scraping configuration controling authenticated requests. - pub scraping: ScrapingConfig, -} - -fn symbolicate_frame( - demangle_cache: &DemangleCache, - caches: &ModuleLookup, - registers: &Registers, - signal: Option, - frame: &mut RawFrame, - index: usize, - adjustment: AdjustInstructionAddr, -) -> Result, FrameStatus> { - let lookup_result = caches - .lookup_cache(frame.instruction_addr.0, frame.addr_mode) - .ok_or(FrameStatus::UnknownImage)?; - - frame.package = lookup_result.object_info.raw.code_file.clone(); - - match lookup_result.cache { - Ok(CacheFileEntry::SymCache(symcache)) => { - let symcache = symcache.get(); - let relative_addr = get_relative_caller_addr( - symcache, - &lookup_result, - registers, - signal, - index, - adjustment, - )?; - symbolicate_native_frame( - demangle_cache, - symcache, - lookup_result, - relative_addr, - frame, - index, - ) - } - Ok(CacheFileEntry::PortablePdbCache(ppdb_cache)) => { - symbolicate_dotnet_frame(ppdb_cache.get(), frame, index) - } - Err(CacheError::Malformed(_)) => Err(FrameStatus::Malformed), - _ => Err(FrameStatus::Missing), - } -} - -fn symbolicate_dotnet_frame( - ppdbcache: &PortablePdbCache, - frame: &RawFrame, - index: usize, -) -> Result, FrameStatus> { - // TODO: Add a new error variant for this? - let function_idx = frame.function_id.ok_or(FrameStatus::MissingSymbol)?.0 as u32; - let il_offset = frame.instruction_addr.0 as u32; - - let line_info = ppdbcache - .lookup(function_idx, il_offset) - .ok_or(FrameStatus::MissingSymbol)?; - - let abs_path = line_info.file_name; - let filename = split_path(abs_path).1; - let result = SymbolicatedFrame { - status: FrameStatus::Symbolicated, - original_index: Some(index), - raw: RawFrame { - lang: Some(line_info.file_lang), - filename: Some(filename.to_string()), - abs_path: Some(abs_path.to_string()), - lineno: Some(line_info.line), - ..frame.clone() - }, - }; - - Ok(vec![result]) -} - -fn get_relative_caller_addr( - symcache: &SymCache, - lookup_result: &CacheLookupResult, - registers: &Registers, - signal: Option, - index: usize, - adjustment: AdjustInstructionAddr, -) -> Result { - if let Some(addr) = lookup_result.relative_addr { - // heuristics currently are only supported when we can work with absolute addresses. - // In cases where this is not possible we skip this part entirely and use the relative - // address calculated by the lookup result as lookup address in the module. - if let Some(absolute_addr) = lookup_result.object_info.rel_to_abs_addr(addr) { - let is_crashing_frame = index == 0; - let ip_register_value = if is_crashing_frame { - symcache - .arch() - .cpu_family() - .ip_register_name() - .and_then(|ip_reg_name| registers.get(ip_reg_name)) - .map(|x| x.0) - } else { - None - }; - - let mut instruction_info = InstructionInfo::new(symcache.arch(), absolute_addr); - let instruction_info = instruction_info - .is_crashing_frame(is_crashing_frame) - .signal(signal.map(|signal| signal.0)) - .ip_register_value(ip_register_value); - - let absolute_caller_addr = match adjustment { - AdjustInstructionAddr::Yes => instruction_info.previous_address(), - AdjustInstructionAddr::No => instruction_info.aligned_address(), - AdjustInstructionAddr::Auto => instruction_info.caller_address(), - }; - - lookup_result - .object_info - .abs_to_rel_addr(absolute_caller_addr) - .ok_or_else(|| { - tracing::warn!( - "Underflow when trying to subtract image start addr from caller address after heuristics" - ); - metric!(counter("relative_addr.underflow") += 1); - FrameStatus::MissingSymbol - }) - } else { - Ok(addr) - } - } else { - tracing::warn!("Underflow when trying to subtract image start addr from caller address before heuristics"); - metric!(counter("relative_addr.underflow") += 1); - Err(FrameStatus::MissingSymbol) - } -} - -fn symbolicate_native_frame( - demangle_cache: &DemangleCache, - symcache: &SymCache, - lookup_result: CacheLookupResult, - relative_addr: u64, - frame: &RawFrame, - index: usize, -) -> Result, FrameStatus> { - tracing::trace!("Symbolicating {:#x}", relative_addr); - let mut rv = vec![]; - - // The symbol addr only makes sense for the outermost top-level function, and not its inlinees. - // We keep track of it while iterating and only set it for the last frame, - // which is the top-level function. - let mut sym_addr = None; - let instruction_addr = HexValue(lookup_result.expose_preferred_addr(relative_addr)); - - for source_location in symcache.lookup(relative_addr) { - let abs_path = source_location - .file() - .map(|f| f.full_path()) - .unwrap_or_default(); - let filename = split_path(&abs_path).1; - - let func = source_location.function(); - let (symbol, function) = demangle_symbol(demangle_cache, &func); - - sym_addr = Some(HexValue( - lookup_result.expose_preferred_addr(func.entry_pc() as u64), - )); - let filename = if !filename.is_empty() { - Some(filename.to_string()) - } else { - frame.filename.clone() - }; - rv.push(SymbolicatedFrame { - status: FrameStatus::Symbolicated, - original_index: Some(index), - raw: RawFrame { - package: lookup_result.object_info.raw.code_file.clone(), - addr_mode: lookup_result.preferred_addr_mode(), - instruction_addr, - adjust_instruction_addr: frame.adjust_instruction_addr, - function_id: frame.function_id, - symbol: Some(symbol), - abs_path: if !abs_path.is_empty() { - Some(abs_path) - } else { - frame.abs_path.clone() - }, - function: Some(function), - filename, - lineno: Some(source_location.line()), - pre_context: vec![], - context_line: None, - post_context: vec![], - source_link: None, - sym_addr: None, - lang: match func.language() { - Language::Unknown => None, - language => Some(language), - }, - in_app: None, - trust: frame.trust, - }, - }); - } - - if let Some(last_frame) = rv.last_mut() { - last_frame.raw.sym_addr = sym_addr; - } - - if rv.is_empty() { - return Err(FrameStatus::MissingSymbol); - } - - Ok(rv) -} - -/// Options for demangling all symbols. -const DEMANGLE_OPTIONS: DemangleOptions = DemangleOptions::complete().return_type(false); - -/// A cache for demangled symbols -type DemangleCache = moka::sync::Cache<(String, Language), String>; - -/// Demangles the name of the given [`Function`]. -fn demangle_symbol(cache: &DemangleCache, func: &Function) -> (String, String) { - let symbol = func.name(); - let key = (symbol.to_string(), func.language()); - - let init = || { - // Detect the language from the bare name, ignoring any pre-set language. There are a few - // languages that we should always be able to demangle. Only complain about those that we - // detect explicitly, but silently ignore the rest. For instance, there are C-identifiers - // reported as C++, which are expected not to demangle. - let detected_language = Name::from(symbol).detect_language(); - let should_demangle = match (func.language(), detected_language) { - (_, Language::Unknown) => false, // can't demangle what we cannot detect - (Language::ObjCpp, Language::Cpp) => true, // C++ demangles even if it was in ObjC++ - (Language::Unknown, _) => true, // if there was no language, then rely on detection - (lang, detected) => lang == detected, // avoid false-positive detections - }; - - let demangled_opt = func.name_for_demangling().demangle(DEMANGLE_OPTIONS); - if should_demangle && demangled_opt.is_none() { - sentry::with_scope( - |scope| scope.set_extra("identifier", symbol.to_string().into()), - || { - let message = format!("Failed to demangle {} identifier", func.language()); - sentry::capture_message(&message, sentry::Level::Error); - }, - ); - } - demangled_opt.unwrap_or_else(|| symbol.to_string()) - }; - - let entry = cache.entry_by_ref(&key).or_insert_with(init); - - (key.0, entry.into_value()) -} - -/// Configuration for scraping of JS Sources, Source Maps and Source Context from the web. -#[derive(Debug, Clone, Deserialize, Serialize)] -pub struct ScrapingConfig { - /// Whether scraping should happen at all. - pub enabled: bool, - // TODO: Can we even use this? - // pub verify_ssl: bool, - /// A list of "allowed origin patterns" that control: - /// - for sourcemaps: what URLs we are allowed to scrape from. - /// - for source context: which URLs should be authenticated using attached headers - /// - /// Allowed origins may be defined in several ways: - /// - `http://domain.com[:port]`: Exact match for base URI (must include port). - /// - `*`: Allow any domain. - /// - `*.domain.com`: Matches domain.com and all subdomains, on any port. - /// - `domain.com`: Matches domain.com on any port. - /// - `*:port`: Wildcard on hostname, but explicit match on port. - pub allowed_origins: Vec, - /// A map of headers to send with every HTTP request while scraping. - pub headers: BTreeMap, -} - -impl Default for ScrapingConfig { - fn default() -> Self { - Self { - enabled: true, - // verify_ssl: false, - allowed_origins: vec!["*".to_string()], - headers: Default::default(), - } - } -} - -/// Stacktrace related Metrics -/// -/// This gives some metrics about the quality of the stack traces included -/// in a symbolication request. See the individual members for more information. -/// -/// These numbers are being accumulated across one symbolication request, and are emitted -/// as a histogram. -#[derive(Default)] -struct StacktraceMetrics { - /// A truncated stack trace is one that does not end in a - /// well known thread base. - truncated_traces: u64, - - /// We classify a short stacktrace as one that has less that 5 frames. - short_traces: u64, - - /// This indicated a stack trace that has at least one bad frame - /// from the below categories. - bad_traces: u64, - - /// Frames that were scanned. - /// - /// These are frequently wrong and lead to bad and incomplete stack traces. - /// We can improve (lower) these numbers by having more usable CFI info. - scanned_frames: u64, - - /// Unsymbolicated Frames. - /// - /// These may be the result of unavailable or broken debug info. - /// We can improve (lower) these numbers by having more usable debug info. - unsymbolicated_frames: u64, - - /// Unsymbolicated Context Frames. - /// - /// This is an indication of broken contexts, or failure to extract it from minidumps. - unsymbolicated_context_frames: u64, - - /// Unsymbolicated Frames found by scanning. - unsymbolicated_scanned_frames: u64, - - /// Unsymbolicated Frames found by CFI. - /// - /// These are the result of the *previous* frame being wrongly scanned. - unsymbolicated_cfi_frames: u64, - - /// Frames referencing unmapped memory regions. - /// - /// These may be the result of issues in the client-side module finder, or - /// broken debug-id information. - /// - /// We can improve this by fixing client-side implementations and having - /// proper debug-ids. - unmapped_frames: u64, -} - -/// Determine if the [`SymbolicatedFrame`] is likely to be a thread base. -/// -/// This is just a heuristic that matches the function to well known thread entry points. -fn is_likely_base_frame(frame: &SymbolicatedFrame) -> bool { - let function = match frame - .raw - .function - .as_deref() - .or(frame.raw.symbol.as_deref()) - { - Some(f) => f, - None => return false, - }; - - // C start/main - if matches!(function, "main" | "start" | "_start") { - return true; - } - - // Windows and posix thread base. These often have prefixes depending on the OS and Version, so - // we use a substring match here. - if function.contains("UserThreadStart") - || function.contains("thread_start") - || function.contains("start_thread") - || function.contains("start_wqthread") - { - return true; - } - - false -} - -fn record_symbolication_metrics( - origin: StacktraceOrigin, - metrics: StacktraceMetrics, - modules: &[CompleteObjectInfo], - stacktraces: &[CompleteStacktrace], -) { - let origin = origin.to_string(); - - let platform = modules - .iter() - .find_map(|m| { - if m.raw.ty == ObjectType::Unknown { - None - } else { - Some(m.raw.ty) - } - }) - .unwrap_or(ObjectType::Unknown) - .to_string(); - - // Unusable modules that don’t have any kind of ID to look them up with - let mut unusable_modules = 0; - // Modules that failed parsing - let mut unparsable_modules = 0; - - for m in modules { - metric!( - counter("symbolication.debug_status") += 1, - "status" => m.debug_status.name() - ); - - let usable_code_id = !matches!(m.raw.code_id.as_deref(), None | Some("")); - - // NOTE: this is a closure as a way to short-circuit the computation because - // it is expensive - let usable_debug_id = || match m.raw.debug_id.as_deref() { - None | Some("") => false, - Some(string) => string.parse::().is_ok(), - }; - - if !usable_code_id && !usable_debug_id() { - unusable_modules += 1; - } - - if m.debug_status == ObjectFileStatus::Malformed { - unparsable_modules += 1; - } - } - - metric!( - time_raw("symbolication.num_modules") = modules.len() as u64, - "platform" => &platform, "origin" => &origin - ); - metric!( - time_raw("symbolication.unusable_modules") = unusable_modules, - "platform" => &platform, "origin" => &origin - ); - metric!( - time_raw("symbolication.unparsable_modules") = unparsable_modules, - "platform" => &platform, "origin" => &origin - ); - - metric!( - time_raw("symbolication.num_stacktraces") = stacktraces.len() as u64, - "platform" => &platform, "origin" => &origin - ); - metric!( - time_raw("symbolication.short_stacktraces") = metrics.short_traces, - "platform" => &platform, "origin" => &origin - ); - metric!( - time_raw("symbolication.truncated_stacktraces") = metrics.truncated_traces, - "platform" => &platform, "origin" => &origin - ); - metric!( - time_raw("symbolication.bad_stacktraces") = metrics.bad_traces, - "platform" => &platform, "origin" => &origin - ); - - metric!( - time_raw("symbolication.num_frames") = - stacktraces.iter().map(|s| s.frames.len() as u64).sum::(), - "platform" => &platform, "origin" => &origin - ); - metric!( - time_raw("symbolication.scanned_frames") = metrics.scanned_frames, - "platform" => &platform, "origin" => &origin - ); - metric!( - time_raw("symbolication.unsymbolicated_frames") = metrics.unsymbolicated_frames, - "platform" => &platform, "origin" => &origin - ); - metric!( - time_raw("symbolication.unsymbolicated_context_frames") = - metrics.unsymbolicated_context_frames, - "platform" => &platform, "origin" => &origin - ); - metric!( - time_raw("symbolication.unsymbolicated_cfi_frames") = - metrics.unsymbolicated_cfi_frames, - "platform" => &platform, "origin" => &origin - ); - metric!( - time_raw("symbolication.unsymbolicated_scanned_frames") = - metrics.unsymbolicated_scanned_frames, - "platform" => &platform, "origin" => &origin - ); - metric!( - time_raw("symbolication.unmapped_frames") = metrics.unmapped_frames, - "platform" => &platform, "origin" => &origin - ); -} - -fn symbolicate_stacktrace( - demangle_cache: &DemangleCache, - thread: RawStacktrace, - caches: &ModuleLookup, - metrics: &mut StacktraceMetrics, - signal: Option, -) -> CompleteStacktrace { - let default_adjustment = AdjustInstructionAddr::default_for_thread(&thread); - let mut symbolicated_frames = vec![]; - let mut unsymbolicated_frames_iter = thread.frames.into_iter().enumerate().peekable(); - - while let Some((index, mut frame)) = unsymbolicated_frames_iter.next() { - let adjustment = AdjustInstructionAddr::for_frame(&frame, default_adjustment); - match symbolicate_frame( - demangle_cache, - caches, - &thread.registers, - signal, - &mut frame, - index, - adjustment, - ) { - Ok(frames) => { - if matches!(frame.trust, FrameTrust::Scan) { - metrics.scanned_frames += 1; - } - symbolicated_frames.extend(frames) - } - Err(status) => { - // Since symbolication failed, the function name was not demangled. In case there is - // either one of `function` or `symbol`, treat that as mangled name and try to - // demangle it. If that succeeds, write the demangled name back. - let mangled = frame.function.as_deref().xor(frame.symbol.as_deref()); - let demangled = mangled.and_then(|m| Name::from(m).demangle(DEMANGLE_OPTIONS)); - if let Some(demangled) = demangled { - if let Some(old_mangled) = frame.function.replace(demangled) { - frame.symbol = Some(old_mangled); - } - } - - // Temporary workaround: Skip false-positive frames from stack scanning after the - // fact. - // - // Usually, the stack scanner would skip all scanned frames when it *knows* that - // they cannot be symbolized. However, in our case we supply breakpad symbols - // without function records. This breaks its original heuristic, since it would now - // *always* skip scan frames. Our patch in breakpad omits this check. - // - // Here, we fix this after the fact. - // - // - MissingSymbol: If symbolication failed for a scanned frame where we *know* we - // have a debug info, but the lookup inside that file failed. - // - UnknownImage: If symbolication failed because the stackscanner found an - // instruction_addr that is not in any image *we* consider valid. We discard - // images which do not have a debug id, while the stackscanner considers them - // perfectly fine. - if frame.trust == FrameTrust::Scan - && (status == FrameStatus::MissingSymbol || status == FrameStatus::UnknownImage) - { - continue; - } - - // Glibc inserts an explicit `DW_CFA_undefined: RIP` DWARF rule to say that `_start` - // has no return address. - // See https://sourceware.org/git/?p=glibc.git;a=blob;f=sysdeps/x86_64/start.S;h=1b3e36826b8a477474cee24d1c931429fbdf6d8f;hb=HEAD#l59 - // We do not support this due to lack of breakpad support, and will thus use the - // previous rule for RIP, which says to look it up the value on the stack, - // resulting in an unmapped garbage frame. We work around this by trimming the - // trailing garbage frame on the following conditions: - // * it is unmapped (UnknownImage) - // * this is the last frame to symbolicate (via peek) - // * the previous symbolicated frame is `_start` - let is_start = - |frame: &SymbolicatedFrame| frame.raw.function.as_deref() == Some("_start"); - if status == FrameStatus::UnknownImage - && unsymbolicated_frames_iter.peek().is_none() - && symbolicated_frames.last().map_or(false, is_start) - { - continue; - } - - metrics.unsymbolicated_frames += 1; - match frame.trust { - FrameTrust::Scan => { - metrics.scanned_frames += 1; - metrics.unsymbolicated_scanned_frames += 1; - } - FrameTrust::Cfi => metrics.unsymbolicated_cfi_frames += 1, - FrameTrust::Context => metrics.unsymbolicated_context_frames += 1, - _ => {} - } - if status == FrameStatus::UnknownImage { - metrics.unmapped_frames += 1; - } - - symbolicated_frames.push(SymbolicatedFrame { - status, - original_index: Some(index), - raw: frame, - }); - } - } - } - - // we try to find a base frame among the bottom 5 - if !symbolicated_frames - .iter() - .rev() - .take(5) - .any(is_likely_base_frame) - { - metrics.truncated_traces += 1; - } - // macOS has some extremely short but perfectly fine stacks, such as: - // `__workq_kernreturn` > `_pthread_wqthread` > `start_wqthread` - if symbolicated_frames.len() < 3 { - metrics.short_traces += 1; - } - - if metrics.scanned_frames > 0 || metrics.unsymbolicated_frames > 0 { - metrics.bad_traces += 1; - } - - CompleteStacktrace { - thread_id: thread.thread_id, - thread_name: thread.thread_name, - is_requesting: thread.is_requesting, - registers: thread.registers, - frames: symbolicated_frames, - } -} - -#[derive(Debug, Copy, Clone)] -/// Where the Stack Traces in the [`SymbolicateStacktraces`] originated from. -pub enum StacktraceOrigin { - /// The stack traces came from a direct request to symbolicate. - Symbolicate, - /// The stack traces were extracted from a minidump. - Minidump, - /// The stack traces came from an Apple Crash Report. - AppleCrashReport, -} - -impl std::fmt::Display for StacktraceOrigin { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(match self { - StacktraceOrigin::Symbolicate => "symbolicate", - StacktraceOrigin::Minidump => "minidump", - StacktraceOrigin::AppleCrashReport => "applecrashreport", - }) - } -} diff --git a/crates/symbolicator-service/src/services/symbolication/source_context.rs b/crates/symbolicator-service/src/source_context.rs similarity index 100% rename from crates/symbolicator-service/src/services/symbolication/source_context.rs rename to crates/symbolicator-service/src/source_context.rs diff --git a/crates/symbolicator-service/src/types.rs b/crates/symbolicator-service/src/types.rs new file mode 100644 index 000000000..db9d07a49 --- /dev/null +++ b/crates/symbolicator-service/src/types.rs @@ -0,0 +1,159 @@ +//! Types for the Symbolicator API. +//! +//! This module contains some types which (de)serialise to/from JSON to make up the public +//! HTTP API. Its messy and things probably need a better place and different way to signal +//! they are part of the public API. + +use std::collections::BTreeMap; +use std::fmt; +use std::sync::Arc; + +use serde::{Deserialize, Serialize}; +use symbolicator_sources::ObjectType; + +use crate::utils::hex::HexValue; + +/// The scope of a source or debug file. +/// +/// Based on scopes, access to debug files that have been cached is determined. If a file comes from +/// a public source, it can be used for any symbolication request. Otherwise, the symbolication +/// request must match the scope of a file. +#[derive(Debug, Clone, Deserialize, Serialize, Eq, Ord, PartialEq, PartialOrd, Hash)] +#[serde(untagged)] +#[derive(Default)] +pub enum Scope { + #[serde(rename = "global")] + #[default] + Global, + Scoped(Arc), +} + +impl AsRef for Scope { + fn as_ref(&self) -> &str { + match *self { + Scope::Global => "global", + Scope::Scoped(ref s) => s, + } + } +} + +impl fmt::Display for Scope { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match *self { + Scope::Global => f.write_str("global"), + Scope::Scoped(ref scope) => f.write_str(scope), + } + } +} + +/// Configuration for scraping of JS Sources, Source Maps and Source Context from the web. +#[derive(Debug, Clone, Deserialize, Serialize)] +pub struct ScrapingConfig { + /// Whether scraping should happen at all. + pub enabled: bool, + // TODO: Can we even use this? + // pub verify_ssl: bool, + /// A list of "allowed origin patterns" that control: + /// - for sourcemaps: what URLs we are allowed to scrape from. + /// - for source context: which URLs should be authenticated using attached headers + /// + /// Allowed origins may be defined in several ways: + /// - `http://domain.com[:port]`: Exact match for base URI (must include port). + /// - `*`: Allow any domain. + /// - `*.domain.com`: Matches domain.com and all subdomains, on any port. + /// - `domain.com`: Matches domain.com on any port. + /// - `*:port`: Wildcard on hostname, but explicit match on port. + pub allowed_origins: Vec, + /// A map of headers to send with every HTTP request while scraping. + pub headers: BTreeMap, +} + +impl Default for ScrapingConfig { + fn default() -> Self { + Self { + enabled: true, + // verify_ssl: false, + allowed_origins: vec!["*".to_string()], + headers: Default::default(), + } + } +} + +/// Specification of a module loaded into the process. +#[derive(Serialize, Deserialize, Debug, Clone, Eq, PartialEq)] +pub struct RawObjectInfo { + /// Platform image file type (container format). + #[serde(rename = "type")] + pub ty: ObjectType, + + /// Identifier of the code file. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code_id: Option, + + /// Name of the code file. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub code_file: Option, + + /// Identifier of the debug file. + #[serde(skip_serializing_if = "Option::is_none")] + pub debug_id: Option, + + /// Name of the debug file. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub debug_file: Option, + + /// Checksum of the file's contents. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub debug_checksum: Option, + + /// Absolute address at which the image was mounted into virtual memory. + /// + /// We do allow the `image_addr` to be skipped if it is zero. This is because systems like WASM + /// do not require modules to be mounted at a specific absolute address. Per definition, a + /// module mounted at `0` does not support absolute addressing. + #[serde(default)] + pub image_addr: HexValue, + + /// Size of the image in virtual memory. + /// + /// The size is infered from the module list if not specified. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub image_size: Option, +} + +/// Information on a debug information file. +#[derive(Debug, Clone, Copy, Serialize, Deserialize, Eq, PartialEq)] +#[serde(rename_all = "snake_case")] +#[derive(Default)] +pub enum ObjectFileStatus { + /// The file was found and successfully processed. + Found, + /// The image was not referenced in the stack trace and not further handled. + #[default] + Unused, + /// The file could not be found in any of the specified sources. + Missing, + /// The file failed to process. + Malformed, + /// The file could not be downloaded. + FetchingFailed, + /// Downloading or processing the file took too long. + Timeout, + /// An internal error while handling this image. + Other, +} + +impl ObjectFileStatus { + pub fn name(self) -> &'static str { + // used for metrics + match self { + ObjectFileStatus::Found => "found", + ObjectFileStatus::Unused => "unused", + ObjectFileStatus::Missing => "missing", + ObjectFileStatus::Malformed => "malformed", + ObjectFileStatus::FetchingFailed => "fetching_failed", + ObjectFileStatus::Timeout => "timeout", + ObjectFileStatus::Other => "other", + } + } +} diff --git a/crates/symbolicator-service/src/utils/addr.rs b/crates/symbolicator-service/src/utils/addr.rs deleted file mode 100644 index b12eef9c9..000000000 --- a/crates/symbolicator-service/src/utils/addr.rs +++ /dev/null @@ -1,69 +0,0 @@ -use std::borrow::Cow; -use std::fmt; -use std::str::FromStr; - -use serde::de::{self, Deserialize, Deserializer}; -use serde::ser::{Serialize, Serializer}; -use thiserror::Error; - -/// Defines the addressing mode. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd, Hash, Default)] -pub enum AddrMode { - /// Declares addresses to be absolute with a shared memory space. - #[default] - Abs, - /// Declares an address to be relative to an indexed module. - Rel(usize), -} - -impl fmt::Display for AddrMode { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - match *self { - AddrMode::Abs => write!(f, "abs"), - AddrMode::Rel(idx) => write!(f, "rel:{idx}"), - } - } -} - -impl Serialize for AddrMode { - fn serialize(&self, serializer: S) -> Result - where - S: Serializer, - { - serializer.serialize_str(&self.to_string()) - } -} - -#[derive(Debug, Error)] -#[error("invalid address mode")] -pub struct ParseAddrModeError; - -impl FromStr for AddrMode { - type Err = ParseAddrModeError; - - fn from_str(s: &str) -> Result { - if s == "abs" { - return Ok(AddrMode::Abs); - } - let mut iter = s.splitn(2, ':'); - let kind = iter.next().ok_or(ParseAddrModeError)?; - let index = iter - .next() - .and_then(|x| x.parse().ok()) - .ok_or(ParseAddrModeError)?; - match kind { - "rel" => Ok(AddrMode::Rel(index)), - _ => Err(ParseAddrModeError), - } - } -} - -impl<'de> Deserialize<'de> for AddrMode { - fn deserialize(deserializer: D) -> Result - where - D: Deserializer<'de>, - { - let s = Cow::::deserialize(deserializer).map_err(de::Error::custom)?; - AddrMode::from_str(&s).map_err(de::Error::custom) - } -} diff --git a/crates/symbolicator-service/src/utils/mod.rs b/crates/symbolicator-service/src/utils/mod.rs index b3e6e6cad..af968168c 100644 --- a/crates/symbolicator-service/src/utils/mod.rs +++ b/crates/symbolicator-service/src/utils/mod.rs @@ -1,5 +1,3 @@ -pub mod addr; -pub mod compression; pub mod futures; pub mod gcs; pub mod hex; diff --git a/crates/symbolicator-stress/Cargo.toml b/crates/symbolicator-stress/Cargo.toml index ece5fbdc8..0caa30560 100644 --- a/crates/symbolicator-stress/Cargo.toml +++ b/crates/symbolicator-stress/Cargo.toml @@ -15,6 +15,7 @@ serde = { version = "1.0.137", features = ["derive"] } serde_json = "1.0.81" serde_yaml = "0.9.14" symbolicator-js = { path = "../symbolicator-js" } +symbolicator-native = { path = "../symbolicator-native" } symbolicator-service = { path = "../symbolicator-service" } symbolicator-test = { path = "../symbolicator-test" } tempfile = "3.2.0" diff --git a/crates/symbolicator-stress/src/stresstest.rs b/crates/symbolicator-stress/src/stresstest.rs index 0dcc9a99a..ea4e35083 100644 --- a/crates/symbolicator-stress/src/stresstest.rs +++ b/crates/symbolicator-stress/src/stresstest.rs @@ -6,8 +6,8 @@ use anyhow::{Context, Result}; use sentry::SentryFutureExt; use symbolicator_js::SourceMapService; +use symbolicator_native::SymbolicationActor; use symbolicator_service::config::Config as SymbolicatorConfig; -use symbolicator_service::services::symbolication::SymbolicationActor; use symbolicator_service::services::SharedServices; use symbolicator_service::types::Scope; use tokio::sync::Semaphore; diff --git a/crates/symbolicator-stress/src/workloads.rs b/crates/symbolicator-stress/src/workloads.rs index a3ec944d6..8b1ce9062 100644 --- a/crates/symbolicator-stress/src/workloads.rs +++ b/crates/symbolicator-stress/src/workloads.rs @@ -7,11 +7,10 @@ use serde::{Deserialize, Serialize}; use symbolicator_js::interface::{JsStacktrace, SymbolicateJsStacktraces}; use symbolicator_js::SourceMapService; -use symbolicator_service::services::download::SourceConfig; -use symbolicator_service::services::symbolication::{ - StacktraceOrigin, SymbolicateStacktraces, SymbolicationActor, -}; -use symbolicator_service::types::{RawObjectInfo, RawStacktrace, Scope}; +use symbolicator_native::interface::{RawStacktrace, StacktraceOrigin, SymbolicateStacktraces}; +use symbolicator_native::SymbolicationActor; +use symbolicator_service::download::SourceConfig; +use symbolicator_service::types::{RawObjectInfo, Scope}; #[derive(Debug, Deserialize, Serialize)] pub struct WorkloadsConfig { diff --git a/crates/symbolicator/Cargo.toml b/crates/symbolicator/Cargo.toml index e3bfbbc05..c22ca692d 100644 --- a/crates/symbolicator/Cargo.toml +++ b/crates/symbolicator/Cargo.toml @@ -23,6 +23,7 @@ serde_json = "1.0.81" symbolic = "12.4.0" symbolicator-crash = { path = "../symbolicator-crash", optional = true } symbolicator-js = { path = "../symbolicator-js" } +symbolicator-native = { path = "../symbolicator-native" } symbolicator-service = { path = "../symbolicator-service" } symbolicator-sources = { path = "../symbolicator-sources" } tempfile = "3.2.0" diff --git a/crates/symbolicator/src/endpoints/symbolicate.rs b/crates/symbolicator/src/endpoints/symbolicate.rs index 62ed5484c..4d1380898 100644 --- a/crates/symbolicator/src/endpoints/symbolicate.rs +++ b/crates/symbolicator/src/endpoints/symbolicate.rs @@ -2,12 +2,14 @@ use axum::extract; use axum::response::Json; use serde::{Deserialize, Serialize}; -use symbolicator_service::services::ScrapingConfig; +use symbolicator_native::interface::{ + RawStacktrace, Signal, StacktraceOrigin, SymbolicateStacktraces, +}; +use symbolicator_service::types::RawObjectInfo; use symbolicator_sources::SourceConfig; use crate::service::{ - RawObjectInfo, RawStacktrace, RequestOptions, RequestService, Scope, Signal, StacktraceOrigin, - SymbolicateStacktraces, SymbolicationResponse, + RequestOptions, RequestService, Scope, ScrapingConfig, SymbolicationResponse, }; use crate::utils::sentry::ConfigureScope; @@ -89,7 +91,7 @@ mod tests { use super::*; use reqwest::{Client, StatusCode}; - use symbolicator_service::types::CompletedSymbolicationResponse; + use symbolicator_native::interface::CompletedSymbolicationResponse; use crate::test; diff --git a/crates/symbolicator/src/endpoints/symbolicate_js.rs b/crates/symbolicator/src/endpoints/symbolicate_js.rs index 69a446e1c..92b7c9c7a 100644 --- a/crates/symbolicator/src/endpoints/symbolicate_js.rs +++ b/crates/symbolicator/src/endpoints/symbolicate_js.rs @@ -4,13 +4,12 @@ use axum::extract; use axum::response::Json; use serde::{Deserialize, Serialize}; use symbolicator_js::interface::{JsStacktrace, SymbolicateJsStacktraces}; -use symbolicator_service::services::ScrapingConfig; use symbolicator_service::types::RawObjectInfo; use symbolicator_sources::SentrySourceConfig; use url::Url; use crate::endpoints::symbolicate::SymbolicationRequestQueryParams; -use crate::service::{RequestService, SymbolicationResponse}; +use crate::service::{RequestService, ScrapingConfig, SymbolicationResponse}; use crate::utils::sentry::ConfigureScope; use super::ResponseError; diff --git a/crates/symbolicator/src/service.rs b/crates/symbolicator/src/service.rs index cfe7b81b9..a0852dc0a 100644 --- a/crates/symbolicator/src/service.rs +++ b/crates/symbolicator/src/service.rs @@ -26,25 +26,23 @@ use sentry::SentryFutureExt; use serde::{Deserialize, Deserializer, Serialize}; use symbolicator_js::interface::{CompletedJsSymbolicationResponse, SymbolicateJsStacktraces}; use symbolicator_js::SourceMapService; -use symbolicator_service::services::{ScrapingConfig, SharedServices}; -use tempfile::TempPath; -use uuid::Uuid; - +use symbolicator_native::interface::{CompletedSymbolicationResponse, SymbolicateStacktraces}; +use symbolicator_native::SymbolicationActor; use symbolicator_service::caching::CacheEntry; use symbolicator_service::config::Config; use symbolicator_service::metric; -use symbolicator_service::services::objects::ObjectsActor; -use symbolicator_service::services::symbolication::SymbolicationActor; -use symbolicator_service::types::CompletedSymbolicationResponse; +use symbolicator_service::objects::ObjectsActor; +use symbolicator_service::services::SharedServices; use symbolicator_service::utils::futures::CallOnDrop; use symbolicator_service::utils::futures::{m, measure}; use symbolicator_sources::SourceConfig; +use tempfile::TempPath; +use uuid::Uuid; -pub use symbolicator_service::services::objects::{ +pub use symbolicator_service::objects::{ FindObject, FindResult, ObjectHandle, ObjectMetaHandle, ObjectPurpose, }; -pub use symbolicator_service::services::symbolication::{StacktraceOrigin, SymbolicateStacktraces}; -pub use symbolicator_service::types::{RawObjectInfo, RawStacktrace, Scope, Signal}; +pub use symbolicator_service::types::{Scope, ScrapingConfig}; /// Symbolication task identifier. #[derive(Debug, Clone, Copy, Serialize, Ord, PartialOrd, Eq, PartialEq)] @@ -120,7 +118,7 @@ pub struct RequestOptions { /// influences the quality of symbolication. Enabling this will return extra /// information in the modules list section of the response detailing all DIF objects /// considered, any problems with them and what they were used for. See the - /// [`ObjectCandidate`](symbolicator_service::types::ObjectCandidate) struct + /// [`ObjectCandidate`](symbolicator_service::objects::ObjectCandidate) struct /// for which extra information is returned for DIF objects. #[serde(default)] pub dif_candidates: bool, @@ -540,7 +538,10 @@ pub fn record_task_metrics(name: &str, metrics: &tokio_metrics::TaskMetrics) { #[cfg(test)] mod tests { - use symbolicator_service::types::{CompleteObjectInfo, RawFrame}; + use symbolicator_native::interface::{ + CompleteObjectInfo, RawFrame, RawStacktrace, StacktraceOrigin, SymbolicateStacktraces, + }; + use symbolicator_service::types::RawObjectInfo; use symbolicator_service::utils::hex::HexValue; use symbolicator_sources::ObjectType; diff --git a/crates/symbolicli/Cargo.toml b/crates/symbolicli/Cargo.toml index 3f7fa7b8a..2e3988c97 100644 --- a/crates/symbolicli/Cargo.toml +++ b/crates/symbolicli/Cargo.toml @@ -16,6 +16,7 @@ serde_json = "1.0.81" serde_yaml = "0.9.14" symbolic = "12.4.0" symbolicator-js = { path = "../symbolicator-js" } +symbolicator-native = { path = "../symbolicator-native" } symbolicator-service = { path = "../symbolicator-service" } symbolicator-sources = { path = "../symbolicator-sources" } tempfile = "3.3.0" diff --git a/crates/symbolicli/src/main.rs b/crates/symbolicli/src/main.rs index c1a53a7bf..e0609cc7f 100644 --- a/crates/symbolicli/src/main.rs +++ b/crates/symbolicli/src/main.rs @@ -9,7 +9,7 @@ use remote::EventKey; use settings::Mode; use symbolicator_js::SourceMapService; -use symbolicator_service::services::symbolication::SymbolicationActor; +use symbolicator_native::SymbolicationActor; use symbolicator_service::services::SharedServices; use symbolicator_service::types::Scope; use symbolicator_sources::{ @@ -410,12 +410,12 @@ mod event { use symbolicator_js::interface::{ JsFrame, JsFrameData, JsStacktrace, SymbolicateJsStacktraces, }; - use symbolicator_service::services::symbolication::{StacktraceOrigin, SymbolicateStacktraces}; - use symbolicator_service::services::ScrapingConfig; - use symbolicator_service::types::{ - CompleteObjectInfo, FrameTrust, RawFrame, RawObjectInfo, RawStacktrace, Scope, Signal, + use symbolicator_native::interface::{ + AddrMode, CompleteObjectInfo, FrameTrust, RawFrame, RawStacktrace, Signal, + StacktraceOrigin, SymbolicateStacktraces, }; - use symbolicator_service::utils::{addr::AddrMode, hex::HexValue}; + use symbolicator_service::types::{RawObjectInfo, Scope, ScrapingConfig}; + use symbolicator_service::utils::hex::HexValue; use symbolicator_sources::{SentrySourceConfig, SourceConfig}; pub fn create_js_symbolication_request( diff --git a/crates/symbolicli/src/output.rs b/crates/symbolicli/src/output.rs index 8c98ffd93..47c871641 100644 --- a/crates/symbolicli/src/output.rs +++ b/crates/symbolicli/src/output.rs @@ -3,7 +3,7 @@ use std::{collections::HashMap, iter::Peekable, vec::IntoIter}; use prettytable::{cell, format::consts::FORMAT_CLEAN, row, Row, Table}; use symbolic::common::split_path; use symbolicator_js::interface::{CompletedJsSymbolicationResponse, JsFrame}; -use symbolicator_service::types::{ +use symbolicator_native::interface::{ CompleteObjectInfo, CompletedSymbolicationResponse, FrameTrust, SymbolicatedFrame, };