Add span snapshots to the v1 payload for caching purposes (#612)

## Goal

Add snapshots to v1 payload so we can revive any unterminated spans when we send a crashed session payload. This will ensure that any data stored in a session span will not be lost, and any spans terminated by a crash that isn't terminated during the crash can be terminated after the fact.

## Testing

Updated payload tests and the tracing integration test

embrace-android-sdk/src/integrationTest/kotlin/io/embrace/android/embracesdk/assertions/SpanAssertions.kt

@@ -7,11 +7,16 @@ import io.embrace.android.embracesdk.arch.assertNotKeySpan
 import io.embrace.android.embracesdk.arch.assertNotPrivateSpan
 import io.embrace.android.embracesdk.arch.assertSuccessful
 import io.embrace.android.embracesdk.internal.clock.nanosToMillis
+import io.embrace.android.embracesdk.internal.payload.Span
+import io.embrace.android.embracesdk.internal.payload.SpanEvent
+import io.embrace.android.embracesdk.internal.payload.toNewPayload
 import io.embrace.android.embracesdk.internal.spans.EmbraceSpanData
 import io.embrace.android.embracesdk.spans.EmbraceSpanEvent
 import io.embrace.android.embracesdk.spans.ErrorCode
 import io.opentelemetry.api.trace.StatusCode
+import org.junit.Assert.assertArrayEquals
 import org.junit.Assert.assertEquals
+import org.junit.Assert.assertTrue
 
 /**
  * Assert the [EmbraceSpanData] is as expected
@@ -60,4 +65,52 @@ internal fun assertEmbraceSpanData(
             assertNotKeySpan()
         }
     }
+}
+
+internal fun Span.assertSpanPayload(
+    expectedStartTimeMs: Long,
+    expectedEndTimeMs: Long?,
+    expectedParentId: String?,
+    expectedTraceId: String? = null,
+    errorCode: ErrorCode? = null,
+    expectedCustomAttributes: Map<String, String> = emptyMap(),
+    expectedEvents: List<SpanEvent> = emptyList(),
+    private: Boolean = false,
+    key: Boolean = false,
+) {
+    assertEquals(expectedStartTimeMs, startTimeUnixNano?.nanosToMillis())
+    assertEquals(expectedEndTimeMs, endTimeUnixNano?.nanosToMillis())
+    assertEquals(expectedParentId, parentSpanId)
+    if (expectedTraceId != null) {
+        assertEquals(expectedTraceId, traceId)
+    } else {
+        assertEquals(32, traceId?.length)
+    }
+
+    if (endTimeUnixNano == null) {
+        assertEquals(Span.Status.UNSET, status)
+    } else if (errorCode == null) {
+        assertSuccessful()
+    } else {
+        assertError(errorCode)
+    }
+
+    val attributeSet = attributes?.toHashSet() ?: emptySet()
+    expectedCustomAttributes.toNewPayload().forEach {
+        assertTrue("$it is missing", attributeSet.contains(it))
+    }
+
+    assertArrayEquals(expectedEvents.toList().toTypedArray(), checkNotNull(events).toTypedArray())
+
+    if (private) {
+        assertIsPrivateSpan()
+    } else {
+        assertNotPrivateSpan()
+    }
+
+    if (key) {
+        assertIsKeySpan()
+    } else {
+        assertNotKeySpan()
+    }
 }

embrace-android-sdk/src/integrationTest/kotlin/io/embrace/android/embracesdk/testcases/TracingApiTest.kt

@@ -3,18 +3,22 @@ package io.embrace.android.embracesdk.testcases
 import android.os.Build.VERSION_CODES.TIRAMISU
 import androidx.test.ext.junit.runners.AndroidJUnit4
 import io.embrace.android.embracesdk.IntegrationTestRule
+import io.embrace.android.embracesdk.arch.assertIsTypePerformance
 import io.embrace.android.embracesdk.assertions.assertEmbraceSpanData
+import io.embrace.android.embracesdk.assertions.assertSpanPayload
 import io.embrace.android.embracesdk.fakes.FakeSpanExporter
 import io.embrace.android.embracesdk.fixtures.TOO_LONG_ATTRIBUTE_KEY
 import io.embrace.android.embracesdk.fixtures.TOO_LONG_ATTRIBUTE_VALUE
 import io.embrace.android.embracesdk.getSentBackgroundActivities
 import io.embrace.android.embracesdk.internal.clock.millisToNanos
+import io.embrace.android.embracesdk.internal.payload.toNewPayload
 import io.embrace.android.embracesdk.internal.spans.EmbraceSpanData
 import io.embrace.android.embracesdk.recordSession
 import io.embrace.android.embracesdk.spans.EmbraceSpanEvent
 import io.embrace.android.embracesdk.spans.ErrorCode
 import io.opentelemetry.api.trace.SpanId
 import io.opentelemetry.api.trace.StatusCode
+import org.junit.Assert.assertEquals
 import org.junit.Assert.assertSame
 import org.junit.Assert.assertTrue
 import org.junit.Before
@@ -111,14 +115,19 @@ internal class TracingApiTest {
                 assertTrue(bonusSpan.stop(endTimeMs = harness.fakeClock.now() + 1))
                 harness.fakeClock.tick(300L)
                 assertTrue(bonusSpan2.stop())
+                val unendingSpan = checkNotNull(embrace.startSpan("unending-span"))
+                harness.fakeClock.tick(100L)
+                unendingSpan.addAttribute("unending-key", "unending-value")
+                unendingSpan.addEvent("unending-event")
                 results.add("\nSpans exported before ending startup: ${spanExporter.exportedSpans.toList().map { it.name }}")
                 embrace.endAppStartup()
             }
             results.add("\nSpans exported after session ends: ${spanExporter.exportedSpans.toList().map { it.name }}")
             val sessionEndTime = harness.fakeClock.now()
+            val session = checkNotNull(sessionMessage)
             val allSpans = getSdkInitSpanFromBackgroundActivity() +
-                checkNotNull(sessionMessage?.spans) +
-                checkNotNull(harness.openTelemetryModule.spanSink.completedSpans())
+                checkNotNull(session.spans) +
+                harness.openTelemetryModule.spanSink.completedSpans()
 
             val spansMap = allSpans.associateBy { it.name }
             val sessionSpan = checkNotNull(spansMap["emb-session"])
@@ -231,6 +240,25 @@ internal class TracingApiTest {
                 expectedParentId = SpanId.getInvalid(),
                 private = true
             )
+
+            assertEquals(1, checkNotNull(sessionMessage.spanSnapshots).size)
+            val snapshot = sessionMessage.spanSnapshots.first()
+
+            snapshot.assertSpanPayload(
+                expectedStartTimeMs = testStartTimeMs + 700,
+                expectedEndTimeMs = null,
+                expectedParentId = null,
+                expectedCustomAttributes = mapOf(Pair("unending-key", "unending-value")),
+                expectedEvents = listOfNotNull(
+                    EmbraceSpanEvent.create(
+                        name = "unending-event",
+                        timestampMs = testStartTimeMs + 800,
+                        attributes = null
+                    )?.toNewPayload()
+                ),
+                key = true
+            )
+            snapshot.assertIsTypePerformance()
         }
     }
 

embrace-android-sdk/src/main/java/io/embrace/android/embracesdk/injection/SessionModule.kt

@@ -60,6 +60,7 @@ internal class SessionModuleImpl(
             dataCaptureServiceModule.breadcrumbService,
             essentialServiceModule.userService,
             androidServicesModule.preferencesService,
+            openTelemetryModule.spanRepository,
             openTelemetryModule.spanSink,
             openTelemetryModule.currentSessionSpan,
             sessionPropertiesService,

embrace-android-sdk/src/main/java/io/embrace/android/embracesdk/payload/SessionMessage.kt

@@ -6,6 +6,7 @@ import io.embrace.android.embracesdk.comms.api.ApiClient
 import io.embrace.android.embracesdk.internal.payload.EnvelopeMetadata
 import io.embrace.android.embracesdk.internal.payload.EnvelopeResource
 import io.embrace.android.embracesdk.internal.payload.SessionPayload
+import io.embrace.android.embracesdk.internal.payload.Span
 import io.embrace.android.embracesdk.internal.spans.EmbraceSpanData
 
 /**
@@ -54,6 +55,9 @@ internal data class SessionMessage @JvmOverloads internal constructor(
     @Json(name = "spans")
     val spans: List<EmbraceSpanData>? = null,
 
+    @Json(name = "span_snapshots")
+    val spanSnapshots: List<Span>? = null,
+
     @Json(name = "v")
     val version: Int? = ApiClient.MESSAGE_VERSION,
 

embrace-android-sdk/src/main/java/io/embrace/android/embracesdk/session/message/V1PayloadMessageCollator.kt

@@ -15,6 +15,7 @@ import io.embrace.android.embracesdk.event.LogMessageService
 import io.embrace.android.embracesdk.gating.GatingService
 import io.embrace.android.embracesdk.internal.spans.CurrentSessionSpan
 import io.embrace.android.embracesdk.internal.spans.EmbraceSpanData
+import io.embrace.android.embracesdk.internal.spans.SpanRepository
 import io.embrace.android.embracesdk.internal.spans.SpanSink
 import io.embrace.android.embracesdk.internal.utils.Uuid
 import io.embrace.android.embracesdk.logging.InternalErrorService
@@ -39,10 +40,11 @@ internal class V1PayloadMessageCollator(
     private val breadcrumbService: BreadcrumbService,
     private val userService: UserService,
     private val preferencesService: PreferencesService,
+    private val spanRepository: SpanRepository,
     private val spanSink: SpanSink,
     private val currentSessionSpan: CurrentSessionSpan,
     private val sessionPropertiesService: SessionPropertiesService,
-    private val startupService: StartupService
+    private val startupService: StartupService,
 ) : PayloadMessageCollator {
 
     /**
@@ -177,6 +179,9 @@ internal class V1PayloadMessageCollator(
                 else -> breadcrumbService.getBreadcrumbs()
             }
         }
+        val spanSnapshots = captureDataSafely {
+            spanRepository.getActiveSpans().mapNotNull { it.snapshot() }
+        }
 
         return SessionMessage(
             session = finalPayload,
@@ -192,7 +197,8 @@ internal class V1PayloadMessageCollator(
                 )
             },
             breadcrumbs = breadcrumbs,
-            spans = spans
+            spans = spans,
+            spanSnapshots = spanSnapshots,
         )
     }
 }

embrace-android-sdk/src/test/java/io/embrace/android/embracesdk/payload/SessionMessageTest.kt

@@ -4,8 +4,10 @@ import com.squareup.moshi.JsonDataException
 import io.embrace.android.embracesdk.assertJsonMatchesGoldenFile
 import io.embrace.android.embracesdk.deserializeEmptyJsonString
 import io.embrace.android.embracesdk.deserializeJsonFromResource
+import io.embrace.android.embracesdk.fakes.FakeClock.Companion.DEFAULT_FAKE_CURRENT_TIME
 import io.embrace.android.embracesdk.fakes.fakeSession
 import io.embrace.android.embracesdk.internal.payload.SessionPayload
+import io.embrace.android.embracesdk.internal.payload.Span
 import io.embrace.android.embracesdk.internal.spans.EmbraceSpanData
 import io.opentelemetry.api.trace.StatusCode
 import org.junit.Assert.assertEquals
@@ -36,6 +38,19 @@ internal class SessionMessageTest {
             emptyList()
         )
     )
+    private val spanSnapshots = listOfNotNull(
+        Span(
+            traceId = "snapshot-trace-id",
+            spanId = "snapshot-span-id",
+            parentSpanId = null,
+            name = "snapshot",
+            startTimeUnixNano = DEFAULT_FAKE_CURRENT_TIME,
+            endTimeUnixNano = null,
+            status = Span.Status.UNSET,
+            events = emptyList(),
+            attributes = emptyList()
+        )
+    )
 
     private val info = SessionMessage(
         session,
@@ -44,7 +59,8 @@ internal class SessionMessageTest {
         deviceInfo,
         performanceInfo,
         breadcrumbs,
-        spans
+        spans,
+        spanSnapshots
     )
 
     @Test
@@ -63,6 +79,7 @@ internal class SessionMessageTest {
         assertEquals(performanceInfo, obj.performanceInfo)
         assertEquals(breadcrumbs, obj.breadcrumbs)
         assertEquals(spans, obj.spans)
+        assertEquals(spanSnapshots, obj.spanSnapshots)
     }
 
     @Test(expected = JsonDataException::class)

embrace-android-sdk/src/test/java/io/embrace/android/embracesdk/session/PayloadFactoryBaTest.kt

@@ -34,6 +34,7 @@ import io.embrace.android.embracesdk.fakes.FakeWebViewService
 import io.embrace.android.embracesdk.fakes.injection.FakeInitModule
 import io.embrace.android.embracesdk.internal.serialization.EmbraceSerializer
 import io.embrace.android.embracesdk.internal.spans.CurrentSessionSpan
+import io.embrace.android.embracesdk.internal.spans.SpanRepository
 import io.embrace.android.embracesdk.internal.spans.SpanService
 import io.embrace.android.embracesdk.internal.spans.SpanSink
 import io.embrace.android.embracesdk.logging.InternalErrorService
@@ -64,6 +65,7 @@ internal class PayloadFactoryBaTest {
     private lateinit var ndkService: FakeNdkService
     private lateinit var configService: FakeConfigService
     private lateinit var localConfig: LocalConfig
+    private lateinit var spanRepository: SpanRepository
     private lateinit var spanSink: SpanSink
     private lateinit var currentSessionSpan: CurrentSessionSpan
     private lateinit var spanService: SpanService
@@ -86,6 +88,7 @@ internal class PayloadFactoryBaTest {
         preferencesService = FakePreferenceService(backgroundActivityEnabled = true)
         userService = FakeUserService()
         val initModule = FakeInitModule(clock = clock)
+        spanRepository = initModule.openTelemetryModule.spanRepository
         spanSink = initModule.openTelemetryModule.spanSink
         currentSessionSpan = initModule.openTelemetryModule.currentSessionSpan
         spanService = initModule.openTelemetryModule.spanService
@@ -174,6 +177,7 @@ internal class PayloadFactoryBaTest {
             breadcrumbService,
             userService,
             preferencesService,
+            spanRepository,
             spanSink,
             currentSessionSpan,
             FakeSessionPropertiesService(),

embrace-android-sdk/src/test/java/io/embrace/android/embracesdk/session/SessionHandlerTest.kt

@@ -154,6 +154,7 @@ internal class SessionHandlerTest {
             breadcrumbService,
             userService,
             preferencesService,
+            initModule.openTelemetryModule.spanRepository,
             initModule.openTelemetryModule.spanSink,
             initModule.openTelemetryModule.currentSessionSpan,
             FakeSessionPropertiesService(),

embrace-android-sdk/src/test/java/io/embrace/android/embracesdk/session/V1PayloadMessageCollatorTest.kt

@@ -59,10 +59,11 @@ internal class V1PayloadMessageCollatorTest {
             breadcrumbService = FakeBreadcrumbService(),
             metadataService = FakeMetadataService(),
             performanceInfoService = FakePerformanceInfoService(),
+            spanRepository = initModule.openTelemetryModule.spanRepository,
             spanSink = initModule.openTelemetryModule.spanSink,
             currentSessionSpan = initModule.openTelemetryModule.currentSessionSpan,
             sessionPropertiesService = FakeSessionPropertiesService(),
-            startupService = FakeStartupService()
+            startupService = FakeStartupService(),
         )
     }
 

embrace-android-sdk/src/test/java/io/embrace/android/embracesdk/session/message/PayloadFactoryImplTest.kt

@@ -61,6 +61,7 @@ internal class PayloadFactoryImplTest {
             breadcrumbService = FakeBreadcrumbService(),
             metadataService = FakeMetadataService(),
             performanceInfoService = FakePerformanceInfoService(),
+            spanRepository = initModule.openTelemetryModule.spanRepository,
             spanSink = initModule.openTelemetryModule.spanSink,
             currentSessionSpan = initModule.openTelemetryModule.currentSessionSpan,
             sessionPropertiesService = FakeSessionPropertiesService(),

embrace-android-sdk/src/test/java/io/embrace/android/embracesdk/session/message/V2PayloadMessageCollatorTest.kt

@@ -60,10 +60,11 @@ internal class V2PayloadMessageCollatorTest {
             breadcrumbService = FakeBreadcrumbService(),
             metadataService = FakeMetadataService(),
             performanceInfoService = FakePerformanceInfoService(),
+            spanRepository = initModule.openTelemetryModule.spanRepository,
             spanSink = initModule.openTelemetryModule.spanSink,
             currentSessionSpan = initModule.openTelemetryModule.currentSessionSpan,
             sessionPropertiesService = FakeSessionPropertiesService(),
-            startupService = FakeStartupService()
+            startupService = FakeStartupService(),
         )
         val sessionEnvelopeSource = SessionEnvelopeSourceImpl(
             metadataSource = FakeEnvelopeMetadataSource(),

embrace-android-sdk/src/test/resources/session_message_expected.json

@@ -45,5 +45,16 @@
       "attributes": {}
     }
   ],
+  "span_snapshots": [
+    {
+      "trace_id": "snapshot-trace-id",
+      "span_id": "snapshot-span-id",
+      "name": "snapshot",
+      "start_time_unix_nano": 1692201601000,
+      "status": "Unset",
+      "events": [],
+      "attributes": []
+    }
+  ],
   "v": 13
 }