[8.x] [Entity Analytics] [Entity Store] [API] Changes to support even…

…t.ingested as a configurable timestamp field for init and enable endpoints (#208201) (#208866) # Backport This will backport the following commits from `main` to `8.x`: - [[Entity Analytics] [Entity Store] [API] Changes to support event.ingested as a configurable timestamp field for init and enable endpoints (#208201)](#208201)  ### Questions ? Please refer to the [Backport tool documentation](https://github.com/sorenlouv/backport)  Co-authored-by: Abhishek Bhatia <[email protected]>
elastic · Jan 30, 2025 · ad15e7d · ad15e7d
1 parent afc479c
commit ad15e7d
Show file tree

Hide file tree

Showing 21 changed files with 185 additions and 32 deletions.
diff --git a/oas_docs/output/kibana.serverless.yaml b/oas_docs/output/kibana.serverless.yaml
@@ -7489,6 +7489,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp.
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -7629,6 +7633,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp for the entity type.
+                  type: string
         description: Schema for the engine initialization
         required: true
       responses:
@@ -7646,7 +7654,7 @@ paths:
     post:
       operationId: StartEntityEngine
       parameters:
-        - description: The entity type of the engine (either 'user' or 'host').
+        - description: The entity type of the engine
           in: path
           name: entityType
           required: true
@@ -47684,6 +47692,8 @@ components:
           default: 180s
           pattern: '[smdh]$'
           type: string
+        timestampField:
+          type: string
         type:
           $ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
       required:
@@ -47811,6 +47821,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         host:
           type: object
           properties:
@@ -47849,7 +47865,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - host
         - entity
     Security_Entity_Analytics_API_IdField:
@@ -47956,6 +47971,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         service:
           type: object
           properties:
@@ -47966,7 +47987,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - service
         - entity
     Security_Entity_Analytics_API_StoreStatus:
@@ -48012,6 +48032,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         user:
           type: object
           properties:
@@ -48046,7 +48072,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - user
         - entity
     Security_Exceptions_API_CreateExceptionListItemComment:

diff --git a/oas_docs/output/kibana.yaml b/oas_docs/output/kibana.yaml
@@ -13070,6 +13070,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp.
+                  type: string
         description: Schema for the entity store initialization
         required: true
       responses:
@@ -13206,6 +13210,10 @@ paths:
                   description: The timeout for initializing the aggregating transform.
                   pattern: '[smdh]$'
                   type: string
+                timestampField:
+                  default: '@timestamp'
+                  description: The field to use as the timestamp for the entity type.
+                  type: string
         description: Schema for the engine initialization
         required: true
       responses:
@@ -13222,7 +13230,7 @@ paths:
     post:
       operationId: StartEntityEngine
       parameters:
-        - description: The entity type of the engine (either 'user' or 'host').
+        - description: The entity type of the engine
           in: path
           name: entityType
           required: true
@@ -36075,6 +36083,8 @@ components:
           default: 180s
           pattern: '[smdh]$'
           type: string
+        timestampField:
+          type: string
         type:
           $ref: '#/components/schemas/Security_Entity_Analytics_API_EntityType'
       required:
@@ -36202,6 +36212,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         host:
           type: object
           properties:
@@ -36240,7 +36256,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - host
         - entity
     Security_Entity_Analytics_API_IdField:
@@ -36347,6 +36362,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         service:
           type: object
           properties:
@@ -36357,7 +36378,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - service
         - entity
     Security_Entity_Analytics_API_StoreStatus:
@@ -36403,6 +36423,12 @@ components:
           required:
             - name
             - source
+        event:
+          type: object
+          properties:
+            ingested:
+              format: date-time
+              type: string
         user:
           type: object
           properties:
@@ -36437,7 +36463,6 @@ components:
           required:
             - name
       required:
-        - '@timestamp'
         - user
         - entity
     Security_Exceptions_API_CreateExceptionListItemComment:

diff --git a/packages/kbn-check-mappings-update-cli/current_fields.json b/packages/kbn-check-mappings-update-cli/current_fields.json
@@ -321,6 +321,7 @@
     "filter",
     "indexPattern",
     "status",
+    "timestampField",
     "type"
   ],
   "epm-packages": [

diff --git a/packages/kbn-check-mappings-update-cli/current_mappings.json b/packages/kbn-check-mappings-update-cli/current_mappings.json
@@ -1092,6 +1092,9 @@
       "status": {
         "type": "keyword"
       },
+      "timestampField": {
+        "type": "keyword"
+      },
       "type": {
         "type": "keyword"
       }

diff --git a/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts b/src/core/server/integration_tests/ci_checks/saved_objects/check_registered_types.test.ts
@@ -94,7 +94,7 @@ describe('checking migration metadata changes on all registered SO types', () =>
         "enterprise_search_telemetry": "9ac912e1417fc8681e0cd383775382117c9e3d3d",
         "entity-definition": "1c6bff35c423d5dc5650bc806cf2899e4706a0bc",
         "entity-discovery-api-key": "c267a65c69171d1804362155c1378365f5acef88",
-        "entity-engine-status": "e2de87d84e9f1f72726eb28b7e670ff8021b5eb4",
+        "entity-engine-status": "09f6a617020708e4f638137e5ef35bd9534133be",
         "epm-packages": "8042d4a1522f6c4e6f5486e791b3ffe3a22f88fd",
         "epm-packages-assets": "7a3e58efd9a14191d0d1a00b8aaed30a145fd0b1",
         "event-annotation-group": "715ba867d8c68f3c9438052210ea1c30a9362582",

diff --git a/...security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts b/...security/plugins/security_solution/common/api/entity_analytics/entity_store/common.gen.ts
@@ -41,6 +41,7 @@ export const EngineDescriptor = z.object({
     .regex(/[smdh]$/)
     .optional()
     .default('24h'),
+  timestampField: z.string().optional(),
   timeout: z
     .string()
     .regex(/[smdh]$/)

diff --git a/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml b/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/common.schema.yaml
@@ -36,6 +36,8 @@ components:
           type: string
           default: 24h
           pattern: '[smdh]$'
+        timestampField:
+          type: string
         timeout:
           type: string
           default: 180s

diff --git a/...security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts b/...security/plugins/security_solution/common/api/entity_analytics/entity_store/enable.gen.ts
@@ -28,6 +28,10 @@ export const InitEntityStoreRequestBody = z.object({
   filter: z.string().optional(),
   entityTypes: z.array(EntityType).optional(),
   enrichPolicyExecutionInterval: Interval.optional(),
+  /**
+   * The field to use as the timestamp.
+   */
+  timestampField: z.string().optional().default('@timestamp'),
   /**
    * The amount of time the transform looks back to calculate the aggregations.
    */

diff --git a/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml b/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/enable.schema.yaml
@@ -33,6 +33,10 @@ paths:
                     $ref: './common.schema.yaml#/components/schemas/EntityType'
                 enrichPolicyExecutionInterval:
                   $ref: './common.schema.yaml#/components/schemas/Interval'
+                timestampField:
+                  type: string
+                  description: The field to use as the timestamp.
+                  default: '@timestamp'
                 lookbackPeriod: 
                   type: string
                   default: 24h

diff --git a/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.gen.ts b/...ity/plugins/security_solution/common/api/entity_analytics/entity_store/engine/init.gen.ts
@@ -36,6 +36,10 @@ export const InitEntityEngineRequestBody = z.object({
   indexPattern: IndexPattern.optional(),
   filter: z.string().optional(),
   enrichPolicyExecutionInterval: Interval.optional(),
+  /**
+   * The field to use as the timestamp for the entity type.
+   */
+  timestampField: z.string().optional().default('@timestamp'),
   /**
    * The amount of time the transform looks back to calculate the aggregations.
    */

diff --git a/...lugins/security_solution/common/api/entity_analytics/entity_store/engine/init.schema.yaml b/...lugins/security_solution/common/api/entity_analytics/entity_store/engine/init.schema.yaml
@@ -35,7 +35,10 @@ paths:
                   type: string
                 enrichPolicyExecutionInterval:
                   $ref: '../common.schema.yaml#/components/schemas/Interval'
-
+                timestampField:
+                  type: string
+                  description: The field to use as the timestamp for the entity type.
+                  default: '@timestamp'
                 lookbackPeriod: 
                   type: string
                   default: 24h

diff --git a/...ty/plugins/security_solution/common/api/entity_analytics/entity_store/engine/start.gen.ts b/...ty/plugins/security_solution/common/api/entity_analytics/entity_store/engine/start.gen.ts
@@ -21,7 +21,7 @@ import { EntityType } from '../common.gen';
 export type StartEntityEngineRequestParams = z.infer<typeof StartEntityEngineRequestParams>;
 export const StartEntityEngineRequestParams = z.object({
   /**
-   * The entity type of the engine (either 'user' or 'host').
+   * The entity type of the engine
    */
   entityType: EntityType,
 });

diff --git a/...ugins/security_solution/common/api/entity_analytics/entity_store/engine/start.schema.yaml b/...ugins/security_solution/common/api/entity_analytics/entity_store/engine/start.schema.yaml
@@ -16,7 +16,7 @@ paths:
           required: true
           schema:
             $ref: '../common.schema.yaml#/components/schemas/EntityType'
-          description: The entity type of the engine (either 'user' or 'host').
+          description: The entity type of the engine
       responses:
         '200':
           description: Successful response

diff --git a/...plugins/security_solution/common/api/entity_analytics/entity_store/entities/common.gen.ts b/...plugins/security_solution/common/api/entity_analytics/entity_store/entities/common.gen.ts
@@ -21,7 +21,7 @@ import { AssetCriticalityLevel } from '../../asset_criticality/common.gen';
 
 export type UserEntity = z.infer<typeof UserEntity>;
 export const UserEntity = z.object({
-  '@timestamp': z.string().datetime(),
+  '@timestamp': z.string().datetime().optional(),
   entity: z.object({
     name: z.string(),
     source: z.string(),
@@ -41,11 +41,16 @@ export const UserEntity = z.object({
       criticality: AssetCriticalityLevel,
     })
     .optional(),
+  event: z
+    .object({
+      ingested: z.string().datetime().optional(),
+    })
+    .optional(),
 });
 
 export type HostEntity = z.infer<typeof HostEntity>;
 export const HostEntity = z.object({
-  '@timestamp': z.string().datetime(),
+  '@timestamp': z.string().datetime().optional(),
   entity: z.object({
     name: z.string(),
     source: z.string(),
@@ -66,11 +71,16 @@ export const HostEntity = z.object({
       criticality: AssetCriticalityLevel,
     })
     .optional(),
+  event: z
+    .object({
+      ingested: z.string().datetime().optional(),
+    })
+    .optional(),
 });
 
 export type ServiceEntity = z.infer<typeof ServiceEntity>;
 export const ServiceEntity = z.object({
-  '@timestamp': z.string().datetime(),
+  '@timestamp': z.string().datetime().optional(),
   entity: z.object({
     name: z.string(),
     source: z.string(),
@@ -84,6 +94,11 @@ export const ServiceEntity = z.object({
       criticality: AssetCriticalityLevel,
     })
     .optional(),
+  event: z
+    .object({
+      ingested: z.string().datetime().optional(),
+    })
+    .optional(),
 });
 
 export const EntityInternal = z.union([UserEntity, HostEntity, ServiceEntity]);