[Fleet] Enrich agent with agent policies

x-pack/plugins/fleet/common/constants/mappings.ts

@@ -102,6 +102,40 @@ export const AGENT_MAPPINGS = {
         },
       },
     },
+    // Will see if can put that behind the feature flag
+    agent_policy: {
+      properties: {
+        name: {
+          type: 'text',
+        },
+        namespace: {
+          type: 'keyword',
+        },
+        package_policies: {
+          properties: {
+            id: {
+              type: 'keyword',
+            },
+            name: {
+              type: 'text',
+            },
+            namespace: {
+              type: 'keyword',
+            },
+            package: {
+              properties: {
+                name: {
+                  type: 'keyword',
+                },
+                version: {
+                  type: 'keyword',
+                },
+              },
+            },
+          },
+        },
+      },
+    },
     default_api_key: {
       type: 'keyword',
     },

x-pack/plugins/fleet/common/experimental_features.ts

@@ -28,6 +28,7 @@ const _allowedExperimentalValues = {
   enableReusableIntegrationPolicies: true,
   asyncDeployPolicies: true,
   enableExportCSV: false,
+  enrichAgentPolicies: false,
 };
 
 /**

x-pack/plugins/fleet/server/services/agent_policies/agent_policies_enrich.ts

@@ -0,0 +1,104 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0; you may not use this file except in compliance with the Elastic License
+ * 2.0.
+ */
+
+import pMap from 'p-map';
+import { type ElasticsearchClient } from '@kbn/core-elasticsearch-server';
+import { type SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server';
+import { type BulkResponseItem } from '@elastic/elasticsearch/lib/api/typesWithBodyKey';
+
+import { agentPolicyService } from '../agent_policy';
+import { appContextService } from '../app_context';
+
+export async function updatePoliciesEnrich(
+  soClient: SavedObjectsClientContract,
+  esClient: ElasticsearchClient,
+  agentPolicyIds: string[]
+) {
+  // TODO move outside of that service
+  const agentPoliciesBulkBody = (
+    await pMap(agentPolicyIds, (policyId) => {
+      return agentPolicyService.get(soClient, policyId, true);
+    })
+  ).flatMap((agentPolicy) =>
+    agentPolicy
+      ? [
+          {
+            update: {
+              _id: agentPolicy.id,
+              _index: '.fleet-agent-policies-metadata',
+              retry_on_conflict: 3,
+            },
+          },
+          {
+            doc: {
+              policy_id: agentPolicy.id,
+              agent_policy: {
+                id: agentPolicy.id,
+                name: agentPolicy.name,
+                namespace: agentPolicy.namespace,
+                inactivity_timeout: agentPolicy.inactivity_timeout,
+                is_managed: agentPolicy.is_managed,
+                package_policies: agentPolicy.package_policies?.map((packagePolicy) => ({
+                  id: agentPolicy.id,
+                  name: packagePolicy.name,
+                  namespace: packagePolicy.namespace,
+                  package: packagePolicy.package
+                    ? {
+                        name: packagePolicy.package.name,
+                        version: packagePolicy.package.version,
+                      }
+                    : undefined,
+                })),
+              },
+            },
+            doc_as_upsert: true,
+          },
+        ]
+      : []
+  );
+
+  // Deploy fleet-policies-metadata
+  // Could be optimized if feature flag is adopted
+  const agentPoliciesBulkResponse = await esClient.bulk({
+    index: '.fleet-agent-policies-metadata', // TODO use a constant
+    operations: agentPoliciesBulkBody,
+    refresh: 'wait_for',
+  });
+
+  if (agentPoliciesBulkResponse.errors) {
+    const logger = appContextService.getLogger();
+    const erroredDocuments = agentPoliciesBulkResponse.items.reduce((acc, item) => {
+      const value: BulkResponseItem | undefined = item.index;
+      if (!value || !value.error) {
+        return acc;
+      }
+
+      acc.push(value);
+      return acc;
+    }, [] as BulkResponseItem[]);
+
+    logger.warn(
+      `Failed to index agent policy metadata during policy deployment: ${JSON.stringify(
+        erroredDocuments
+      )}`
+    );
+  }
+  // execute enrich policy
+  await esClient.enrich.executePolicy({
+    name: 'fleet-agents-enrich-agent-policies',
+    wait_for_completion: true,
+  });
+
+  await pMap(agentPolicyIds, (policyId) => {
+    // Update will go through the ingest pipeline again
+    return esClient.updateByQuery({
+      index: '.fleet-agents',
+      q: `policy_id:"${policyId}"`,
+      ignore_unavailable: true,
+    });
+  });
+}

x-pack/plugins/fleet/server/services/agent_policy.ts

@@ -122,6 +122,7 @@ import { validatePolicyNamespaceForSpace } from './spaces/policy_namespaces';
 import { isSpaceAwarenessEnabled } from './spaces/helpers';
 import { agentlessAgentService } from './agents/agentless_agent';
 import { scheduleDeployAgentPoliciesTask } from './agent_policies/deploy_agent_policies_task';
+import { updatePoliciesEnrich } from './agent_policies/agent_policies_enrich';
 
 const KEY_EDITABLE_FOR_MANAGED_POLICIES = ['namespace'];
 
@@ -1400,6 +1401,10 @@ class AgentPolicyService {
       fleetServerPolicy,
     ]);
 
+    if (appContextService.getExperimentalFeatures()?.enrichAgentPolicies) {
+      await updatePoliciesEnrich(soClient, esClient, agentPolicyIds);
+    }
+
     const bulkResponse = await esClient.bulk({
       index: AGENT_POLICY_INDEX,
       operations: fleetServerPoliciesBulkBody,

x-pack/plugins/fleet/server/services/setup.ts

@@ -12,7 +12,7 @@ import apm from 'elastic-apm-node';
 import { compact } from 'lodash';
 import pMap from 'p-map';
 import { v4 as uuidv4 } from 'uuid';
-import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server';
+import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server';
 import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants';
 
 import { MessageSigningError } from '../../common/errors';
@@ -340,6 +340,9 @@ export async function ensureFleetGlobalEsAssets(
     ensureDefaultComponentTemplates(esClient, logger), // returns an array
     ensureFleetFinalPipelineIsInstalled(esClient, logger),
     ensureFleetEventIngestedPipelineIsInstalled(esClient, logger),
+    appContextService.getExperimentalFeatures()?.enrichAgentPolicies
+      ? ensureFleetAgentsEnrichAgentPolicies(esClient, logger)
+      : { isCreated: false },
   ]);
   const assetResults = globalAssetsRes.flat();
   if (assetResults.some((asset) => asset.isCreated)) {
@@ -419,3 +422,55 @@ export async function ensureFleetDirectories() {
     );
   }
 }
+async function ensureFleetAgentsEnrichAgentPolicies(esClient: ElasticsearchClient, logger: Logger) {
+  // ensure index is created
+  await esClient.indices
+    .create({
+      index: '.fleet-agent-policies-metadata',
+    })
+    .catch((err) => {
+      // Ignore already created
+    });
+
+  await esClient.enrich
+    .putPolicy({
+      name: 'fleet-agents-enrich-agent-policies',
+      match: {
+        indices: '.fleet-agent-policies-metadata',
+        match_field: 'policy_id',
+        enrich_fields: ['agent_policy'],
+      },
+    })
+    .catch((err) => {
+      // Ignore already created
+    });
+
+  await esClient.enrich.executePolicy({
+    name: 'fleet-agents-enrich-agent-policies',
+    wait_for_completion: true,
+  });
+
+  await esClient.ingest.putPipeline({
+    id: 'fleet-agents@enrich-agent-policies-pipeline',
+    processors: [
+      {
+        enrich: {
+          description: "Add 'user' data based on 'email'",
+          policy_name: 'fleet-agents-enrich-agent-policies',
+          field: 'policy_id',
+          target_field: 'agent_policy',
+          max_matches: 1,
+        },
+      },
+      {
+        set: {
+          field: 'agent_policy',
+          copy_from: 'agent_policy.agent_policy',
+        },
+      },
+    ],
+  });
+
+  // No need to reinstall package after that one
+  return { isCreated: false };
+}