diff --git a/x-pack/plugins/fleet/common/constants/mappings.ts b/x-pack/plugins/fleet/common/constants/mappings.ts index f3d2b200cac58..f90738fef173c 100644 --- a/x-pack/plugins/fleet/common/constants/mappings.ts +++ b/x-pack/plugins/fleet/common/constants/mappings.ts @@ -102,6 +102,40 @@ export const AGENT_MAPPINGS = { }, }, }, + // Will see if can put that behind the feature flag + agent_policy: { + properties: { + name: { + type: 'text', + }, + namespace: { + type: 'keyword', + }, + package_policies: { + properties: { + id: { + type: 'keyword', + }, + name: { + type: 'text', + }, + namespace: { + type: 'keyword', + }, + package: { + properties: { + name: { + type: 'keyword', + }, + version: { + type: 'keyword', + }, + }, + }, + }, + }, + }, + }, default_api_key: { type: 'keyword', }, diff --git a/x-pack/plugins/fleet/common/experimental_features.ts b/x-pack/plugins/fleet/common/experimental_features.ts index 07fd2caf0f061..7c0fd425ae180 100644 --- a/x-pack/plugins/fleet/common/experimental_features.ts +++ b/x-pack/plugins/fleet/common/experimental_features.ts @@ -28,6 +28,7 @@ const _allowedExperimentalValues = { enableReusableIntegrationPolicies: true, asyncDeployPolicies: true, enableExportCSV: false, + enrichAgentPolicies: false, }; /** diff --git a/x-pack/plugins/fleet/server/services/agent_policies/agent_policies_enrich.ts b/x-pack/plugins/fleet/server/services/agent_policies/agent_policies_enrich.ts new file mode 100644 index 0000000000000..49312005ae32b --- /dev/null +++ b/x-pack/plugins/fleet/server/services/agent_policies/agent_policies_enrich.ts @@ -0,0 +1,104 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import pMap from 'p-map'; +import { type ElasticsearchClient } from '@kbn/core-elasticsearch-server'; +import { type SavedObjectsClientContract } from '@kbn/core-saved-objects-api-server'; +import { type BulkResponseItem } from '@elastic/elasticsearch/lib/api/typesWithBodyKey'; + +import { agentPolicyService } from '../agent_policy'; +import { appContextService } from '../app_context'; + +export async function updatePoliciesEnrich( + soClient: SavedObjectsClientContract, + esClient: ElasticsearchClient, + agentPolicyIds: string[] +) { + // TODO move outside of that service + const agentPoliciesBulkBody = ( + await pMap(agentPolicyIds, (policyId) => { + return agentPolicyService.get(soClient, policyId, true); + }) + ).flatMap((agentPolicy) => + agentPolicy + ? [ + { + update: { + _id: agentPolicy.id, + _index: '.fleet-agent-policies-metadata', + retry_on_conflict: 3, + }, + }, + { + doc: { + policy_id: agentPolicy.id, + agent_policy: { + id: agentPolicy.id, + name: agentPolicy.name, + namespace: agentPolicy.namespace, + inactivity_timeout: agentPolicy.inactivity_timeout, + is_managed: agentPolicy.is_managed, + package_policies: agentPolicy.package_policies?.map((packagePolicy) => ({ + id: agentPolicy.id, + name: packagePolicy.name, + namespace: packagePolicy.namespace, + package: packagePolicy.package + ? { + name: packagePolicy.package.name, + version: packagePolicy.package.version, + } + : undefined, + })), + }, + }, + doc_as_upsert: true, + }, + ] + : [] + ); + + // Deploy fleet-policies-metadata + // Could be optimized if feature flag is adopted + const agentPoliciesBulkResponse = await esClient.bulk({ + index: '.fleet-agent-policies-metadata', // TODO use a constant + operations: agentPoliciesBulkBody, + refresh: 'wait_for', + }); + + if (agentPoliciesBulkResponse.errors) { + const logger = appContextService.getLogger(); + const erroredDocuments = agentPoliciesBulkResponse.items.reduce((acc, item) => { + const value: BulkResponseItem | undefined = item.index; + if (!value || !value.error) { + return acc; + } + + acc.push(value); + return acc; + }, [] as BulkResponseItem[]); + + logger.warn( + `Failed to index agent policy metadata during policy deployment: ${JSON.stringify( + erroredDocuments + )}` + ); + } + // execute enrich policy + await esClient.enrich.executePolicy({ + name: 'fleet-agents-enrich-agent-policies', + wait_for_completion: true, + }); + + await pMap(agentPolicyIds, (policyId) => { + // Update will go through the ingest pipeline again + return esClient.updateByQuery({ + index: '.fleet-agents', + q: `policy_id:"${policyId}"`, + ignore_unavailable: true, + }); + }); +} diff --git a/x-pack/plugins/fleet/server/services/agent_policy.ts b/x-pack/plugins/fleet/server/services/agent_policy.ts index 21614d2a97481..051796cbe13cb 100644 --- a/x-pack/plugins/fleet/server/services/agent_policy.ts +++ b/x-pack/plugins/fleet/server/services/agent_policy.ts @@ -122,6 +122,7 @@ import { validatePolicyNamespaceForSpace } from './spaces/policy_namespaces'; import { isSpaceAwarenessEnabled } from './spaces/helpers'; import { agentlessAgentService } from './agents/agentless_agent'; import { scheduleDeployAgentPoliciesTask } from './agent_policies/deploy_agent_policies_task'; +import { updatePoliciesEnrich } from './agent_policies/agent_policies_enrich'; const KEY_EDITABLE_FOR_MANAGED_POLICIES = ['namespace']; @@ -1400,6 +1401,10 @@ class AgentPolicyService { fleetServerPolicy, ]); + if (appContextService.getExperimentalFeatures()?.enrichAgentPolicies) { + await updatePoliciesEnrich(soClient, esClient, agentPolicyIds); + } + const bulkResponse = await esClient.bulk({ index: AGENT_POLICY_INDEX, operations: fleetServerPoliciesBulkBody, diff --git a/x-pack/plugins/fleet/server/services/setup.ts b/x-pack/plugins/fleet/server/services/setup.ts index ab882a013ebe3..b1d5bebfc2950 100644 --- a/x-pack/plugins/fleet/server/services/setup.ts +++ b/x-pack/plugins/fleet/server/services/setup.ts @@ -12,7 +12,7 @@ import apm from 'elastic-apm-node'; import { compact } from 'lodash'; import pMap from 'p-map'; import { v4 as uuidv4 } from 'uuid'; -import type { ElasticsearchClient, SavedObjectsClientContract } from '@kbn/core/server'; +import type { ElasticsearchClient, Logger, SavedObjectsClientContract } from '@kbn/core/server'; import { DEFAULT_SPACE_ID } from '@kbn/spaces-plugin/common/constants'; import { MessageSigningError } from '../../common/errors'; @@ -340,6 +340,9 @@ export async function ensureFleetGlobalEsAssets( ensureDefaultComponentTemplates(esClient, logger), // returns an array ensureFleetFinalPipelineIsInstalled(esClient, logger), ensureFleetEventIngestedPipelineIsInstalled(esClient, logger), + appContextService.getExperimentalFeatures()?.enrichAgentPolicies + ? ensureFleetAgentsEnrichAgentPolicies(esClient, logger) + : { isCreated: false }, ]); const assetResults = globalAssetsRes.flat(); if (assetResults.some((asset) => asset.isCreated)) { @@ -419,3 +422,55 @@ export async function ensureFleetDirectories() { ); } } +async function ensureFleetAgentsEnrichAgentPolicies(esClient: ElasticsearchClient, logger: Logger) { + // ensure index is created + await esClient.indices + .create({ + index: '.fleet-agent-policies-metadata', + }) + .catch((err) => { + // Ignore already created + }); + + await esClient.enrich + .putPolicy({ + name: 'fleet-agents-enrich-agent-policies', + match: { + indices: '.fleet-agent-policies-metadata', + match_field: 'policy_id', + enrich_fields: ['agent_policy'], + }, + }) + .catch((err) => { + // Ignore already created + }); + + await esClient.enrich.executePolicy({ + name: 'fleet-agents-enrich-agent-policies', + wait_for_completion: true, + }); + + await esClient.ingest.putPipeline({ + id: 'fleet-agents@enrich-agent-policies-pipeline', + processors: [ + { + enrich: { + description: "Add 'user' data based on 'email'", + policy_name: 'fleet-agents-enrich-agent-policies', + field: 'policy_id', + target_field: 'agent_policy', + max_matches: 1, + }, + }, + { + set: { + field: 'agent_policy', + copy_from: 'agent_policy.agent_policy', + }, + }, + ], + }); + + // No need to reinstall package after that one + return { isCreated: false }; +}