From 015bbaafb480b0f5a0c10b064b59fdb330162fbb Mon Sep 17 00:00:00 2001 From: Wafaa Nasr Date: Sat, 28 Oct 2023 21:37:57 +0200 Subject: [PATCH] [Security Solution][Exceptions][API testing] Refactor commands in package.json (#170025) ## Summary - Refactor package.json to remove duplicates - Move the new term rules --- .../security_and_spaces/group1/index.ts | 1 - .../README.md | 46 +++++++++++++++ .../package.json | 52 +++++++++-------- .../scripts/index.js | 57 +++++++++++++++++++ .../rule_creation}/create_new_terms.ts | 12 ++-- .../default_license/rule_creation/index.ts | 1 + 6 files changed, 138 insertions(+), 31 deletions(-) create mode 100644 x-pack/test/security_solution_api_integration/scripts/index.js rename x-pack/test/{detection_engine_api_integration/security_and_spaces/group1 => security_solution_api_integration/test_suites/detections_response/default_license/rule_creation}/create_new_terms.ts (90%) diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/index.ts b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/index.ts index 2969429494b28..2ce7684dedd48 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/index.ts +++ b/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/index.ts @@ -21,7 +21,6 @@ export default ({ loadTestFile }: FtrProviderContext): void => { loadTestFile(require.resolve('./create_index')); loadTestFile(require.resolve('./preview_rules')); loadTestFile(require.resolve('./create_rules_bulk')); - loadTestFile(require.resolve('./create_new_terms')); loadTestFile(require.resolve('./delete_rules')); loadTestFile(require.resolve('./delete_rules_bulk')); loadTestFile(require.resolve('./export_rules')); diff --git a/x-pack/test/security_solution_api_integration/README.md b/x-pack/test/security_solution_api_integration/README.md index 5995b41164aec..ebdf649e1e2bb 100644 --- a/x-pack/test/security_solution_api_integration/README.md +++ b/x-pack/test/security_solution_api_integration/README.md @@ -40,7 +40,53 @@ ex: In the `package.json` file, you'll find commands to configure the server for each environment and to run tests against that specific environment. These commands adhere to the Mocha tagging system, allowing for the inclusion and exclusion of tags, mirroring the setup of the CI pipeline. +## Running Commands with Different Parameters +In this project, you can run various commands to execute tests and workflows, each of which can be customized by specifying different parameters. Below, how to define the commands based on the parameters and their order. +### Command Structure +The command structure follows this pattern: +- ``: The name of the specific command or test case. +- ``: The test folder or workflow you want to run. +- ``: The type of operation, either "server" or "runner." +- ``: The testing environment, such as "serverlessEnv," "essEnv," or "qaEnv." +- ``: The license folder the test is defined under such as "default_license", by default the value is "default_license" +- ``: The area the test is defined under, such as "detection_engine", by default the value is "detection_engine" + +### Serverless and Ess Configuration + +- When using "serverless" or "ess" in the script, it specifies the correct configuration file for the tests. +- "Serverless" and "ess" help determine the configuration specific to the chosen test. + +### serverlessEnv, essEnv, qaEnv Grep Command + +- When using "serverlessEnv,.." in the script, it appends the correct grep command for filtering tests in the serverless testing environment. +- "serverlessEnv,..." is used to customize the test execution based on the serverless environment. + + +### Command Examples + +Here are some command examples using the provided parameters: + +1. **Run the server for "exception_workflows" in the "serverlessEnv" environment:** + ```shell + npm run initialize-server exceptions/workflows serverless + ``` +2. **To run tests for the "exception_workflows" using the serverless runner in the "serverlessEnv" environment, you can use the following command:** + ```shell + npm run run-tests exceptions/workflows serverless serverlessEnv + ``` +3. **Run tests for "exception_workflows" using the serverless runner in the "qaEnv" environment:** + ```shell + npm run run-tests exceptions/workflows serverless qaEnv + ``` +4. **Run the server for "exception_workflows" in the "essEnv" environment:** + ```shell + npm run initialize-server exceptions/workflows ess + ``` +5. **Run tests for "exception_workflows" using the ess runner in the "essEnv" environment:** + ```shell + npm run run-tests exceptions/workflows ess essEnv + ``` \ No newline at end of file diff --git a/x-pack/test/security_solution_api_integration/package.json b/x-pack/test/security_solution_api_integration/package.json index d362078fc03b1..4562cfc82cfc9 100644 --- a/x-pack/test/security_solution_api_integration/package.json +++ b/x-pack/test/security_solution_api_integration/package.json @@ -5,30 +5,32 @@ "private": true, "license": "Elastic License 2.0", "scripts": { - "exception_workflows:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts", - "exception_workflows:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts --grep @serverless --grep @brokenInServerless --invert", - "exception_workflows:qa:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/workflows/configs/serverless.config.ts --grep @serverless --grep '@brokenInServerless|@skipInQA' --invert", - "exception_workflows:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/workflows/configs/ess.config.ts", - "exception_workflows:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/workflows/configs/ess.config.ts --grep @ess", - "exception_operators_date_numeric_types:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts", - "exception_operators_date_numeric_types:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts --grep @serverless --grep @brokenInServerless --invert", - "exception_operators_date_numeric_types:qa:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/serverless.config.ts --grep @serverless --grep '@brokenInServerless|@skipInQA' --invert", - "exception_operators_date_numeric_types:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/ess.config.ts", - "exception_operators_date_numeric_types:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/date_numeric_types/configs/ess.config.ts --grep @ess", - "exception_operators_keyword_text_long:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/serverless.config.ts", - "exception_operators_keyword_text_long:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/serverless.config.ts --grep @serverless --grep @brokenInServerless --invert", - "exception_operators_keyword_text_long:qa:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/serverless.config.ts --grep @serverless --grep '@brokenInServerless|@skipInQA' --invert", - "exception_operators_keyword_text_long:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/ess.config.ts", - "exception_operators_keyword_text_long:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/keyword_text_long/configs/ess.config.ts --grep @ess", - "exception_operators_ips_text_array:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/serverless.config.ts", - "exception_operators_ips_text_array:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/serverless.config.ts --grep @serverless --grep @brokenInServerless --invert", - "exception_operators_ips_text_array:qa:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/serverless.config.ts --grep @serverless --grep '@brokenInServerless|@skipInQA' --invert", - "exception_operators_ips_text_array:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/ess.config.ts", - "exception_operators_ips_text_array:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/exceptions/operators_data_types/ips_text_array/configs/ess.config.ts --grep @ess", - "rule_creation:server:serverless": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts", - "rule_creation:runner:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts --grep @serverless --grep @brokenInServerless --invert", - "rule_creation:qa:serverless": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/rule_creation/configs/serverless.config.ts --grep @serverless --grep '@brokenInServerless|@skipInQA' --invert", - "rule_creation:server:ess": "node ../../../scripts/functional_tests_server.js --config ./test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts", - "rule_creation:runner:ess": "node ../../../scripts/functional_test_runner --config=test_suites/detections_response/default_license/rule_creation/configs/ess.config.ts --grep @ess" + "initialize-server": "node ./scripts/index.js server", + "run-tests": "node ./scripts/index.js runner", + "exception_workflows:server:serverless": "npm run initialize-server exceptions/workflows serverless", + "exception_workflows:runner:serverless": "npm run run-tests exceptions/workflows serverless serverlessEnv", + "exception_workflows:qa:serverless": "npm run run-tests exceptions/workflows serverless qaEnv", + "exception_workflows:server:ess": "npm run initialize-server exceptions/workflows ess", + "exception_workflows:runner:ess": "npm run run-tests exceptions/workflows ess essEnv", + "exception_operators_date_numeric_types:server:serverless": "npm run initialize-server exceptions/operators_data_types/date_numeric_types serverless", + "exception_operators_date_numeric_types:runner:serverless": "npm run run-tests exceptions/operators_data_types/date_numeric_types serverless serverlessEnv", + "exception_operators_date_numeric_types:qa:serverless": "npm run run-tests exceptions/operators_data_types/date_numeric_types serverless qaEnv", + "exception_operators_date_numeric_types:server:ess": "npm run initialize-server exceptions/operators_data_types/date_numeric_types ess", + "exception_operators_date_numeric_types:runner:ess": "npm run run-tests exceptions/operators_data_types/date_numeric_types ess essEnv", + "exception_operators_keyword_text_long:server:serverless": "npm run initialize-server exceptions/operators_data_types/keyword_text_long serverless", + "exception_operators_keyword_text_long:runner:serverless": "npm run run-tests exceptions/operators_data_types/keyword_text_long serverless serverlessEnv", + "exception_operators_keyword_text_long:qa:serverless": "npm run run-tests exceptions/operators_data_types/keyword_text_long serverless qaEnv", + "exception_operators_keyword_text_long:server:ess": "npm run initialize-server exceptions/operators_data_types/keyword_text_long ess", + "exception_operators_keyword_text_long:runner:ess": "npm run run-tests exceptions/operators_data_types/keyword_text_long ess essEnv", + "exception_operators_ips_text_array:server:serverless": "npm run initialize-server exceptions/operators_data_types/ips_text_array serverless", + "exception_operators_ips_text_array:runner:serverless": "npm run run-tests exceptions/operators_data_types/ips_text_array serverless serverlessEnv", + "exception_operators_ips_text_array:qa:serverless": "npm run run-tests exceptions/operators_data_types/ips_text_array serverless qaEnv", + "exception_operators_ips_text_array:server:ess": "npm run initialize-server exceptions/operators_data_types/ips_text_array ess", + "exception_operators_ips_text_array:runner:ess": "npm run run-tests exceptions/operators_data_types/ips_text_array ess essEnv", + "rule_creation:server:serverless": "npm run initialize-server rule_creation serverless", + "rule_creation:runner:serverless": "npm run run-tests rule_creation serverless serverlessEnv", + "rule_creation:qa:serverless": "npm run run-tests rule_creation serverless qaEnv", + "rule_creation:server:ess": "npm run initialize-server rule_creation ess", + "rule_creation:runner:ess": "npm run run-tests rule_creation ess essEnv" } } diff --git a/x-pack/test/security_solution_api_integration/scripts/index.js b/x-pack/test/security_solution_api_integration/scripts/index.js new file mode 100644 index 0000000000000..635c135e2c8b1 --- /dev/null +++ b/x-pack/test/security_solution_api_integration/scripts/index.js @@ -0,0 +1,57 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +const { spawn } = require('child_process'); + +const [ + , + , + type, + folder, + projectType, + environment, + area = 'detections_response', + licenseFolder = 'default_license', + ...args +] = process.argv; + +const configPath = `./test_suites/${area}/${licenseFolder}/${folder}/configs/${projectType}.config.ts`; + +const command = + type === 'server' + ? '../../scripts/functional_tests_server.js' + : '../../scripts/functional_test_runner'; + +let grepArgs = []; + +if (type !== 'server') { + switch (environment) { + case 'serverlessEnv': + grepArgs = ['--grep', '@serverless', '--grep', '@brokenInServerless', '--invert']; + break; + + case 'essEnv': + grepArgs = ['--grep', '@ess']; + break; + + case 'qaEnv': + grepArgs = ['--grep', '@serverless', '--grep', '@brokenInServerless|@skipInQA', '--invert']; + break; + + default: + console.error(`Unsupported environment: ${environment}`); + process.exit(1); + } +} + +const child = spawn('node', [command, '--config', configPath, ...grepArgs, ...args], { + stdio: 'inherit', +}); + +child.on('close', (code) => { + console.log(`child process exited with code ${code}`); +}); diff --git a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/create_new_terms.ts similarity index 90% rename from x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts rename to x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/create_new_terms.ts index 4353775eab467..459b0e656fae0 100644 --- a/x-pack/test/detection_engine_api_integration/security_and_spaces/group1/create_new_terms.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/create_new_terms.ts @@ -9,10 +9,10 @@ import expect from '@kbn/expect'; import { DETECTION_ENGINE_RULES_URL } from '@kbn/security-solution-plugin/common/constants'; import { getCreateNewTermsRulesSchemaMock } from '@kbn/security-solution-plugin/common/api/detection_engine/model/rule_schema/mocks'; -import { FtrProviderContext } from '../../common/ftr_provider_context'; + import { deleteAllRules } from '../../utils'; +import { FtrProviderContext } from '../../../../ftr_provider_context'; -// eslint-disable-next-line import/no-default-export export default ({ getService }: FtrProviderContext) => { const supertest = getService('supertest'); const log = getService('log'); @@ -20,11 +20,13 @@ export default ({ getService }: FtrProviderContext) => { /** * Specific api integration tests for new terms rule type */ - describe('create_new_terms', () => { - afterEach(async () => { + describe('@serverless @ess create_new_terms', () => { + beforeEach(async () => { + await deleteAllRules(supertest, log); + }); + after(async () => { await deleteAllRules(supertest, log); }); - it('should not be able to create a new terms rule with too small history window', async () => { const rule = { ...getCreateNewTermsRulesSchemaMock('rule-1'), diff --git a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/index.ts b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/index.ts index 49268f31ed9f9..a3e706c580e5c 100644 --- a/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/index.ts +++ b/x-pack/test/security_solution_api_integration/test_suites/detections_response/default_license/rule_creation/index.ts @@ -9,5 +9,6 @@ import { FtrProviderContext } from '../../../../ftr_provider_context'; export default function ({ loadTestFile }: FtrProviderContext) { describe('Rule creation API', function () { loadTestFile(require.resolve('./create_rules')); + loadTestFile(require.resolve('./create_new_terms')); }); }