From b066026e6cf4b34b61f6e55cf9880d1ab316de5a Mon Sep 17 00:00:00 2001 From: Yashvi Jain <91940132+YashviJain01@users.noreply.github.com> Date: Tue, 17 Dec 2024 07:00:13 +0530 Subject: [PATCH] New version of opensearch in automate (#8670) * Testing changes Signed-off-by: Yashvi Jain * Testing changes Signed-off-by: Yashvi Jain * Adding env in user bin Signed-off-by: Yashvi Jain * Cleaning the docs correction in integration tests, exclusing system level index Signed-off-by: Yashvi Jain * fixing pipelines Signed-off-by: Yashvi Jain * Adding Signed-off-by: Yashvi Jain * Adding Signed-off-by: Yashvi Jain * Adding index names in cleanup Signed-off-by: Yashvi Jain * Adding correct number of indicies Signed-off-by: Yashvi Jain --------- Signed-off-by: Yashvi Jain Co-authored-by: Yashvi Jain --- .studio/common | 1 + .studio/deployment-service | 2 + .studiorc | 3 + .../habitat/config/opensearch.yml | 1 + .../config/securityconfig/internal_users.yml | 1 + .../habitat/hooks/post-run | 5 +- .../automate-opensearch/habitat/plan.sh | 21 ++++++- .../integration_test/suite_test.go | 56 +++++++++---------- .../integration_test/purge_test.go | 2 +- .../integration_test/suite_test.go | 13 +++-- .../integration_test/suite_test.go | 12 ++-- 11 files changed, 73 insertions(+), 44 deletions(-) diff --git a/.studio/common b/.studio/common index 6c2d231e2ef..7160a2101ee 100644 --- a/.studio/common +++ b/.studio/common @@ -362,6 +362,7 @@ function build_all_changed_components() { hab pkg install core/coreutils >/dev/null mkdir -p /usr/bin ln -sv "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env 2>/dev/null + cp "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env pushd /src >/dev/null for component in $(./scripts/changed_components.rb) diff --git a/.studio/deployment-service b/.studio/deployment-service index 9fc3d8d92c1..749c3431776 100644 --- a/.studio/deployment-service +++ b/.studio/deployment-service @@ -10,6 +10,8 @@ function start_deployment_service { local local_manifest="" local_manifest=$(find_local_manifest) + cp "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env + chef-automate dev configure-deployment --config dev/config.toml --manifest-dir "$local_manifest" restart_gateway_if_up diff --git a/.studiorc b/.studiorc index 9931437ac32..a99d0d6784c 100644 --- a/.studiorc +++ b/.studiorc @@ -159,6 +159,9 @@ DOC start_all_services() { start_deployment_service chef-automate dev deployinate + + cp "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env + if [[ -f "/src/dev/license.jwt" ]]; then chef-automate license apply "/src/dev/license.jwt" fi diff --git a/components/automate-opensearch/habitat/config/opensearch.yml b/components/automate-opensearch/habitat/config/opensearch.yml index 48145e8ff39..3637e2d6025 100644 --- a/components/automate-opensearch/habitat/config/opensearch.yml +++ b/components/automate-opensearch/habitat/config/opensearch.yml @@ -162,6 +162,7 @@ plugins.security.authcz.admin_dn: plugins.security.audit.type: internal_opensearch plugins.security.enable_snapshot_restore_privilege: true plugins.security.check_snapshot_restore_write_privileges: true +plugins.security.restapi.admin.enabled: true plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"] plugins.security.system_indices.enabled: true plugins.security.system_indices.indices: [".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opendistro-notifications-*", ".opendistro-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"] diff --git a/components/automate-opensearch/habitat/config/securityconfig/internal_users.yml b/components/automate-opensearch/habitat/config/securityconfig/internal_users.yml index 6b21189d8ff..3128ad0e474 100644 --- a/components/automate-opensearch/habitat/config/securityconfig/internal_users.yml +++ b/components/automate-opensearch/habitat/config/securityconfig/internal_users.yml @@ -15,6 +15,7 @@ admin: reserved: true backend_roles: - "admin" + - "security_rest_api_access" description: "Demo admin user" kibanaserver: diff --git a/components/automate-opensearch/habitat/hooks/post-run b/components/automate-opensearch/habitat/hooks/post-run index 8509f37dc06..7ddd0b65838 100644 --- a/components/automate-opensearch/habitat/hooks/post-run +++ b/components/automate-opensearch/habitat/hooks/post-run @@ -2,9 +2,10 @@ exec 2>&1 +sleep 80 # Executing opensearch securityadmin echo "Executing opensearch securityadmin" -hab pkg exec core/bash {{pkg.path}}/os/plugins/opensearch-security/tools/securityadmin.sh -cd {{pkg.svc_config_path}}/securityconfig -icl -key {{pkg.svc_config_path}}/admin-key.pem -cert {{pkg.svc_config_path}}/admin.pem -cacert {{pkg.svc_config_path}}/root-ca.pem -nhnv -h "localhost" -p {{cfg.transport.port}} +hab pkg exec core/bash {{pkg.path}}/os/plugins/opensearch-security/tools/securityadmin.sh -cd {{pkg.svc_config_path}}/securityconfig -icl -key {{pkg.svc_config_path}}/admin-key.pem -cert {{pkg.svc_config_path}}/admin.pem -cacert {{pkg.svc_config_path}}/root-ca.pem -nhnv -h "localhost" -p {{cfg.network.port}} echo "Done !!!! Executing opensearch securityadmin" source {{pkg.svc_config_path}}/health_check @@ -62,7 +63,7 @@ curl \ -H 'Content-Type: application/json'\ -X PUT "https://${HOST}:{{cfg.network.port}}/_all/_settings?preserve_existing=true" \ -k \ - -u admin:admin \ + --cert {{pkg.svc_config_path}}/admin.pem --key {{pkg.svc_config_path}}/admin-key.pem \ -d '{ "index.number_of_replicas": "{{cfg.index.number_of_replicas}}", "index.refresh_interval": "{{cfg.index.refresh_interval}}" diff --git a/components/automate-opensearch/habitat/plan.sh b/components/automate-opensearch/habitat/plan.sh index 79284a7debe..6c27cfdd0b2 100644 --- a/components/automate-opensearch/habitat/plan.sh +++ b/components/automate-opensearch/habitat/plan.sh @@ -5,12 +5,14 @@ pkg_name="automate-opensearch" pkg_description="Wrapper package for core/elasticsearch" pkg_origin="chef" -pkg_version="1.3.19" +pkg_version="2.18.0" pkg_maintainer="Chef Software Inc. " pkg_license=("Chef-MLSA") pkg_upstream_url="https://www.chef.io/automate" -pkg_source="https://artifacts.opensearch.org/releases/bundle/opensearch/1.3.19/opensearch-1.3.19-linux-x64.tar.gz" -pkg_shasum=af901097211df4c3d2ef75f0c3699452d82a87adac60c240520a3ca03f969595 +pkg_source="https://artifacts.opensearch.org/releases/bundle/opensearch/2.18.0/opensearch-2.18.0-linux-x64.tar.gz" +pkg_shasum=ae3cb4107b2e0cdbb9b98bb4e5f2f019b736b68e995442c718e459d39ff01df1 +# pkg_source="https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.1/opensearch-2.11.1-linux-x64.tar.gz" +# pkg_shasum=926eb3f99f592e7518d5dc1e250ae11d1bb45ef766a90222a8df11bb2ee35a9c pkg_build_deps=( @@ -52,7 +54,20 @@ do_build() { return 0 } +do_after() { + if [ ! -f /usr/bin/env ]; then + echo "inside the /usr/ib/env" + cp $(pkg_path_for core/coreutils)/bin/env /usr/bin/env + fi + +} + do_install() { + if [ ! -f /usr/bin/env ]; then + echo "inside the /isr/ib/env" + cp $(pkg_path_for core/coreutils)/bin/env /usr/bin/env + fi + cd "$HAB_CACHE_SRC_PATH/opensearch-${pkg_version}" chown -RL hab:hab ${pkg_prefix} mkdir -p "${pkg_prefix}/os" diff --git a/components/config-mgmt-service/integration_test/suite_test.go b/components/config-mgmt-service/integration_test/suite_test.go index 6058dce624b..857ab07d4da 100644 --- a/components/config-mgmt-service/integration_test/suite_test.go +++ b/components/config-mgmt-service/integration_test/suite_test.go @@ -56,11 +56,11 @@ const ( // multiple tests, consider putting it here so that we have them available globally // // This struct holds: -// * A Ingest backend client, that you can leverate to do all sorts of ingestion. -// => Check this for the list of things this client can do: -// https://github.com/github.com/chef/automate/components/ingest-service/blob/master/backend/client.go#L1 -// * A Elasticsearch client, that you can use to throw ES queries. -// => Docs: https://godoc.org/gopkg.in/olivere/elastic.v5 +// - A Ingest backend client, that you can leverate to do all sorts of ingestion. +// => Check this for the list of things this client can do: +// https://github.com/github.com/chef/automate/components/ingest-service/blob/master/backend/client.go#L1 +// - A Elasticsearch client, that you can use to throw ES queries. +// => Docs: https://godoc.org/gopkg.in/olivere/elastic.v5 type Suite struct { ingest iBackend.Client client *elastic.Client @@ -128,16 +128,12 @@ func (s *Suite) GlobalTeardown() { } indicesToDelete := make([]string, 0) - for _, index := range indices { - //don't ever delete node run info.. we'll do that after each test when needed - if index != mappings.IndexNameNodeRunInfo { - indicesToDelete = append(indicesToDelete, index) - } - } - for i, v := range indicesToDelete { - if v == ".opendistro_security" { - indicesToDelete = append(indicesToDelete[:i], indicesToDelete[i+1:]...) - break + + for _, v := range indices { + if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" || v != mappings.IndexNameNodeRunInfo { + continue + } else { + indicesToDelete = append(indicesToDelete, v) } } time.Sleep(2 * time.Second) @@ -221,15 +217,17 @@ func (s *Suite) Indices() []string { // // You should call this method on every single test as the following example: // ``` -// func TestGrpcFunc(t *testing.T) { -// // Here we are ingesting a number of nodes -// suite.IngestNodes(nodes) // -// // Immediately after the ingestion add the hook to clean all documents, -// // by using `defer` you will ensure that the next test will have clean -// // data regardless if this test passes or fails -// defer suite.DeleteAllDocuments() -// } +// func TestGrpcFunc(t *testing.T) { +// // Here we are ingesting a number of nodes +// suite.IngestNodes(nodes) +// +// // Immediately after the ingestion add the hook to clean all documents, +// // by using `defer` you will ensure that the next test will have clean +// // data regardless if this test passes or fails +// defer suite.DeleteAllDocuments() +// } +// // ``` func (s *Suite) DeleteAllDocuments() { // ES Query to match all documents @@ -237,14 +235,16 @@ func (s *Suite) DeleteAllDocuments() { // Make sure we clean them all! indices, _ := s.client.IndexNames() - for i, v := range indices { - if v == ".opendistro_security" { - indices = append(indices[:i], indices[i+1:]...) - break + indicesToDelete := make([]string, 0) + for _, v := range indices { + if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" { + continue + } else { + indicesToDelete = append(indicesToDelete, v) } } _, err := s.client.DeleteByQuery(). - Index(indices...). + Index(indicesToDelete...). Query(q). IgnoreUnavailable(true). Refresh("true"). diff --git a/components/es-sidecar-service/integration_test/purge_test.go b/components/es-sidecar-service/integration_test/purge_test.go index 85c5aa93985..dffbd14d4da 100644 --- a/components/es-sidecar-service/integration_test/purge_test.go +++ b/components/es-sidecar-service/integration_test/purge_test.go @@ -32,7 +32,7 @@ func TestPurgeTimeSeriesIndicesByAge(t *testing.T) { fmt.Sprintf("When starting with %v indices and removing older than %v days, %v remain", test.startingCount, test.olderThanDays, test.expectedRemainingCount), func(t *testing.T) { - doPurgeTest(t, TimeSeriesTestIndex, test.startingCount, test.olderThanDays, test.expectedRemainingCount+2) + doPurgeTest(t, TimeSeriesTestIndex, test.startingCount, test.olderThanDays, test.expectedRemainingCount+4) }, ) } diff --git a/components/es-sidecar-service/integration_test/suite_test.go b/components/es-sidecar-service/integration_test/suite_test.go index 826bd09c0c7..c2a3e4b6e4a 100644 --- a/components/es-sidecar-service/integration_test/suite_test.go +++ b/components/es-sidecar-service/integration_test/suite_test.go @@ -79,13 +79,16 @@ func (suite *Suite) DeleteAllIndices() { if err != nil { fmt.Printf("Could not retrieve index list: %s\n", err) } - for i, v := range indices { - if v == ".opendistro_security" { - indices = append(indices[:i], indices[i+1:]...) - break + + indicesToDelete := make([]string, 0) + for _, v := range indices { + if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" { + continue + } else { + indicesToDelete = append(indicesToDelete, v) } } - suite.esClient.DeleteIndex(indices...).Do(context.Background()) + suite.esClient.DeleteIndex(indicesToDelete...).Do(context.Background()) } func (suite Suite) GlobalTeardown() { diff --git a/components/ingest-service/integration_test/suite_test.go b/components/ingest-service/integration_test/suite_test.go index 28ee3a3db24..7a303610633 100644 --- a/components/ingest-service/integration_test/suite_test.go +++ b/components/ingest-service/integration_test/suite_test.go @@ -265,14 +265,16 @@ func (s *Suite) DeleteAllDocuments() { q := elastic.RawStringQuery("{\"match_all\":{}}") // Make sure we clean them all! indices, _ := s.client.IndexNames() - for i, v := range indices { - if v == ".opendistro_security" { - indices = append(indices[:i], indices[i+1:]...) - break + indicesToDelete := make([]string, 0) + for _, v := range indices { + if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" { + continue + } else { + indicesToDelete = append(indicesToDelete, v) } } _, err := s.client.DeleteByQuery(). - Index(indices...). + Index(indicesToDelete...). Query(q). IgnoreUnavailable(true). Refresh("true").