Upgrade Postgres 17

.studio/common

@@ -362,6 +362,7 @@ function build_all_changed_components() {
   hab pkg install core/coreutils >/dev/null
   mkdir -p /usr/bin
   ln -sv "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env 2>/dev/null
+  cp "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env
 
   pushd /src >/dev/null
     for component in $(./scripts/changed_components.rb)

.studio/deployment-service

@@ -10,6 +10,8 @@ function start_deployment_service {
   local local_manifest=""
   local_manifest=$(find_local_manifest)
 
+  cp "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env
+
   chef-automate dev configure-deployment --config dev/config.toml --manifest-dir "$local_manifest"
 
   restart_gateway_if_up

.studiorc

@@ -159,6 +159,9 @@ DOC
 start_all_services() {
   start_deployment_service
   chef-automate dev deployinate
+
+  cp "$(hab pkg path core/coreutils)/bin/env" /usr/bin/env
+
   if [[ -f "/src/dev/license.jwt" ]]; then
      chef-automate license apply "/src/dev/license.jwt"
   fi

components/automate-deployment/habitat/plan.sh

@@ -32,7 +32,7 @@ pkg_deps=(
   chef/mlsa
   # deployment-service uses the postgres11 client to backup/restore postgresql.
   # we need pg11 because the ha backend uses postgres 11
-  core/postgresql13-client
+  core/postgresql17-client
 )
 
 pkg_bin_dirs=(bin)

components/automate-opensearch/habitat/config/opensearch.yml

@@ -162,6 +162,7 @@ plugins.security.authcz.admin_dn:
 plugins.security.audit.type: internal_opensearch
 plugins.security.enable_snapshot_restore_privilege: true
 plugins.security.check_snapshot_restore_write_privileges: true
+plugins.security.restapi.admin.enabled: true
 plugins.security.restapi.roles_enabled: ["all_access", "security_rest_api_access"]
 plugins.security.system_indices.enabled: true
 plugins.security.system_indices.indices: [".opendistro-alerting-config", ".opendistro-alerting-alert*", ".opendistro-anomaly-results*", ".opendistro-anomaly-detector*", ".opendistro-anomaly-checkpoints", ".opendistro-anomaly-detection-state", ".opendistro-reports-*", ".opendistro-notifications-*", ".opendistro-notebooks", ".opensearch-observability", ".opendistro-asynchronous-search-response*", ".replication-metadata-store"]

components/automate-opensearch/habitat/config/securityconfig/internal_users.yml

@@ -15,6 +15,7 @@ admin:
   reserved: true
   backend_roles:
   - "admin"
+  - "security_rest_api_access"
   description: "Demo admin user"
 
 kibanaserver:

components/automate-opensearch/habitat/hooks/post-run

@@ -2,9 +2,10 @@
 
 exec 2>&1
 
+sleep 80
 # Executing opensearch securityadmin
 echo "Executing opensearch securityadmin"
-hab pkg exec core/bash {{pkg.path}}/os/plugins/opensearch-security/tools/securityadmin.sh -cd {{pkg.svc_config_path}}/securityconfig -icl -key {{pkg.svc_config_path}}/admin-key.pem -cert {{pkg.svc_config_path}}/admin.pem -cacert {{pkg.svc_config_path}}/root-ca.pem -nhnv -h "localhost" -p {{cfg.transport.port}}
+hab pkg exec core/bash {{pkg.path}}/os/plugins/opensearch-security/tools/securityadmin.sh -cd {{pkg.svc_config_path}}/securityconfig -icl -key {{pkg.svc_config_path}}/admin-key.pem -cert {{pkg.svc_config_path}}/admin.pem -cacert {{pkg.svc_config_path}}/root-ca.pem -nhnv -h "localhost" -p {{cfg.network.port}}
 echo "Done !!!! Executing opensearch securityadmin"
 
 source {{pkg.svc_config_path}}/health_check
@@ -62,7 +63,7 @@ curl \
   -H 'Content-Type: application/json'\
   -X PUT "https://${HOST}:{{cfg.network.port}}/_all/_settings?preserve_existing=true" \
   -k \
-  -u admin:admin \
+  --cert {{pkg.svc_config_path}}/admin.pem --key {{pkg.svc_config_path}}/admin-key.pem \
   -d '{
     "index.number_of_replicas": "{{cfg.index.number_of_replicas}}",
     "index.refresh_interval": "{{cfg.index.refresh_interval}}"

components/automate-opensearch/habitat/plan.sh

@@ -5,12 +5,14 @@
 pkg_name="automate-opensearch"
 pkg_description="Wrapper package for core/elasticsearch"
 pkg_origin="chef"
-pkg_version="1.3.19"
+pkg_version="2.18.0"
 pkg_maintainer="Chef Software Inc. <[email protected]>"
 pkg_license=("Chef-MLSA")
 pkg_upstream_url="https://www.chef.io/automate"
-pkg_source="https://artifacts.opensearch.org/releases/bundle/opensearch/1.3.19/opensearch-1.3.19-linux-x64.tar.gz"
-pkg_shasum=af901097211df4c3d2ef75f0c3699452d82a87adac60c240520a3ca03f969595
+pkg_source="https://artifacts.opensearch.org/releases/bundle/opensearch/2.18.0/opensearch-2.18.0-linux-x64.tar.gz"
+pkg_shasum=ae3cb4107b2e0cdbb9b98bb4e5f2f019b736b68e995442c718e459d39ff01df1
+# pkg_source="https://artifacts.opensearch.org/releases/bundle/opensearch/2.11.1/opensearch-2.11.1-linux-x64.tar.gz"
+# pkg_shasum=926eb3f99f592e7518d5dc1e250ae11d1bb45ef766a90222a8df11bb2ee35a9c
 
 
 pkg_build_deps=(
@@ -52,7 +54,20 @@ do_build() {
   return 0
 }
 
+do_after() {
+  if [ ! -f /usr/bin/env ]; then
+    echo "inside the /usr/ib/env"
+    cp $(pkg_path_for core/coreutils)/bin/env /usr/bin/env
+  fi
+
+}
+
 do_install() {
+  if [ ! -f /usr/bin/env ]; then
+    echo "inside the /isr/ib/env"
+    cp $(pkg_path_for core/coreutils)/bin/env /usr/bin/env
+  fi
+
   cd "$HAB_CACHE_SRC_PATH/opensearch-${pkg_version}"
   chown -RL hab:hab ${pkg_prefix}
   mkdir -p "${pkg_prefix}/os"

components/automate-postgresql/habitat/config/functions.sh

@@ -33,7 +33,7 @@ ensure_dir_ownership() {
   fi
   echo "Ensuring proper ownership: $ownership_command"
   $ownership_command
-  chmod 0700 {{pkg.svc_data_path}}/pgdata13
+  chmod 0700 {{pkg.svc_data_path}}/pgdata17
 }
 
 ensure_key_ownership() {

components/automate-postgresql/habitat/config/pg_hba.conf

@@ -12,7 +12,7 @@
 local   all             all                                     md5
 
 # TODO: perhaps we could restrict this to just peers in the hab ring
-hostssl all             all             0.0.0.0/0               cert clientcert=1 map=service-map
+hostssl all             all             0.0.0.0/0               cert clientcert=verify-full map=service-map
 
 # Always allow MD5 from localhost
 host    all             all             127.0.0.1/8             md5

components/automate-postgresql/habitat/config/postgresql.conf

@@ -18,7 +18,7 @@ datestyle = 'iso, mdy'
 
 default_text_search_config = 'pg_catalog.english'
 
-data_directory = '{{pkg.svc_data_path}}/pgdata13'
+data_directory = '{{pkg.svc_data_path}}/pgdata17'
 hba_file = '{{pkg.svc_config_path}}/pg_hba.conf'
 ident_file = '{{pkg.svc_config_path}}/pg_ident.conf'
 

components/automate-postgresql/habitat/hooks/init

@@ -27,25 +27,25 @@ mkdir -p {{pkg.svc_var_path}}/pg_stat_tmp
 
 # Auto-detect pg data in the root of svc_data_path, where older versions of this plan had it
 if [[ -f "{{pkg.svc_data_path}}/PG_VERSION" ]]; then
-  echo "PGDATA detected in the root of the data path ( {{pkg.svc_data_path}} ), relocating it to {{pkg.svc_data_path}}/pgdata13"
-  mkdir -p {{pkg.svc_data_path}}/pgdata13
+  echo "PGDATA detected in the root of the data path ( {{pkg.svc_data_path}} ), relocating it to {{pkg.svc_data_path}}/pgdata17"
+  mkdir -p {{pkg.svc_data_path}}/pgdata17
   # bash extended globbing can cleanly move everything under a subfolder http://www.linuxjournal.com/content/bash-extended-globbing
-  mv {{pkg.svc_data_path}}/!(pgdata13) {{pkg.svc_data_path}}/pgdata13/
-  chmod 0700 {{pkg.svc_data_path}}/pgdata13
+  mv {{pkg.svc_data_path}}/!(pgdata73) {{pkg.svc_data_path}}/pgdata17/
+  chmod 0700 {{pkg.svc_data_path}}/pgdata17
 else
-  mkdir -p {{pkg.svc_data_path}}/pgdata13
+  mkdir -p {{pkg.svc_data_path}}/pgdata17
 fi
 
 mkdir -p {{pkg.svc_data_path}}/archive
 
 ensure_dir_ownership
 ensure_key_ownership
 
-if [[ ! -f "{{pkg.svc_data_path}}/pgdata13/PG_VERSION" ]]; then
+if [[ ! -f "{{pkg.svc_data_path}}/pgdata17/PG_VERSION" ]]; then
   echo " Database does not exist, creating with 'initdb'"
     initdb -U {{cfg.superuser.name}} \
     -E utf8 \
-    -D {{pkg.svc_data_path}}/pgdata13 \
+    -D {{pkg.svc_data_path}}/pgdata17 \
     --locale POSIX \
     --data-checksums
 fi

components/automate-postgresql/habitat/hooks/run

@@ -23,7 +23,7 @@ ensure_dir_ownership
 ensure_key_ownership
 
 echo "Starting PostgreSQL"
-export PGDATA={{pkg.svc_data_path}}/pgdata13
+export PGDATA={{pkg.svc_data_path}}/pgdata17
 
 pidfile="$PGDATA/postmaster.pid"
 if [ -f  "$pidfile" ]; then

components/automate-postgresql/habitat/plan.sh

@@ -4,22 +4,22 @@
 pkg_name="automate-postgresql"
 pkg_description="Wrapper package for core/postgresql"
 pkg_origin="chef"
-pkg_version="13.18.0"
+pkg_version="17.2.0"
 vendor_origin="core"
 pkg_maintainer="Chef Software Inc. <[email protected]>"
 pkg_license=("Chef-MLSA")
 pkg_upstream_url="https://www.chef.io/automate"
 
 # pinned PG version. We need to also update components/automate-cli/cmd/chef-automate/migration_pg.go (line)
-pg_version="13.18"
+pg_version="17.2"
 
 # pinned PG version. We need to also update components/automate-cli/cmd/chef-automate/migration_pg.go (line)
-ident="20241203070217"
+ident="20241203071443"
 
 pkg_deps=(
-  core/coreutils
-  chef/mlsa
-  ${vendor_origin}/postgresql13/${pg_version}/${ident}
+  core/coreutils/9.4/20241017111752
+  chef/mlsa/1.0.1/20241018111404
+  ${vendor_origin}/postgresql17/${pg_version}/${ident}
 )
 
 pkg_exports=(

components/config-mgmt-service/integration_test/suite_test.go

@@ -56,11 +56,11 @@ const (
 // multiple tests, consider putting it here so that we have them available globally
 //
 // This struct holds:
-// * A Ingest backend client, that you can leverate to do all sorts of ingestion.
-//   => Check this for the list of things this client can do:
-//      https://github.com/github.com/chef/automate/components/ingest-service/blob/master/backend/client.go#L1
-// * A Elasticsearch client, that you can use to throw ES queries.
-//   => Docs: https://godoc.org/gopkg.in/olivere/elastic.v5
+//   - A Ingest backend client, that you can leverate to do all sorts of ingestion.
+//     => Check this for the list of things this client can do:
+//     https://github.com/github.com/chef/automate/components/ingest-service/blob/master/backend/client.go#L1
+//   - A Elasticsearch client, that you can use to throw ES queries.
+//     => Docs: https://godoc.org/gopkg.in/olivere/elastic.v5
 type Suite struct {
 	ingest iBackend.Client
 	client *elastic.Client
@@ -128,16 +128,12 @@ func (s *Suite) GlobalTeardown() {
 	}
 
 	indicesToDelete := make([]string, 0)
-	for _, index := range indices {
-		//don't ever delete node run info.. we'll do that after each test when needed
-		if index != mappings.IndexNameNodeRunInfo {
-			indicesToDelete = append(indicesToDelete, index)
-		}
-	}
-	for i, v := range indicesToDelete {
-		if v == ".opendistro_security" {
-			indicesToDelete = append(indicesToDelete[:i], indicesToDelete[i+1:]...)
-			break
+
+	for _, v := range indices {
+		if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" || v != mappings.IndexNameNodeRunInfo {
+			continue
+		} else {
+			indicesToDelete = append(indicesToDelete, v)
 		}
 	}
 	time.Sleep(2 * time.Second)
@@ -221,30 +217,34 @@ func (s *Suite) Indices() []string {
 //
 // You should call this method on every single test as the following example:
 // ```
-//  func TestGrpcFunc(t *testing.T) {
-//    // Here we are ingesting a number of nodes
-//    suite.IngestNodes(nodes)
 //
-//    // Immediately after the ingestion add the hook to clean all documents,
-//    // by using `defer` you will ensure that the next test will have clean
-//    // data regardless if this test passes or fails
-//    defer suite.DeleteAllDocuments()
-//  }
+//	func TestGrpcFunc(t *testing.T) {
+//	  // Here we are ingesting a number of nodes
+//	  suite.IngestNodes(nodes)
+//
+//	  // Immediately after the ingestion add the hook to clean all documents,
+//	  // by using `defer` you will ensure that the next test will have clean
+//	  // data regardless if this test passes or fails
+//	  defer suite.DeleteAllDocuments()
+//	}
+//
 // ```
 func (s *Suite) DeleteAllDocuments() {
 	// ES Query to match all documents
 	q := elastic.RawStringQuery("{\"match_all\":{}}")
 
 	// Make sure we clean them all!
 	indices, _ := s.client.IndexNames()
-	for i, v := range indices {
-		if v == ".opendistro_security" {
-			indices = append(indices[:i], indices[i+1:]...)
-			break
+	indicesToDelete := make([]string, 0)
+	for _, v := range indices {
+		if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" {
+			continue
+		} else {
+			indicesToDelete = append(indicesToDelete, v)
 		}
 	}
 	_, err := s.client.DeleteByQuery().
-		Index(indices...).
+		Index(indicesToDelete...).
 		Query(q).
 		IgnoreUnavailable(true).
 		Refresh("true").

components/es-sidecar-service/integration_test/purge_test.go

@@ -32,7 +32,7 @@ func TestPurgeTimeSeriesIndicesByAge(t *testing.T) {
 			fmt.Sprintf("When starting with %v indices and removing older than %v days, %v remain",
 				test.startingCount, test.olderThanDays, test.expectedRemainingCount),
 			func(t *testing.T) {
-				doPurgeTest(t, TimeSeriesTestIndex, test.startingCount, test.olderThanDays, test.expectedRemainingCount+2)
+				doPurgeTest(t, TimeSeriesTestIndex, test.startingCount, test.olderThanDays, test.expectedRemainingCount+4)
 			},
 		)
 	}

components/es-sidecar-service/integration_test/suite_test.go

@@ -79,13 +79,16 @@ func (suite *Suite) DeleteAllIndices() {
 	if err != nil {
 		fmt.Printf("Could not retrieve index list: %s\n", err)
 	}
-	for i, v := range indices {
-		if v == ".opendistro_security" {
-			indices = append(indices[:i], indices[i+1:]...)
-			break
+
+	indicesToDelete := make([]string, 0)
+	for _, v := range indices {
+		if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" {
+			continue
+		} else {
+			indicesToDelete = append(indicesToDelete, v)
 		}
 	}
-	suite.esClient.DeleteIndex(indices...).Do(context.Background())
+	suite.esClient.DeleteIndex(indicesToDelete...).Do(context.Background())
 }
 
 func (suite Suite) GlobalTeardown() {

components/ingest-service/integration_test/suite_test.go

@@ -265,14 +265,16 @@ func (s *Suite) DeleteAllDocuments() {
 	q := elastic.RawStringQuery("{\"match_all\":{}}")
 	// Make sure we clean them all!
 	indices, _ := s.client.IndexNames()
-	for i, v := range indices {
-		if v == ".opendistro_security" {
-			indices = append(indices[:i], indices[i+1:]...)
-			break
+	indicesToDelete := make([]string, 0)
+	for _, v := range indices {
+		if v == ".plugins-ml-config" || v == ".opensearch-observability" || v == ".opendistro_security" {
+			continue
+		} else {
+			indicesToDelete = append(indicesToDelete, v)
 		}
 	}
 	_, err := s.client.DeleteByQuery().
-		Index(indices...).
+		Index(indicesToDelete...).
 		Query(q).
 		IgnoreUnavailable(true).
 		Refresh("true").

components/pg-sidecar-service/habitat/plan.sh

@@ -22,8 +22,8 @@ pkg_deps=(
   chef/automate-platform-tools
   # Pin postgresql, sqitch_pg, and postgresql client until automate-postgresql
   # is updated.
-  core/postgresql13/13.18 # for psql and pg_dump
-  core/postgresql13-client/13.18
+  core/postgresql17/17.2 # for psql and pg_dump
+  core/postgresql17-client/17.2
   core/sqitch_pg
 )
 pkg_bin_dirs=(bin)