OpenMetadata Operations Command Line Utility (open-metadata#14504)

* Add new OpenMetadataSetup command line application to migrate/deploy and re-index * Add new OpenMetadataSetup command line application to migrate/deploy and re-index * Add deployPipelines option * Add reIndex option * add subcommands * add provision to store upgrade metrics * rename bootstrap script * fix styling checks * Add changelog and store metrics into SERVER_CHANGE_LOG * Cast jsonb * Cast jsonb --------- Co-authored-by: Pere Miquel Brull <[email protected]>
gorgias · Dec 29, 2023 · 0303b44 · 0303b44
1 parent 2f679a9
commit 0303b44
Show file tree

Hide file tree

Showing 18 changed files with 796 additions and 50 deletions.
diff --git a/bootstrap/openmetadata-ops.sh b/bootstrap/openmetadata-ops.sh
@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+#  Copyright 2021 Collate
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+#  http://www.apache.org/licenses/LICENSE-2.0
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+
+# Resolve links - $0 may be a softlink
+PRG="${0}"
+while [ -h "${PRG}" ]; do
+  ls=`ls -ld "${PRG}"`
+  link=`expr "$ls" : '.*-> \(.*\)$'`
+  if expr "$link" : '/.*' > /dev/null; then
+    PRG="$link"
+  else
+    PRG=`dirname "${PRG}"`/"$link"
+  fi
+done
+
+BOOTSTRAP_DIR=`dirname ${PRG}`
+CONFIG_FILE_PATH=${BOOTSTRAP_DIR}/../conf/openmetadata.yaml
+
+# Which java to use
+if [ -z "${JAVA_HOME}" ]; then
+  JAVA="java"
+else
+  JAVA="${JAVA_HOME}/bin/java"
+fi
+
+OPENMETADATA_SETUP_MAIN_CLASS=org.openmetadata.service.util.OpenMetadataOperations
+LIBS_DIR="${BOOTSTRAP_DIR}"/../libs/
+if  [ ${debug} ] ; then
+  echo $LIBS_DIR
+fi
+if [ -d "${LIBS_DIR}" ]; then
+  for file in "${LIBS_DIR}"*.jar;
+  do
+      CLASSPATH="$CLASSPATH":"$file"
+  done
+else
+  CLASSPATH=`mvn -pl openmetadata-service -q exec:exec -Dexec.executable=echo -Dexec.args="%classpath"`
+fi
+
+${JAVA} -Dbootstrap.dir=$BOOTSTRAP_DIR  -cp ${CLASSPATH} ${OPENMETADATA_SETUP_MAIN_CLASS} -c $CONFIG_FILE_PATH "$@"
+
diff --git a/bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v015__update_server_change_log.sql b/bootstrap/sql/migrations/flyway/com.mysql.cj.jdbc.Driver/v015__update_server_change_log.sql
@@ -0,0 +1 @@
+ALTER TABLE SERVER_CHANGE_LOG ADD COLUMN metrics JSON;
diff --git a/bootstrap/sql/migrations/flyway/org.postgresql.Driver/v015__update_server_change_log.sql b/bootstrap/sql/migrations/flyway/org.postgresql.Driver/v015__update_server_change_log.sql
@@ -0,0 +1 @@
+ALTER TABLE SERVER_CHANGE_LOG ADD COLUMN metrics jsonb;
diff --git a/conf/openmetadata.yaml b/conf/openmetadata.yaml
@@ -91,7 +91,19 @@ server:
 logging:
   level: ${LOG_LEVEL:-INFO}
   loggers:
-    io.swagger: DEBUG
+    org.openmetadata.service.util.OpenMetadataSetup:
+      level: INFO
+      appenders:
+        - type: console
+          logFormat: "%msg%n"
+          timeZone: UTC
+        - type: file
+          logFormat: "%level [%d{ISO8601,UTC}] [%t] %logger{5} - %msg%n"
+          currentLogFilename: ./logs/openmetadata-operations.log
+          archivedLogFilenamePattern: ./logs/openmetadata-operations-%d{yyyy-MM-dd}-%i.log.gz
+          archivedFileCount: 7
+          timeZone: UTC
+          maxFileSize: 50MB
   appenders:
     - type: console
       threshold: TRACE
@@ -249,7 +261,7 @@ pipelineServiceClientConfiguration:
   # If we don't need this, set "org.openmetadata.service.clients.pipeline.noop.NoopClient"
   className: ${PIPELINE_SERVICE_CLIENT_CLASS_NAME:-"org.openmetadata.service.clients.pipeline.airflow.AirflowRESTClient"}
   apiEndpoint: ${PIPELINE_SERVICE_CLIENT_ENDPOINT:-http://localhost:8080}
-  metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://localhost:8585/api}
+  metadataApiEndpoint: ${SERVER_HOST_API_URL:-http://host.docker.internal:8585/api}
   ingestionIpInfoEnabled: ${PIPELINE_SERVICE_IP_INFO_ENABLED:-false}
   hostIp: ${PIPELINE_SERVICE_CLIENT_HOST_IP:-""}
   healthCheckInterval: ${PIPELINE_SERVICE_CLIENT_HEALTH_CHECK_INTERVAL:-300}

diff --git a/openmetadata-service/pom.xml b/openmetadata-service/pom.xml
@@ -485,6 +485,10 @@
       <version>1.9.3</version>
       <scope>test</scope>
     </dependency>
+    <dependency>
+      <groupId>info.picocli</groupId>
+      <artifactId>picocli</artifactId>
+    </dependency>
   </dependencies>
 
   <profiles>

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java b/openmetadata-service/src/main/java/org/openmetadata/service/Entity.java
@@ -297,7 +297,7 @@ public static <T extends EntityInterface> void registerEntity(
     ENTITY_LIST.add(entity);
     Collections.sort(ENTITY_LIST);
 
-    LOG.info("Registering entity {} {}", clazz, entity);
+    LOG.debug("Registering entity {} {}", clazz, entity);
   }
 
   public static <T extends EntityTimeSeriesInterface> void registerEntity(
@@ -309,7 +309,7 @@ public static <T extends EntityTimeSeriesInterface> void registerEntity(
     ENTITY_LIST.add(entity);
     Collections.sort(ENTITY_LIST);
 
-    LOG.info("Registering entity time series {} {}", clazz, entity);
+    LOG.debug("Registering entity time series {} {}", clazz, entity);
   }
 
   public static void registerResourcePermissions(

diff --git a/...rvice/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java b/...rvice/src/main/java/org/openmetadata/service/apps/bundles/searchIndex/SearchIndexApp.java
@@ -13,6 +13,7 @@
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 import lombok.Getter;
 import lombok.SneakyThrows;
 import lombok.extern.slf4j.Slf4j;
@@ -54,6 +55,40 @@
 
 @Slf4j
 public class SearchIndexApp extends AbstractNativeApplication {
+
+  private static final String ALL = "all";
+  private static final Set<String> ALL_ENTITIES =
+      Set.of(
+          "table",
+          "dashboard",
+          "topic",
+          "pipeline",
+          "searchIndex",
+          "user",
+          "team",
+          "glossaryTerm",
+          "mlmodel",
+          "tag",
+          "classification",
+          "query",
+          "container",
+          "database",
+          "databaseSchema",
+          "testCase",
+          "testSuite",
+          "chart",
+          "dashboardDataModel",
+          "databaseService",
+          "messagingService",
+          "dashboardService",
+          "pipelineService",
+          "mlmodelService",
+          "searchService",
+          "entityReportData",
+          "webAnalyticEntityViewReportData",
+          "webAnalyticUserActivityReportData",
+          "domain",
+          "storedProcedure");
   private static final String ENTITY_TYPE_ERROR_MSG = "EntityType: %s %n Cause: %s %n Stack: %s";
   private final List<PaginatedEntitiesSource> paginatedEntitiesSources = new ArrayList<>();
   private final List<PaginatedDataInsightSource> paginatedDataInsightSources = new ArrayList<>();
@@ -67,12 +102,14 @@ public class SearchIndexApp extends AbstractNativeApplication {
   @Override
   public void init(App app, CollectionDAO dao, SearchRepository searchRepository) {
     super.init(app, dao, searchRepository);
-
     // request for reindexing
     EventPublisherJob request =
         JsonUtils.convertValue(app.getAppConfiguration(), EventPublisherJob.class)
             .withStats(new Stats())
             .withFailure(new Failure());
+    if (request.getEntities().contains(ALL)) {
+      request.setEntities(ALL_ENTITIES);
+    }
     int totalRecords = getTotalRequestToProcess(request.getEntities(), collectionDAO);
     this.jobData = request;
     this.jobData.setStats(

diff --git a/...src/main/java/org/openmetadata/service/clients/pipeline/PipelineServiceClientFactory.java b/...src/main/java/org/openmetadata/service/clients/pipeline/PipelineServiceClientFactory.java
@@ -35,7 +35,7 @@ public static PipelineServiceClient createPipelineServiceClient(
     }
 
     String pipelineServiceClientClass = config.getClassName();
-    LOG.info("Registering PipelineServiceClient: {}", pipelineServiceClientClass);
+    LOG.debug("Registering PipelineServiceClient: {}", pipelineServiceClientClass);
 
     try {
       pipelineServiceClient =

diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MigrationDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/MigrationDAO.java
@@ -3,13 +3,20 @@
 import static org.openmetadata.service.jdbi3.locator.ConnectionType.MYSQL;
 import static org.openmetadata.service.jdbi3.locator.ConnectionType.POSTGRES;
 
+import java.sql.ResultSet;
+import java.sql.SQLException;
 import java.util.List;
 import java.util.Optional;
+import lombok.Builder;
 import lombok.Getter;
 import lombok.Setter;
+import org.jdbi.v3.core.mapper.RowMapper;
+import org.jdbi.v3.core.statement.StatementContext;
 import org.jdbi.v3.core.statement.StatementException;
 import org.jdbi.v3.sqlobject.SingleValue;
+import org.jdbi.v3.sqlobject.config.RegisterRowMapper;
 import org.jdbi.v3.sqlobject.customizer.Bind;
+import org.jdbi.v3.sqlobject.statement.SqlQuery;
 import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlQuery;
 import org.openmetadata.service.jdbi3.locator.ConnectionAwareSqlUpdate;
 
@@ -53,26 +60,29 @@ String getSqlQuery(@Bind("version") String version, @Bind("checksum") String che
 
   @ConnectionAwareSqlUpdate(
       value =
-          "INSERT INTO SERVER_CHANGE_LOG (version, migrationFileName, checksum, installed_on)"
-              + "VALUES (:version, :migrationFileName, :checksum, CURRENT_TIMESTAMP) "
+          "INSERT INTO SERVER_CHANGE_LOG (version, migrationFileName, checksum, metrics, installed_on)"
+              + "VALUES (:version, :migrationFileName, :checksum, :metrics, CURRENT_TIMESTAMP) "
               + "ON DUPLICATE KEY UPDATE "
               + "migrationFileName = :migrationFileName, "
               + "checksum = :checksum, "
+              + "metrics = :metrics,"
               + "installed_on = CURRENT_TIMESTAMP",
       connectionType = MYSQL)
   @ConnectionAwareSqlUpdate(
       value =
-          "INSERT INTO server_change_log (version, migrationFileName, checksum, installed_on)"
-              + "VALUES (:version, :migrationFileName, :checksum, current_timestamp) "
+          "INSERT INTO server_change_log (version, migrationFileName, checksum, metrics, installed_on)"
+              + "VALUES (:version, :migrationFileName, :checksum, to_jsonb(:metrics::text), current_timestamp) "
               + "ON CONFLICT (version) DO UPDATE SET "
               + "migrationFileName = EXCLUDED.migrationFileName, "
+              + "metrics = to_jsonb(:metrics::text),"
               + "checksum = EXCLUDED.checksum, "
               + "installed_on = EXCLUDED.installed_on",
       connectionType = POSTGRES)
   void upsertServerMigration(
       @Bind("version") String version,
       @Bind("migrationFileName") String migrationFileName,
-      @Bind("checksum") String checksum);
+      @Bind("checksum") String checksum,
+      @Bind("metrics") String metrics);
 
   @ConnectionAwareSqlUpdate(
       value =
@@ -113,11 +123,42 @@ void upsertServerMigrationSQL(
       connectionType = POSTGRES)
   String checkIfQueryPreviouslyRan(@Bind("checksum") String checksum);
 
+  @SqlQuery(
+      "SELECT installed_rank, version, migrationFileName, checksum, installed_on, metrics FROM SERVER_CHANGE_LOG ORDER BY version ASC")
+  @RegisterRowMapper(FromServerChangeLogMapper.class)
+  List<ServerChangeLog> listMetricsFromDBMigrations();
+
   @Getter
   @Setter
   class ServerMigrationSQLTable {
     private String version;
     private String sqlStatement;
     private String checkSum;
   }
+
+  @Getter
+  @Setter
+  @Builder
+  class ServerChangeLog {
+    private Integer installedRank;
+    private String version;
+    private String migrationFileName;
+    private String checksum;
+    private String installedOn;
+    private String metrics;
+  }
+
+  class FromServerChangeLogMapper implements RowMapper<ServerChangeLog> {
+    @Override
+    public ServerChangeLog map(ResultSet rs, StatementContext ctx) throws SQLException {
+      return ServerChangeLog.builder()
+          .installedRank(rs.getInt("installed_rank"))
+          .version(rs.getString("version"))
+          .migrationFileName(rs.getString("migrationFileName"))
+          .checksum(rs.getString("checksum"))
+          .installedOn(rs.getString("installed_on"))
+          .metrics(rs.getString("metrics"))
+          .build();
+    }
+  }
 }
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		ALTER TABLE SERVER_CHANGE_LOG ADD COLUMN metrics JSON;