DAT-18792: added failing tblProperties filtering, wrapping of tblProp…

…erties in quotes, createTable changetype namespace as ext.
liquibase · Oct 18, 2024 · f7ef735 · f7ef735
1 parent aeab0cd
commit f7ef735
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 6 deletions.
diff --git a/src/main/java/liquibase/ext/databricks/change/createTable/CreateTableChangeDatabricks.java b/src/main/java/liquibase/ext/databricks/change/createTable/CreateTableChangeDatabricks.java
@@ -6,7 +6,7 @@
 import liquibase.database.Database;
 import liquibase.exception.ValidationErrors;
 import liquibase.ext.databricks.database.DatabricksDatabase;
-import liquibase.ext.databricks.parser.NamespaceDetailsDatabricks;
+import liquibase.parser.core.xml.StandardNamespaceDetails;
 import liquibase.servicelocator.PrioritizedService;
 import liquibase.statement.core.CreateTableStatement;
 import lombok.Setter;
@@ -72,4 +72,9 @@ public ExtendedTableProperties getExtendedTableProperties() {
         return extendedTableProperties;
     }
 
+    @Override
+    public String getSerializedObjectNamespace() {
+        return StandardNamespaceDetails.GENERIC_EXTENSION_XSD;
+    }
+
 }
diff --git a/src/main/java/liquibase/ext/databricks/snapshot/jvm/TableSnapshotGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/snapshot/jvm/TableSnapshotGeneratorDatabricks.java
@@ -11,6 +11,7 @@
 import liquibase.structure.DatabaseObject;
 import liquibase.structure.core.Table;
 
+import java.util.Arrays;
 import java.util.List;
 import java.util.Map;
 import java.util.regex.Pattern;
@@ -22,6 +23,12 @@ public class TableSnapshotGeneratorDatabricks extends TableSnapshotGenerator {
     private static final String TBL_PROPERTIES = "tblProperties";
     private static final String CLUSTER_COLUMNS = "clusteringColumns";
     private static final String DETAILED_TABLE_INFORMATION_NODE = "# Detailed Table Information";
+    private static final List<String> TBL_PROPERTIES_STOP_LIST = Arrays.asList(
+            "delta.columnMapping.maxColumnId",
+            "delta.rowTracking.materializedRowCommitVersionColumnName",
+            "delta.rowTracking.materializedRowIdColumnName",
+            "delta.feature.clustering"
+    );
 
     @Override
     public int getPriority(Class<? extends DatabaseObject> objectType, Database database) {
@@ -54,7 +61,8 @@ protected DatabaseObject snapshotObject(DatabaseObject example, DatabaseSnapshot
             }
             Map<String, String> tblProperties = getTblPropertiesMap(database, example.getName());
             if (tblProperties.containsKey(CLUSTER_COLUMNS)) {
-                // removing clusterColumns, as clusterColumns tblProperty is not allowed in create/alter table statements
+                // removing clusterColumns and other properties which are not allowed in create/alter table statements
+                TBL_PROPERTIES_STOP_LIST.forEach(tblProperties::remove);
                 table.setAttribute(CLUSTER_COLUMNS, sanitizeClusterColumns(tblProperties.remove(CLUSTER_COLUMNS)));
             }
             table.setAttribute(TBL_PROPERTIES, getTblPropertiesString(tblProperties));
@@ -80,8 +88,8 @@ private String getTblPropertiesString(Map<String, String> propertiesMap) {
         propertiesMap.entrySet()
                 .stream()
                 .sorted(Map.Entry.comparingByKey())
-                .forEach(entry -> csvString.append(entry.getKey()).append("=").append(entry.getValue()).append(","));
-        return csvString.toString().replaceAll(",$", "");
+                .forEach(entry -> csvString.append("'").append(entry.getKey()).append("'='").append(entry.getValue()).append("', "));
+        return csvString.toString().replaceAll(", $", "");
     }
 
     private String sanitizeClusterColumns(String clusterColumnProperty) {

diff --git a/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createTable.json b/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createTable.json
@@ -5,14 +5,14 @@
         {
           "table": {
             "name": "test_table",
-            "tblProperties" : "delta.checkpoint.writeStatsAsJson=false,delta.checkpoint.writeStatsAsStruct=true,delta.columnMapping.maxColumnId=2,delta.columnMapping.mode=name,delta.enableDeletionVectors=true,delta.feature.allowColumnDefaults=supported,delta.feature.columnMapping=supported,delta.feature.deletionVectors=supported,delta.feature.invariants=supported,delta.minReaderVersion=3,delta.minWriterVersion=7"
+            "tblProperties" : "'delta.checkpoint.writeStatsAsJson'='false', 'delta.checkpoint.writeStatsAsStruct'='true', 'delta.columnMapping.maxColumnId'='2', 'delta.columnMapping.mode'='name', 'delta.enableDeletionVectors'='true', 'delta.feature.allowColumnDefaults'='supported', 'delta.feature.columnMapping'='supported', 'delta.feature.deletionVectors'='supported', 'delta.feature.invariants'='supported', 'delta.minReaderVersion'='3', 'delta.minWriterVersion'='7'"
           }
         },
         {
           "table": {
             "name": "test_table_properties",
             "Location" : "s3://databricks-th/test_table_properties",
-            "tblProperties" : "delta.checkpoint.writeStatsAsJson=false,delta.checkpoint.writeStatsAsStruct=true,delta.enableDeletionVectors=true,delta.feature.deletionVectors=supported,delta.feature.invariants=supported,delta.minReaderVersion=3,delta.minWriterVersion=7,this.is.my.key=12,this.is.my.key2=true"
+            "tblProperties" : "'delta.checkpoint.writeStatsAsJson'='false', 'delta.checkpoint.writeStatsAsStruct'='true', 'delta.enableDeletionVectors'='true', 'delta.feature.deletionVectors'='supported', 'delta.feature.invariants'='supported', 'delta.minReaderVersion'='3', 'delta.minWriterVersion'='7', 'this.is.my.key'='12', 'this.is.my.key2'='true'"
           }
         }
       ],