From f7ef7358b7b1d2c5be7ace50690cd8d091e8f39b Mon Sep 17 00:00:00 2001 From: Mykhailo Savchenko Date: Fri, 18 Oct 2024 18:10:21 +0300 Subject: [PATCH] DAT-18792: added failing tblProperties filtering, wrapping of tblProperties in quotes, createTable changetype namespace as ext. --- .../createTable/CreateTableChangeDatabricks.java | 7 ++++++- .../jvm/TableSnapshotGeneratorDatabricks.java | 14 +++++++++++--- .../expectedSnapshot/databricks/createTable.json | 4 ++-- 3 files changed, 19 insertions(+), 6 deletions(-) diff --git a/src/main/java/liquibase/ext/databricks/change/createTable/CreateTableChangeDatabricks.java b/src/main/java/liquibase/ext/databricks/change/createTable/CreateTableChangeDatabricks.java index cd486ebe..bed29359 100644 --- a/src/main/java/liquibase/ext/databricks/change/createTable/CreateTableChangeDatabricks.java +++ b/src/main/java/liquibase/ext/databricks/change/createTable/CreateTableChangeDatabricks.java @@ -6,7 +6,7 @@ import liquibase.database.Database; import liquibase.exception.ValidationErrors; import liquibase.ext.databricks.database.DatabricksDatabase; -import liquibase.ext.databricks.parser.NamespaceDetailsDatabricks; +import liquibase.parser.core.xml.StandardNamespaceDetails; import liquibase.servicelocator.PrioritizedService; import liquibase.statement.core.CreateTableStatement; import lombok.Setter; @@ -72,4 +72,9 @@ public ExtendedTableProperties getExtendedTableProperties() { return extendedTableProperties; } + @Override + public String getSerializedObjectNamespace() { + return StandardNamespaceDetails.GENERIC_EXTENSION_XSD; + } + } diff --git a/src/main/java/liquibase/ext/databricks/snapshot/jvm/TableSnapshotGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/snapshot/jvm/TableSnapshotGeneratorDatabricks.java index c004945b..577c644b 100644 --- a/src/main/java/liquibase/ext/databricks/snapshot/jvm/TableSnapshotGeneratorDatabricks.java +++ b/src/main/java/liquibase/ext/databricks/snapshot/jvm/TableSnapshotGeneratorDatabricks.java @@ -11,6 +11,7 @@ import liquibase.structure.DatabaseObject; import liquibase.structure.core.Table; +import java.util.Arrays; import java.util.List; import java.util.Map; import java.util.regex.Pattern; @@ -22,6 +23,12 @@ public class TableSnapshotGeneratorDatabricks extends TableSnapshotGenerator { private static final String TBL_PROPERTIES = "tblProperties"; private static final String CLUSTER_COLUMNS = "clusteringColumns"; private static final String DETAILED_TABLE_INFORMATION_NODE = "# Detailed Table Information"; + private static final List TBL_PROPERTIES_STOP_LIST = Arrays.asList( + "delta.columnMapping.maxColumnId", + "delta.rowTracking.materializedRowCommitVersionColumnName", + "delta.rowTracking.materializedRowIdColumnName", + "delta.feature.clustering" + ); @Override public int getPriority(Class objectType, Database database) { @@ -54,7 +61,8 @@ protected DatabaseObject snapshotObject(DatabaseObject example, DatabaseSnapshot } Map tblProperties = getTblPropertiesMap(database, example.getName()); if (tblProperties.containsKey(CLUSTER_COLUMNS)) { - // removing clusterColumns, as clusterColumns tblProperty is not allowed in create/alter table statements + // removing clusterColumns and other properties which are not allowed in create/alter table statements + TBL_PROPERTIES_STOP_LIST.forEach(tblProperties::remove); table.setAttribute(CLUSTER_COLUMNS, sanitizeClusterColumns(tblProperties.remove(CLUSTER_COLUMNS))); } table.setAttribute(TBL_PROPERTIES, getTblPropertiesString(tblProperties)); @@ -80,8 +88,8 @@ private String getTblPropertiesString(Map propertiesMap) { propertiesMap.entrySet() .stream() .sorted(Map.Entry.comparingByKey()) - .forEach(entry -> csvString.append(entry.getKey()).append("=").append(entry.getValue()).append(",")); - return csvString.toString().replaceAll(",$", ""); + .forEach(entry -> csvString.append("'").append(entry.getKey()).append("'='").append(entry.getValue()).append("', ")); + return csvString.toString().replaceAll(", $", ""); } private String sanitizeClusterColumns(String clusterColumnProperty) { diff --git a/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createTable.json b/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createTable.json index 564b88f6..ffa8b7db 100644 --- a/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createTable.json +++ b/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createTable.json @@ -5,14 +5,14 @@ { "table": { "name": "test_table", - "tblProperties" : "delta.checkpoint.writeStatsAsJson=false,delta.checkpoint.writeStatsAsStruct=true,delta.columnMapping.maxColumnId=2,delta.columnMapping.mode=name,delta.enableDeletionVectors=true,delta.feature.allowColumnDefaults=supported,delta.feature.columnMapping=supported,delta.feature.deletionVectors=supported,delta.feature.invariants=supported,delta.minReaderVersion=3,delta.minWriterVersion=7" + "tblProperties" : "'delta.checkpoint.writeStatsAsJson'='false', 'delta.checkpoint.writeStatsAsStruct'='true', 'delta.columnMapping.maxColumnId'='2', 'delta.columnMapping.mode'='name', 'delta.enableDeletionVectors'='true', 'delta.feature.allowColumnDefaults'='supported', 'delta.feature.columnMapping'='supported', 'delta.feature.deletionVectors'='supported', 'delta.feature.invariants'='supported', 'delta.minReaderVersion'='3', 'delta.minWriterVersion'='7'" } }, { "table": { "name": "test_table_properties", "Location" : "s3://databricks-th/test_table_properties", - "tblProperties" : "delta.checkpoint.writeStatsAsJson=false,delta.checkpoint.writeStatsAsStruct=true,delta.enableDeletionVectors=true,delta.feature.deletionVectors=supported,delta.feature.invariants=supported,delta.minReaderVersion=3,delta.minWriterVersion=7,this.is.my.key=12,this.is.my.key2=true" + "tblProperties" : "'delta.checkpoint.writeStatsAsJson'='false', 'delta.checkpoint.writeStatsAsStruct'='true', 'delta.enableDeletionVectors'='true', 'delta.feature.deletionVectors'='supported', 'delta.feature.invariants'='supported', 'delta.minReaderVersion'='3', 'delta.minWriterVersion'='7', 'this.is.my.key'='12', 'this.is.my.key2'='true'" } } ],