From cbefac190dd29ef06c4b8f76164932a89a0068cf Mon Sep 17 00:00:00 2001 From: filipe Date: Mon, 9 Sep 2024 13:57:49 -0300 Subject: [PATCH 1/4] chore: Override the default implementation to ARRAY, MAP and STRUCT complex types as Liquibase core does not know how to handle values between <> in the data type. --- .../ArrayIntegerDataTypeDatabricks.java | 31 ------------- .../ArrayStringDataTypeDatabricks.java | 31 ------------- .../ColumnSnapshotGeneratorDatabricks.java | 46 +++++++++++++++++++ .../liquibase.datatype.LiquibaseDataType | 4 +- .../liquibase.snapshot.SnapshotGenerator | 2 + 5 files changed, 49 insertions(+), 65 deletions(-) delete mode 100644 src/main/java/liquibase/ext/databricks/datatype/ArrayIntegerDataTypeDatabricks.java delete mode 100644 src/main/java/liquibase/ext/databricks/datatype/ArrayStringDataTypeDatabricks.java create mode 100644 src/main/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricks.java diff --git a/src/main/java/liquibase/ext/databricks/datatype/ArrayIntegerDataTypeDatabricks.java b/src/main/java/liquibase/ext/databricks/datatype/ArrayIntegerDataTypeDatabricks.java deleted file mode 100644 index ab8510c8..00000000 --- a/src/main/java/liquibase/ext/databricks/datatype/ArrayIntegerDataTypeDatabricks.java +++ /dev/null @@ -1,31 +0,0 @@ -package liquibase.ext.databricks.datatype; - -import liquibase.change.core.LoadDataChange; -import liquibase.database.Database; -import liquibase.datatype.DataTypeInfo; -import liquibase.datatype.DatabaseDataType; -import liquibase.datatype.LiquibaseDataType; -import liquibase.ext.databricks.database.DatabricksDatabase; -import liquibase.servicelocator.PrioritizedService; - -@DataTypeInfo(name = "array", minParameters = 0, maxParameters = 0, priority = PrioritizedService.PRIORITY_DATABASE) -public class ArrayIntegerDataTypeDatabricks extends LiquibaseDataType { - - @Override - public DatabaseDataType toDatabaseDataType(Database database) { - if (database instanceof DatabricksDatabase) { - return new DatabaseDataType("ARARY"); - } - return super.toDatabaseDataType(database); - } - - @Override - public boolean supports(Database database) { - return database instanceof DatabricksDatabase; - } - - @Override - public LoadDataChange.LOAD_DATA_TYPE getLoadTypeName() { - return LoadDataChange.LOAD_DATA_TYPE.STRING; - } -} \ No newline at end of file diff --git a/src/main/java/liquibase/ext/databricks/datatype/ArrayStringDataTypeDatabricks.java b/src/main/java/liquibase/ext/databricks/datatype/ArrayStringDataTypeDatabricks.java deleted file mode 100644 index 1c6bed47..00000000 --- a/src/main/java/liquibase/ext/databricks/datatype/ArrayStringDataTypeDatabricks.java +++ /dev/null @@ -1,31 +0,0 @@ -package liquibase.ext.databricks.datatype; - -import liquibase.change.core.LoadDataChange; -import liquibase.database.Database; -import liquibase.datatype.DataTypeInfo; -import liquibase.datatype.DatabaseDataType; -import liquibase.datatype.LiquibaseDataType; -import liquibase.ext.databricks.database.DatabricksDatabase; -import liquibase.servicelocator.PrioritizedService; - -@DataTypeInfo(name = "array", minParameters = 0, maxParameters = 0, priority = PrioritizedService.PRIORITY_DATABASE) -public class ArrayStringDataTypeDatabricks extends LiquibaseDataType { - - @Override - public DatabaseDataType toDatabaseDataType(Database database) { - if (database instanceof DatabricksDatabase) { - return new DatabaseDataType("ARRAY"); - } - return super.toDatabaseDataType(database); - } - - @Override - public boolean supports(Database database) { - return database instanceof DatabricksDatabase; - } - - @Override - public LoadDataChange.LOAD_DATA_TYPE getLoadTypeName() { - return LoadDataChange.LOAD_DATA_TYPE.STRING; - } -} \ No newline at end of file diff --git a/src/main/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricks.java new file mode 100644 index 00000000..e886c477 --- /dev/null +++ b/src/main/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricks.java @@ -0,0 +1,46 @@ +package liquibase.ext.databricks.snapshot.jvm; + +import liquibase.database.Database; +import liquibase.exception.DatabaseException; +import liquibase.ext.databricks.database.DatabricksDatabase; +import liquibase.snapshot.CachedRow; +import liquibase.snapshot.SnapshotGenerator; +import liquibase.snapshot.jvm.ColumnSnapshotGenerator; +import liquibase.structure.DatabaseObject; +import liquibase.structure.core.Column; +import liquibase.structure.core.DataType; + +public class ColumnSnapshotGeneratorDatabricks extends ColumnSnapshotGenerator { + + @Override + public int getPriority(Class objectType, Database database) { + if (database instanceof DatabricksDatabase) { + return super.getPriority(objectType, database) + PRIORITY_DATABASE; + } else { + return PRIORITY_NONE; + } + } + + @Override + public Class[] replaces() { + return new Class[] { ColumnSnapshotGenerator.class }; + } + + /** + * Override the default implementation to ARRAY, MAP and STRUCT complex types as + * Liquibase core does not know how to handle values between <> in the data type. + */ + @Override + protected DataType readDataType(CachedRow columnMetadataResultSet, Column column, Database database) throws DatabaseException { + String dataType = (String) columnMetadataResultSet.get("TYPE_NAME"); + if (dataType != null && database instanceof DatabricksDatabase + && (dataType.toUpperCase().startsWith("ARRAY") + || dataType.toUpperCase().startsWith("MAP") + || dataType.toUpperCase().startsWith("STRUCT"))) { + DataType type = new DataType(dataType); + type.setDataTypeId(columnMetadataResultSet.getInt("DATA_TYPE")); + return type; + } + return super.readDataType(columnMetadataResultSet, column, database); + } +} diff --git a/src/main/resources/META-INF/services/liquibase.datatype.LiquibaseDataType b/src/main/resources/META-INF/services/liquibase.datatype.LiquibaseDataType index eeed1de7..12526d79 100644 --- a/src/main/resources/META-INF/services/liquibase.datatype.LiquibaseDataType +++ b/src/main/resources/META-INF/services/liquibase.datatype.LiquibaseDataType @@ -1,5 +1,3 @@ -liquibase.ext.databricks.datatype.ArrayIntegerDataTypeDatabricks -liquibase.ext.databricks.datatype.ArrayStringDataTypeDatabricks liquibase.ext.databricks.datatype.BigintDatatypeDatabricks liquibase.ext.databricks.datatype.BinaryDataTypeDatabricks liquibase.ext.databricks.datatype.BooleanDatatypeDatabricks @@ -10,4 +8,4 @@ liquibase.ext.databricks.datatype.IntegerDatatypeDatabricks liquibase.ext.databricks.datatype.SmallintDatatypeDatabricks liquibase.ext.databricks.datatype.StringDatatypeDatabricks liquibase.ext.databricks.datatype.TimestampDatatypeDatabricks -liquibase.ext.databricks.datatype.TinyintDatatypeDatabricks \ No newline at end of file +liquibase.ext.databricks.datatype.TinyintDatatypeDatabricks diff --git a/src/main/resources/META-INF/services/liquibase.snapshot.SnapshotGenerator b/src/main/resources/META-INF/services/liquibase.snapshot.SnapshotGenerator index 36c1c6f8..e92aa426 100644 --- a/src/main/resources/META-INF/services/liquibase.snapshot.SnapshotGenerator +++ b/src/main/resources/META-INF/services/liquibase.snapshot.SnapshotGenerator @@ -4,3 +4,5 @@ liquibase.ext.databricks.snapshot.jvm.ForeignKeySnapshotGeneratorDatabricks liquibase.ext.databricks.snapshot.jvm.UniqueConstraintSnapshotGeneratorDatabricks liquibase.ext.databricks.snapshot.jvm.IndexSnapshotGeneratorDatabricks liquibase.ext.databricks.snapshot.jvm.ViewSnapshotGeneratorDatabricks +liquibase.ext.databricks.snapshot.jvm.ColumnSnapshotGeneratorDatabricks + From 59e2042d4857470389f55f2ce981c4928f611d22 Mon Sep 17 00:00:00 2001 From: filipe Date: Mon, 9 Sep 2024 14:32:17 -0300 Subject: [PATCH 2/4] chore: adding tests. --- .../databricks/createComplexTypesTable.xml | 15 +++++++++++++++ .../databricks/createComplexTypesTable.json | 2 ++ .../databricks/createComplexTypesTable.sql | 1 + 3 files changed, 18 insertions(+) create mode 100644 src/test/resources/liquibase/harness/change/changelogs/databricks/createComplexTypesTable.xml create mode 100644 src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createComplexTypesTable.json create mode 100644 src/test/resources/liquibase/harness/change/expectedSql/databricks/createComplexTypesTable.sql diff --git a/src/test/resources/liquibase/harness/change/changelogs/databricks/createComplexTypesTable.xml b/src/test/resources/liquibase/harness/change/changelogs/databricks/createComplexTypesTable.xml new file mode 100644 index 00000000..89d067b9 --- /dev/null +++ b/src/test/resources/liquibase/harness/change/changelogs/databricks/createComplexTypesTable.xml @@ -0,0 +1,15 @@ + + + + + + + + + + + + diff --git a/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createComplexTypesTable.json b/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createComplexTypesTable.json new file mode 100644 index 00000000..2c63c085 --- /dev/null +++ b/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createComplexTypesTable.json @@ -0,0 +1,2 @@ +{ +} diff --git a/src/test/resources/liquibase/harness/change/expectedSql/databricks/createComplexTypesTable.sql b/src/test/resources/liquibase/harness/change/expectedSql/databricks/createComplexTypesTable.sql new file mode 100644 index 00000000..327d384d --- /dev/null +++ b/src/test/resources/liquibase/harness/change/expectedSql/databricks/createComplexTypesTable.sql @@ -0,0 +1 @@ +CREATE TABLE main.filipe.test_table_complex_types (my_arrs ARRAY, my_arrbi ARRAY, my_map MAP, my_struct STRUCT) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name', 'delta.enableDeletionVectors' = true) From 0655eed190a997e36059df33e8b95b745e90007c Mon Sep 17 00:00:00 2001 From: filipe Date: Mon, 9 Sep 2024 16:34:47 -0300 Subject: [PATCH 3/4] chore: fix tests --- .../change/expectedSql/databricks/createComplexTypesTable.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/test/resources/liquibase/harness/change/expectedSql/databricks/createComplexTypesTable.sql b/src/test/resources/liquibase/harness/change/expectedSql/databricks/createComplexTypesTable.sql index 327d384d..aecba1e9 100644 --- a/src/test/resources/liquibase/harness/change/expectedSql/databricks/createComplexTypesTable.sql +++ b/src/test/resources/liquibase/harness/change/expectedSql/databricks/createComplexTypesTable.sql @@ -1 +1 @@ -CREATE TABLE main.filipe.test_table_complex_types (my_arrs ARRAY, my_arrbi ARRAY, my_map MAP, my_struct STRUCT) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name', 'delta.enableDeletionVectors' = true) +CREATE TABLE main.liquibase_harness_test_ds.test_table_complex_types (my_arrs ARRAY, my_arrbi ARRAY, my_map MAP, my_struct STRUCT) USING delta TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'supported', 'delta.columnMapping.mode' = 'name', 'delta.enableDeletionVectors' = true) From d44d6ce2af3c4ae04723a7066ed9a729eafb382d Mon Sep 17 00:00:00 2001 From: KushnirykOleh Date: Wed, 11 Sep 2024 15:57:23 +0300 Subject: [PATCH 4/4] added expected json data --- .../databricks/createComplexTypesTable.json | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createComplexTypesTable.json b/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createComplexTypesTable.json index 2c63c085..41b20307 100644 --- a/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createComplexTypesTable.json +++ b/src/test/resources/liquibase/harness/change/expectedSnapshot/databricks/createComplexTypesTable.json @@ -1,2 +1,47 @@ { -} + "snapshot": { + "objects": { + "liquibase.structure.core.Table": [ + { + "table": { + "name": "test_table_complex_types" + } + } + ], + "liquibase.structure.core.Column": [ + { + "column": { + "name": "my_arrs", + "type": { + "typeName": "ARRAY" + } + } + }, + { + "column": { + "name": "my_arrbi", + "type": { + "typeName": "ARRAY" + } + } + }, + { + "column": { + "name": "my_map", + "type": { + "typeName": "MAP" + } + } + }, + { + "column": { + "name": "my_struct", + "type": { + "typeName": "STRUCT" + } + } + } + ] + } + } +} \ No newline at end of file