diff --git a/src/main/java/liquibase/ext/databricks/diff/output/changelog/MissingColumnChangeGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/diff/output/changelog/MissingColumnChangeGeneratorDatabricks.java new file mode 100644 index 00000000..7a64c256 --- /dev/null +++ b/src/main/java/liquibase/ext/databricks/diff/output/changelog/MissingColumnChangeGeneratorDatabricks.java @@ -0,0 +1,157 @@ +package liquibase.ext.databricks.diff.output.changelog; + +import liquibase.change.AddColumnConfig; +import liquibase.change.Change; +import liquibase.change.ConstraintsConfig; +import liquibase.change.core.AddColumnChange; +import liquibase.change.core.AddDefaultValueChange; +import liquibase.change.core.AddNotNullConstraintChange; +import liquibase.database.Database; +import liquibase.diff.output.DiffOutputControl; +import liquibase.diff.output.changelog.ChangeGeneratorChain; +import liquibase.diff.output.changelog.core.MissingColumnChangeGenerator; +import liquibase.ext.databricks.database.DatabricksDatabase; +import liquibase.statement.DatabaseFunction; +import liquibase.structure.DatabaseObject; +import liquibase.structure.core.Column; +import org.apache.commons.lang3.ObjectUtils; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; +import java.util.Objects; +import java.util.Optional; + +/** + * Custom diff change generator for Databricks + */ +public class MissingColumnChangeGeneratorDatabricks extends MissingColumnChangeGenerator { + + @Override + public int getPriority(Class objectType, Database database) { + if (database instanceof DatabricksDatabase && super.getPriority(objectType, database) > PRIORITY_NONE) { + return PRIORITY_DATABASE; + } + return PRIORITY_NONE; + } + + @Override + public Change[] fixMissing(DatabaseObject missingObject, DiffOutputControl control, Database referenceDatabase, Database comparisonDatabase, ChangeGeneratorChain chain) { + Change[] changes = super.fixMissing(missingObject, control, referenceDatabase, comparisonDatabase, chain); + changes = handleMissingColumnConstraints((Column) missingObject, control, changes); + return changes; + } + + private Change[] handleMissingColumnConstraints(Column column, DiffOutputControl control, Change[] changes) { + Optional addColumnOptional = Arrays.stream(changes) + .filter(change -> isCurrentColumnChange(change, column, control)) + .map(AddColumnChange.class::cast).findFirst(); + if(addColumnOptional.isPresent()) { + AddColumnChange addColumnChange = addColumnOptional.get(); + changes = splitAddColumnChange(column, control, changes, addColumnChange); + } + return changes; + } + + private Change[] splitAddColumnChange(Column column, DiffOutputControl control, Change[] changes, AddColumnChange addColumnChange) { + List changeList = new ArrayList<>(Arrays.asList(changes)); + AddColumnConfig addColumnConfig = addColumnChange.getColumns().get(0); + if(addColumnConfig.getDefaultValue() != null || addColumnConfig.getDefaultValueComputed() != null) { + AddDefaultValueChange addDefaultValueChange = handleDefaultValue(column, control, addColumnChange); + changeList.add(addDefaultValueChange); + } + if(addColumnConfig.getConstraints() != null && Objects.equals(addColumnConfig.getConstraints().isNullable(), Boolean.FALSE)) { + AddNotNullConstraintChange addNotNullConstraintChange = handleNotNull(column, control, addColumnChange); + changeList.add(addNotNullConstraintChange); + } + if(constraintsAreEmpty(addColumnConfig, addColumnConfig.getConstraints())) { + addColumnConfig.setConstraints(null); + } + changes = changeList.toArray(new Change[0]); + return changes; + } + + private AddDefaultValueChange handleDefaultValue(Column column, DiffOutputControl control, AddColumnChange addColumnChange) { + AddColumnConfig addColumnConfig = addColumnChange.getColumns().get(0); + String defaultValue = addColumnConfig.getDefaultValue(); + DatabaseFunction defaultValueComputed = addColumnConfig.getDefaultValueComputed(); + String columnDataType = addColumnConfig.getType(); + addColumnConfig.setDefaultValue(null); + addColumnConfig.setDefaultValueComputed(null); + addColumnConfig.setComputed(null); + AddDefaultValueChange addDefaultValueChange = new AddDefaultValueChange(); + if (control.getIncludeCatalog()) { + addDefaultValueChange.setCatalogName(column.getRelation().getSchema().getCatalog().getName()); + } + if (control.getIncludeSchema()) { + addDefaultValueChange.setSchemaName(column.getRelation().getSchema().getName()); + } + addDefaultValueChange.setTableName(column.getRelation().getName()); + addDefaultValueChange.setColumnName(column.getName()); + addDefaultValueChange.setColumnDataType(columnDataType); + + if (defaultValueComputed != null) { + addDefaultValueChange.setDefaultValueComputed(defaultValueComputed); + } else { + addDefaultValueChange.setDefaultValue(defaultValue); + } + addDefaultValueChange.setDefaultValueConstraintName(column.getDefaultValueConstraintName()); + return addDefaultValueChange; + } + + private AddNotNullConstraintChange handleNotNull(Column column, DiffOutputControl control, AddColumnChange addColumnChange) { + AddColumnConfig addColumnConfig = addColumnChange.getColumns().get(0); + ConstraintsConfig constraints = addColumnConfig.getConstraints(); + constraints.setNullable((Boolean) null); + constraints.setNullable((String) null); + constraints.setNotNullConstraintName(null); + AddNotNullConstraintChange addNotNullConstraintChange = createAddNotNullConstraintChange(addColumnConfig, constraints); + if (control.getIncludeCatalog()) { + addNotNullConstraintChange.setCatalogName(column.getRelation().getSchema().getCatalog().getName()); + } + if (control.getIncludeSchema()) { + addNotNullConstraintChange.setSchemaName(column.getRelation().getSchema().getName()); + } + addNotNullConstraintChange.setTableName(column.getRelation().getName()); + return addNotNullConstraintChange; + } + + private AddNotNullConstraintChange createAddNotNullConstraintChange(AddColumnConfig column, ConstraintsConfig constraints) { + AddNotNullConstraintChange addNotNullConstraintChange = new AddNotNullConstraintChange(); + addNotNullConstraintChange.setColumnName(column.getName()); + addNotNullConstraintChange.setColumnDataType(column.getType()); + addNotNullConstraintChange.setValidate(constraints.getValidateNullable()); + addNotNullConstraintChange.setConstraintName(constraints.getNotNullConstraintName()); + return addNotNullConstraintChange; + } + + /** + * We perform reversed checks that were used in the + * {@link liquibase.change.core.AddColumnChange#generateStatements(Database)} + * to make sure there won't be empty constraints generated in generated changelog files. + * */ + boolean constraintsAreEmpty(AddColumnConfig column, ConstraintsConfig constraints) { + if(constraints != null) { + return ObjectUtils.allNull(constraints.isNullable(), constraints.isUnique(), constraints.isPrimaryKey(), + column.isAutoIncrement(), constraints.getReferences(), constraints.getReferencedColumnNames(), + constraints.getReferencedTableName()); + } + return column.isAutoIncrement() != null && !column.isAutoIncrement(); + } + + private boolean isCurrentColumnChange(Change change, Column currentColumn, DiffOutputControl control) { + if(change instanceof AddColumnChange) { + AddColumnChange addColumnChange = ((AddColumnChange) change); + AddColumnConfig addColumnConfig = addColumnChange.getColumns().get(0); + boolean columnNameEqual = addColumnConfig.getName().equals(currentColumn.getName()); + boolean tableNameEqual = addColumnChange.getTableName().equals(currentColumn.getRelation().getName()); + boolean schemaNameEqual = !control.getIncludeSchema() || + Objects.equals(addColumnChange.getSchemaName(), currentColumn.getRelation().getSchema().getName()); + boolean catalogNameEqual = !control.getIncludeCatalog() || + Objects.equals(addColumnChange.getCatalogName(), currentColumn.getRelation().getSchema().getCatalogName()); + return columnNameEqual && tableNameEqual && schemaNameEqual && catalogNameEqual; + } + return false; + } + +} diff --git a/src/main/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricks.java index 91f446bd..5ab6e57c 100644 --- a/src/main/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricks.java +++ b/src/main/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricks.java @@ -18,9 +18,9 @@ public class ColumnSnapshotGeneratorDatabricks extends ColumnSnapshotGenerator { private static final String ALL_DATA_TYPES = " BIGINT | BINARY | BOOLEAN | DATE | DECIMAL| DECIMAL\\(| DOUBLE | FLOAT | INT | INTERVAL | VOID | SMALLINT | STRING | VARCHAR\\(\\d+\\) | TIMESTAMP | TIMESTAMP_NTZ | TINYINT | ARRAY<| MAP<| STRUCT<| VARIANT| OBJECT<"; - private static final String DEFAULT_CLAUSE_TERMINATORS = "(?i)(\\s+COMMENT\\s+'| PRIMARY\\s+KEY | FOREIGN\\s+KEY | MASK\\s+\\w+|$|,(\\s+\\w+\\s+" + ALL_DATA_TYPES + "|\\)$)"; + private static final String DEFAULT_CLAUSE_TERMINATORS = "(?i)(\\s+COMMENT\\s+'| PRIMARY\\s+KEY | FOREIGN\\s+KEY | MASK\\s+\\w+|$|,(\\s+\\w+\\s+" + ALL_DATA_TYPES + ")?|\\)$"; private static final String GENERATED_BY_DEFAULT_REGEX = "(?i)\\s+GENERATED\\s+(BY\\s+DEFAULT|ALWAYS)\\s+AS\\s+IDENTITY"; - private static final String GENERIC_DEFAULT_VALUE_REGEX = "DEFAULT\\s+(.*?)(" + DEFAULT_CLAUSE_TERMINATORS + "?))"; + private static final String GENERIC_DEFAULT_VALUE_REGEX = "DEFAULT\\s+(.*?)(" + DEFAULT_CLAUSE_TERMINATORS + "))"; private static final String SANITIZE_TABLE_SPECIFICATION_REGEX = "(\\(.*?\\))\\s*(?i)(USING|OPTIONS|PARTITIONED BY|CLUSTER BY|LOCATION|TBLPROPERTIES|WITH|$|;$)"; private static final Pattern DEFAULT_VALUE_PATTERN = Pattern.compile(GENERIC_DEFAULT_VALUE_REGEX); private static final Pattern SANITIZE_TABLE_SPECIFICATION_PATTERN = Pattern.compile(SANITIZE_TABLE_SPECIFICATION_REGEX); @@ -71,12 +71,11 @@ protected DatabaseObject snapshotObject(DatabaseObject example, DatabaseSnapshot String showCreateTableStatement = (String) snapshot.getScratchData(showCreateRelatedTableQuery); String defaultValue = extractDefaultValue(showCreateTableStatement, column.getName()); column.setAutoIncrementInformation(parseAutoIncrementInfo(showCreateTableStatement, column.getName())); - if (defaultValue != null) { + if (defaultValue != null && !defaultValue.equalsIgnoreCase("null")) { Matcher functionMatcher = FUNCTION_PATTERN.matcher(defaultValue); if (functionMatcher.find()) { DatabaseFunction function = new DatabaseFunction(defaultValue); column.setDefaultValue(function); - column.setComputed(true); } else { column.setDefaultValue(defaultValue); } @@ -104,7 +103,8 @@ private String extractDefaultValue(String createTableStatement, String columnNam Matcher defaultValueMatcher = DEFAULT_VALUE_PATTERN.matcher(columnWithPotentialDefault); if (defaultValueMatcher.find()) { defaultValue = defaultValueMatcher.group(1); - if (stringColumnTypeMatcher.find() && defaultStringValueMatcher.find()) { + if (stringColumnTypeMatcher.find() && defaultStringValueMatcher.find() + && (defaultValue.startsWith("'") || defaultValue.startsWith("\""))) { defaultValue = defaultStringValueMatcher.group(2); } } diff --git a/src/main/java/liquibase/ext/databricks/sqlgenerator/AddDefaultValueGeneratorDatabricks.java b/src/main/java/liquibase/ext/databricks/sqlgenerator/AddDefaultValueGeneratorDatabricks.java index 8e200eb4..8fd9272b 100644 --- a/src/main/java/liquibase/ext/databricks/sqlgenerator/AddDefaultValueGeneratorDatabricks.java +++ b/src/main/java/liquibase/ext/databricks/sqlgenerator/AddDefaultValueGeneratorDatabricks.java @@ -13,7 +13,11 @@ import liquibase.statement.core.AddDefaultValueStatement; import liquibase.sqlgenerator.core.AddDefaultValueGenerator; +import java.util.Arrays; +import java.util.List; + public class AddDefaultValueGeneratorDatabricks extends AddDefaultValueGenerator { + private static final List NUMERIC_TYPES = Arrays.asList("TINYINT", "SMALLINT", "INT", "BIGINT", "FLOAT", "DOUBLE", "DECIMAL"); @Override public int getPriority() { return PRIORITY_DATABASE; @@ -40,14 +44,16 @@ public ValidationErrors validate(AddDefaultValueStatement addDefaultValueStateme @Override public Sql[] generateSql(AddDefaultValueStatement statement, Database database, SqlGeneratorChain sqlGeneratorChain) { Object defaultValue = statement.getDefaultValue(); + String columnDataType = statement.getColumnDataType(); String finalDefaultValue; if (defaultValue instanceof DatabaseFunction) { - finalDefaultValue = "("+defaultValue+")"; - if (finalDefaultValue.startsWith("((")) { - finalDefaultValue = defaultValue.toString(); - } + finalDefaultValue = defaultValue.toString(); } else { - finalDefaultValue = DataTypeFactory.getInstance().fromObject(defaultValue, database).objectToSql(defaultValue, database); + if(NUMERIC_TYPES.contains(columnDataType) && defaultValue instanceof String) { + finalDefaultValue = defaultValue.toString().replace("'", "").trim(); + } else { + finalDefaultValue = DataTypeFactory.getInstance().fromObject(defaultValue, database).objectToSql(defaultValue, database); + } } return new Sql[]{ new UnparsedSql("ALTER TABLE " + database.escapeTableName(statement.getCatalogName(), statement.getSchemaName(), statement.getTableName()) + " ALTER COLUMN " + database.escapeColumnName(statement.getCatalogName(), statement.getSchemaName(), statement.getTableName(), statement.getColumnName()) + " SET DEFAULT " + finalDefaultValue, diff --git a/src/main/resources/META-INF/services/liquibase.diff.output.changelog.ChangeGenerator b/src/main/resources/META-INF/services/liquibase.diff.output.changelog.ChangeGenerator index 12441b40..001c63fc 100644 --- a/src/main/resources/META-INF/services/liquibase.diff.output.changelog.ChangeGenerator +++ b/src/main/resources/META-INF/services/liquibase.diff.output.changelog.ChangeGenerator @@ -1,4 +1,5 @@ liquibase.ext.databricks.diff.output.changelog.MissingTableChangeGeneratorDatabricks liquibase.ext.databricks.diff.output.changelog.MissingViewChangeGeneratorDatabricks +liquibase.ext.databricks.diff.output.changelog.MissingColumnChangeGeneratorDatabricks liquibase.ext.databricks.diff.output.changelog.ChangedTableChangeGeneratorDatabricks liquibase.ext.databricks.diff.output.changelog.ChangedViewChangeGeneratorDatabricks diff --git a/src/test/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricksTest.java b/src/test/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricksTest.java index f019af6d..e9530850 100644 --- a/src/test/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricksTest.java +++ b/src/test/java/liquibase/ext/databricks/snapshot/jvm/ColumnSnapshotGeneratorDatabricksTest.java @@ -1,8 +1,6 @@ package liquibase.ext.databricks.snapshot.jvm; -import liquibase.database.jvm.JdbcConnection; import liquibase.exception.DatabaseException; -import liquibase.ext.databricks.database.DatabricksDatabase; import liquibase.snapshot.JdbcDatabaseSnapshot; import liquibase.statement.DatabaseFunction; import liquibase.structure.DatabaseObject; @@ -17,9 +15,6 @@ import org.mockito.Mock; import org.mockito.junit.jupiter.MockitoExtension; -import java.sql.PreparedStatement; -import java.sql.ResultSet; -import java.sql.ResultSetMetaData; import java.sql.SQLException; import java.util.HashMap; import java.util.Map; @@ -90,7 +85,7 @@ void snapshotObjectTest() throws DatabaseException, SQLException { testedColumn.setAttribute("liquibase-complete", true); DatabaseObject databaseObject = snapshotGenerator.snapshotObject(testedColumn, snapshot); assertTrue(databaseObject instanceof Column); - assertTrue(((Column) databaseObject).getComputed()); + assertNull(((Column) databaseObject).getComputed()); assertNotNull(((Column) databaseObject).getDefaultValue()); assertEquals(columnWithDefaultComputed.getValue(), ((Column) databaseObject).getDefaultValue()); }