Skip to content

Commit

Permalink
Patch to syncstores to enable support for PostGIS 3.4 (#1097)
Browse files Browse the repository at this point in the history 
* Add postgis_raster initialization (for PostGIS 3.4)

* Remove close

* Syncstores enables postgis raster extension for PostGIS 3.X or higher

* Conditionally enable postgis raster for postgis version 3+ functional

* Add tests for new postgis 3 functionality

* Black formatting
  • Loading branch information
@swainn
swainn authored Sep 26, 2024
1 parent f618b8c commit 79b2abe
Show file tree
Hide file tree
Showing 2 changed files with 267 additions and 9 deletions.
239 changes: 235 additions & 4 deletions tests/unit_tests/test_tethys_apps/test_models/test_PersistentStoreDatabaseSetting.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -427,16 +427,247 @@ def test_create_persistent_store_database(
).create_persistent_store_database(refresh=True, force_first_time=True)

# Check mock called
rts_get_args = mock_log.getLogger().info.call_args_list
mock_log_info_calls = mock_log.getLogger().info.call_args_list
check_log1 = 'Creating database "spatial_db" for app "test_app"...'
check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...'
check_log3 = (
'Initializing database "spatial_db" for app "test_app" '
'with initializer "appsettings.model.init_spatial_db"...'
)
self.assertEqual(check_log1, rts_get_args[0][0][0])
self.assertEqual(check_log2, rts_get_args[1][0][0])
self.assertEqual(check_log3, rts_get_args[2][0][0])
self.assertEqual(check_log1, mock_log_info_calls[0][0][0])
self.assertEqual(check_log2, mock_log_info_calls[1][0][0])
self.assertEqual(check_log3, mock_log_info_calls[2][0][0])
mock_init.assert_called()

@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.drop_persistent_store_database"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.get_namespaced_persistent_store_name"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.persistent_store_database_exists"
)
@mock.patch("tethys_apps.models.PersistentStoreDatabaseSetting.get_value")
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.initializer_function"
)
@mock.patch("tethys_apps.models.logging")
def test_create_persistent_store_database_postgis2(
self, mock_log, mock_init, mock_get, mock_ps_de, mock_gn, mock_drop
):
# Mock Get Name
mock_gn.return_value = "spatial_db"

# Mock Drop Database
mock_drop.return_value = ""

# Mock persistent_store_database_exists
mock_ps_de.return_value = False # DB does not exist

# Mock get_values
mock_url = mock.MagicMock(username="test_app")
mock_engine = mock.MagicMock()
mock_new_db_engine = mock.MagicMock()
mock_db_connection = mock_new_db_engine.connect()
mock_init_param = mock.MagicMock()
mock_get.side_effect = [
mock_url,
mock_engine,
mock_new_db_engine,
mock_init_param,
]
mock_db_connection.execute.side_effect = [
mock.MagicMock(), # Enable PostGIS Statement
[
mock.MagicMock(postgis_version="2.5 USE_GEOS=1 USE_PROJ=1 USE_STATS=1")
], # Check PostGIS Version
mock.MagicMock(), # Enable PostGIS Raster Statement
]

# Execute
self.test_app.settings_set.select_subclasses().get(
name="spatial_db"
).create_persistent_store_database(refresh=False, force_first_time=False)

# Check mock calls
mock_execute_calls = mock_db_connection.execute.call_args_list
self.assertEqual(2, len(mock_execute_calls))
execute1 = "CREATE EXTENSION IF NOT EXISTS postgis;"
execute2 = "SELECT PostGIS_Version();"
self.assertEqual(execute1, mock_execute_calls[0][0][0])
self.assertEqual(execute2, mock_execute_calls[1][0][0])

mock_log_info_calls = mock_log.getLogger().info.call_args_list
self.assertEqual(4, len(mock_log_info_calls))
check_log1 = 'Creating database "spatial_db" for app "test_app"...'
check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...'
check_log3 = "Detected PostGIS version 2.5"
check_log4 = (
'Initializing database "spatial_db" for app "test_app" '
'with initializer "appsettings.model.init_spatial_db"...'
)
self.assertEqual(check_log1, mock_log_info_calls[0][0][0])
self.assertEqual(check_log2, mock_log_info_calls[1][0][0])
self.assertEqual(check_log3, mock_log_info_calls[2][0][0])
self.assertEqual(check_log4, mock_log_info_calls[3][0][0])
mock_init.assert_called()

@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.drop_persistent_store_database"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.get_namespaced_persistent_store_name"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.persistent_store_database_exists"
)
@mock.patch("tethys_apps.models.PersistentStoreDatabaseSetting.get_value")
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.initializer_function"
)
@mock.patch("tethys_apps.models.logging")
def test_create_persistent_store_database_postgis3(
self, mock_log, mock_init, mock_get, mock_ps_de, mock_gn, mock_drop
):
# Mock Get Name
mock_gn.return_value = "spatial_db"

# Mock Drop Database
mock_drop.return_value = ""

# Mock persistent_store_database_exists
mock_ps_de.return_value = False # DB does not exist

# Mock get_values
mock_url = mock.MagicMock(username="test_app")
mock_engine = mock.MagicMock()
mock_new_db_engine = mock.MagicMock()
mock_db_connection = mock_new_db_engine.connect()
mock_init_param = mock.MagicMock()
mock_get.side_effect = [
mock_url,
mock_engine,
mock_new_db_engine,
mock_init_param,
]
mock_db_connection.execute.side_effect = [
mock.MagicMock(), # Enable PostGIS Statement
[
mock.MagicMock(postgis_version="3.5 USE_GEOS=1 USE_PROJ=1 USE_STATS=1")
], # Check PostGIS Version
mock.MagicMock(), # Enable PostGIS Raster Statement
]

# Execute
self.test_app.settings_set.select_subclasses().get(
name="spatial_db"
).create_persistent_store_database(refresh=False, force_first_time=False)

# Check mock calls
mock_execute_calls = mock_db_connection.execute.call_args_list
self.assertEqual(3, len(mock_execute_calls))
execute1 = "CREATE EXTENSION IF NOT EXISTS postgis;"
execute2 = "SELECT PostGIS_Version();"
execute3 = "CREATE EXTENSION IF NOT EXISTS postgis_raster;"
self.assertEqual(execute1, mock_execute_calls[0][0][0])
self.assertEqual(execute2, mock_execute_calls[1][0][0])
self.assertEqual(execute3, mock_execute_calls[2][0][0])

mock_log_info_calls = mock_log.getLogger().info.call_args_list
self.assertEqual(5, len(mock_log_info_calls))
check_log1 = 'Creating database "spatial_db" for app "test_app"...'
check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...'
check_log3 = "Detected PostGIS version 3.5"
check_log4 = (
'Enabling PostGIS Raster on database "spatial_db" for app "test_app"...'
)
check_log5 = (
'Initializing database "spatial_db" for app "test_app" '
'with initializer "appsettings.model.init_spatial_db"...'
)
self.assertEqual(check_log1, mock_log_info_calls[0][0][0])
self.assertEqual(check_log2, mock_log_info_calls[1][0][0])
self.assertEqual(check_log3, mock_log_info_calls[2][0][0])
self.assertEqual(check_log4, mock_log_info_calls[3][0][0])
self.assertEqual(check_log5, mock_log_info_calls[4][0][0])
mock_init.assert_called()

@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.drop_persistent_store_database"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.get_namespaced_persistent_store_name"
)
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.persistent_store_database_exists"
)
@mock.patch("tethys_apps.models.PersistentStoreDatabaseSetting.get_value")
@mock.patch(
"tethys_apps.models.PersistentStoreDatabaseSetting.initializer_function"
)
@mock.patch("tethys_apps.models.logging")
def test_create_persistent_store_database_postgis3_bad_version_string(
self, mock_log, mock_init, mock_get, mock_ps_de, mock_gn, mock_drop
):
# Mock Get Name
mock_gn.return_value = "spatial_db"

# Mock Drop Database
mock_drop.return_value = ""

# Mock persistent_store_database_exists
mock_ps_de.return_value = False # DB does not exist

# Mock get_values
mock_url = mock.MagicMock(username="test_app")
mock_engine = mock.MagicMock()
mock_new_db_engine = mock.MagicMock()
mock_db_connection = mock_new_db_engine.connect()
mock_init_param = mock.MagicMock()
mock_get.side_effect = [
mock_url,
mock_engine,
mock_new_db_engine,
mock_init_param,
]
mock_db_connection.execute.side_effect = [
mock.MagicMock(), # Enable PostGIS Statement
[
mock.MagicMock(postgis_version="BAD VERSION STRING")
], # Check PostGIS Version
mock.MagicMock(), # Enable PostGIS Raster Statement
]

# Execute
self.test_app.settings_set.select_subclasses().get(
name="spatial_db"
).create_persistent_store_database(refresh=False, force_first_time=False)

# Check mock calls
mock_execute_calls = mock_db_connection.execute.call_args_list
self.assertEqual(2, len(mock_execute_calls))
execute1 = "CREATE EXTENSION IF NOT EXISTS postgis;"
execute2 = "SELECT PostGIS_Version();"
self.assertEqual(execute1, mock_execute_calls[0][0][0])
self.assertEqual(execute2, mock_execute_calls[1][0][0])

mock_log_warning_calls = mock_log.getLogger().warning.call_args_list
self.assertEqual(1, len(mock_log_warning_calls))
check_log1 = 'Could not parse PostGIS version from "BAD VERSION STRING"'
self.assertEqual(check_log1, mock_log_warning_calls[0][0][0])

mock_log_info_calls = mock_log.getLogger().info.call_args_list
self.assertEqual(3, len(mock_log_info_calls))
check_log1 = 'Creating database "spatial_db" for app "test_app"...'
check_log2 = 'Enabling PostGIS on database "spatial_db" for app "test_app"...'
check_log3 = (
'Initializing database "spatial_db" for app "test_app" '
'with initializer "appsettings.model.init_spatial_db"...'
)
self.assertEqual(check_log1, mock_log_info_calls[0][0][0])
self.assertEqual(check_log2, mock_log_info_calls[1][0][0])
self.assertEqual(check_log3, mock_log_info_calls[2][0][0])
mock_init.assert_called()

@mock.patch("sqlalchemy.exc")
Expand Down
37 changes: 32 additions & 5 deletions tethys_apps/models.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1163,19 +1163,46 @@ def create_persistent_store_database(self, refresh=False, force_first_time=False
)
)

enable_postgis_statement = "CREATE EXTENSION IF NOT EXISTS postgis"

# Execute postgis statement
try:
new_db_connection.execute(enable_postgis_statement)
new_db_connection.execute("CREATE EXTENSION IF NOT EXISTS postgis;")

# Get the POSTGIS version
ret = new_db_connection.execute("SELECT PostGIS_Version();")
postgis_version = None
for r in ret:
# Example version string: "3.4 USE_GEOS=1 USE_PROJ=1 USE_STATS=1"
try:
postgis_version = float(r.postgis_version.split(" ")[0])
log.info(f"Detected PostGIS version {postgis_version}")
break
except Exception:
log.warning(
f'Could not parse PostGIS version from "{r.postgis_version}"'
)
continue

# Execute postgis raster statement for verions 3.0 and above
if postgis_version is not None and postgis_version >= 3.0:
log.info(
'Enabling PostGIS Raster on database "{0}" for app "{1}"...'.format(
self.name,
self.tethys_app.package,
)
)
new_db_connection.execute(
"CREATE EXTENSION IF NOT EXISTS postgis_raster;"
)

except sqlalchemy.exc.ProgrammingError:
raise PersistentStorePermissionError(
'Database user "{0}" has insufficient permissions to enable '
'spatial extension on persistent store database "{1}": must be a '
"superuser.".format(url.username, self.name)
)
finally:
new_db_connection.close()

# Close connection
new_db_connection.close()

# -------------------------------------------------------------------------------------------------------------#
# 4. Run initialization function
Expand Down

0 comments on commit 79b2abe

Please sign in to comment.