Migration to clean up overdrive settings.

alembic/versions/20240104_735bf6ced8b9_update_license_provider_settings.py

@@ -0,0 +1,63 @@
+"""Update license provider settings.
+
+Revision ID: 735bf6ced8b9
+Revises: d3cdbea3d43b
+Create Date: 2024-01-04 16:24:32.895789+00:00
+
+"""
+from alembic import op
+from api.integration.registry.license_providers import LicenseProvidersRegistry
+from core.integration.base import HasChildIntegrationConfiguration
+from core.migration.util import migration_logger
+from core.model import json_serializer
+
+# revision identifiers, used by Alembic.
+revision = "735bf6ced8b9"
+down_revision = "d3cdbea3d43b"
+branch_labels = None
+depends_on = None
+
+
+log = migration_logger(revision)
+
+
+def upgrade() -> None:
+    conn = op.get_bind()
+
+    rows = conn.execute(
+        "SELECT ic.id as integration_id, ic.settings, ic.protocol, ic.goal, c.parent_id "
+        "FROM collections c JOIN integration_configurations ic ON c.integration_configuration_id = ic.id"
+    ).all()
+
+    registry = LicenseProvidersRegistry()
+    for row in rows:
+        settings_dict = row.settings.copy()
+        impl_class = registry.get(row.protocol)
+        if impl_class is None:
+            raise RuntimeError(
+                f"Could not find implementation for protocol {row.protocol}"
+            )
+        if row.parent_id is not None:
+            if issubclass(impl_class, HasChildIntegrationConfiguration):
+                settings_obj = impl_class.child_settings_class()(**settings_dict)
+            else:
+                raise RuntimeError(
+                    f"Integration {row.integration_id} is a child integration, "
+                    f"but {row.protocol} does not support child integrations."
+                )
+        else:
+            settings_obj = impl_class.settings_class()(**settings_dict)
+        new_settings_dict = settings_obj.dict(exclude_extra=True)
+        if row.settings != new_settings_dict:
+            new_settings = json_serializer(new_settings_dict)
+            log.info(
+                f"Updating settings for integration {row.integration_id} from {row.settings} to {new_settings}."
+            )
+            conn.execute(
+                "UPDATE integration_configurations SET settings = (%s) WHERE id = (%s)",
+                (new_settings, row.integration_id),
+            )
+
+
+def downgrade() -> None:
+    pass

core/integration/settings.py

@@ -336,8 +336,16 @@ def configuration_form(cls, db: Session) -> list[dict[str, Any]]:
 
     def dict(self, *args: Any, **kwargs: Any) -> dict[str, Any]:
         """Override the dict method to remove the default values"""
+
         if "exclude_defaults" not in kwargs:
             kwargs["exclude_defaults"] = True
+
+        # Allow us to exclude extra fields that are not defined on the model
+        if "exclude_extra" in kwargs:
+            exclude_extra = kwargs.pop("exclude_extra")
+            if exclude_extra:
+                kwargs["exclude"] = self.__fields_set__ - self.__fields__.keys()
+
         return super().dict(*args, **kwargs)
 
     @classmethod

tests/core/integration/test_settings.py

@@ -159,6 +159,13 @@ def test_settings_extra_args(mock_settings, caplog):
     assert len(caplog.records) == 1
     assert "Unexpected extra argument 'extra' for model MockSettings" in caplog.text
 
+    # Exclude extra defaults to False, but we call it explicitly here
+    # to make sure it can be explicitly set to False.
+    assert settings.dict(exclude_extra=False) == {"number": 1, "extra": "extra"}
+
+    # The extra args will be ignored if we call dict with exclude_extra=True
+    assert settings.dict(exclude_extra=True) == {"number": 1}
+
 
 def test_settings_logger(mock_settings):
     log = mock_settings.logger()