revert shared pr (#397)

django_scaffold/management/commands/migrate.py

@@ -0,0 +1,83 @@
+import logging
+import time
+
+import redis_lock
+from django.conf import settings
+from django.core.management.base import BaseCommand, CommandError
+from django.core.management.commands.migrate import Command as MigrateCommand
+from django.db import connections
+from django.db import transaction as django_transaction
+from django.db.utils import IntegrityError, ProgrammingError
+
+from services.redis import get_redis_connection
+
+log = logging.getLogger(__name__)
+
+MIGRATION_LOCK_NAME = "djang-migrations-lock"
+
+
+class MockLock:
+    def release(self):
+        pass
+
+
+class Command(MigrateCommand):
+    """
+    We need to override the migrate command to block on acquiring a lock in Redis.
+    Otherwise, concurrent worker and api deploys could attempt to run migrations
+    at the same time which is not safe.
+    This class is copied from `codecov-api` except it omits logic about faking
+    certain migrations. When the `legacy_migrations` app is moved to `shared`
+    and installed in `worker`, which is a prerequisite for core models, we can
+    delete this.
+    """
+
+    def _obtain_lock(self):
+        """
+        In certain environments we might be running mutliple servers that will try and run the migrations at the same time. This is
+        not safe to do. So we have the command obtain a lock to try and run the migration. If it cannot get a lock, it will wait
+        until it is able to do so before continuing to run. We need to wait for the lock instead of hard exiting on seeing another
+        server running the migrations because we write code in such a way that the server expects for migrations to be applied before
+        new code is deployed (but the opposite of new db with old code is fine).
+        """
+        # If we're running in a non-server environment, we don't need to worry about acquiring a lock
+        if settings.IS_DEV:
+            return MockLock()
+
+        redis_connection = get_redis_connection()
+        lock = redis_lock.Lock(
+            redis_connection, MIGRATION_LOCK_NAME, expire=180, auto_renewal=True
+        )
+        log.info("Trying to acquire migrations lock...")
+        acquired = lock.acquire(timeout=180)
+
+        if not acquired:
+            return None
+
+        return lock
+
+    def handle(self, *args, **options):
+        log.info("Codecov is starting migrations...")
+        database = options["database"]
+        db_connection = connections[database]
+        options["run_syncdb"] = False
+
+        lock = self._obtain_lock()
+
+        # Failed to acquire lock due to timeout
+        if not lock:
+            log.error("Potential deadlock detected in api migrations.")
+            raise Exception("Failed to obtain lock for api migration.")
+
+        try:
+            super().handle(*args, **options)
+
+            # Autocommit is disabled in worker
+            django_transaction.commit(database)
+        except:
+            log.info("Codecov migrations failed.")
+            raise
+        else:
+            log.info("Codecov migrations succeeded.")
+        finally:
+            lock.release()

django_scaffold/settings.py

@@ -9,7 +9,6 @@
 ALLOWED_HOSTS = []
 
 DATABASES["default"]["AUTOCOMMIT"] = False
-DATABASES["default"]["ENGINE"] = "psqlextra.backend"
 if "timeseries" in DATABASES:
     DATABASES["timeseries"]["AUTOCOMMIT"] = False
 
@@ -18,21 +17,10 @@
 # Application definition
 
 INSTALLED_APPS = [
-    "shared.django_apps.legacy_migrations",
-    "shared.django_apps.codecov_auth",
-    "shared.django_apps.core",
-    "shared.django_apps.reports",
+    "django_scaffold",  # must be first to override migrate command
     "shared.django_apps.pg_telemetry",
     "shared.django_apps.ts_telemetry",
     "shared.django_apps.rollouts",
-    "shared.django_apps.user_measurements",
-    # Needed after installing user_measurements
-    "psqlextra",
-    "django.contrib.admin",
-    "django.contrib.contenttypes",
-    "django.contrib.auth",
-    # Needed for the manage.py commands after installing legacy, codecov_auth, core, reports apps
-    "django.contrib.messages",
 ]
 
 TELEMETRY_VANILLA_DB = "default"
@@ -45,36 +33,9 @@
 
 SKIP_RISKY_MIGRATION_STEPS = get_config("migrations", "skip_risky_steps", default=False)
 
+MIDDLEWARE = []
 
-# Needed for makemigrations to work
-MIDDLEWARE = [
-    "django.contrib.auth.middleware.AuthenticationMiddleware",
-    "django.contrib.messages.middleware.MessageMiddleware",
-    "django.contrib.sessions.middleware.SessionMiddleware",
-    "django.middleware.csrf.CsrfViewMiddleware",
-]
-
-# Needed for makemigrations to work
-TEMPLATES = [
-    {
-        "BACKEND": "django.template.backends.django.DjangoTemplates",
-        "DIRS": [],
-        "APP_DIRS": True,
-        "OPTIONS": {
-            "context_processors": [
-                "django.template.context_processors.debug",
-                "django.contrib.auth.context_processors.auth",
-                "django.template.context_processors.request",
-                "django.contrib.messages.context_processors.messages",
-            ]
-        },
-    }
-]
-
-# Allows to use the pgpartition command
-PSQLEXTRA_PARTITIONING_MANAGER = (
-    "shared.django_apps.user_measurements.partitioning.manager"
-)
+TEMPLATES = []
 
 # Password validation
 # https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators

requirements.in

@@ -1,4 +1,4 @@
-https://github.com/codecov/shared/archive/a5c2ad51b22ea71cf7ce1d4af11ee0cadc7aab62.tar.gz#egg=shared
+https://github.com/codecov/shared/archive/2b40636317fa9c242ca0ca130714302b3891fbd4.tar.gz#egg=shared
 https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem
 https://github.com/codecov/test-results-parser/archive/5515e960d5d38881036e9127f86320efca649f13.tar.gz#egg=test-results-parser
 boto3

requirements.txt

@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with Python 3.10
+# This file is autogenerated by pip-compile with Python 3.9
 # by the following command:
 #
 #    pip-compile requirements.in
@@ -363,7 +363,9 @@ requests==2.31.0
 respx==0.20.2
     # via -r requirements.in
 rfc3986[idna2008]==1.4.0
-    # via httpx
+    # via
+    #   httpx
+    #   rfc3986
 rsa==4.7.2
     # via google-auth
 s3transfer==0.3.4
@@ -374,7 +376,7 @@ sentry-sdk==1.40.0
     # via
     #   -r requirements.in
     #   shared
-shared @ https://github.com/codecov/shared/archive/a5c2ad51b22ea71cf7ce1d4af11ee0cadc7aab62.tar.gz
+shared @ https://github.com/codecov/shared/archive/2b40636317fa9c242ca0ca130714302b3891fbd4.tar.gz
     # via -r requirements.in
 six==1.15.0
     # via
@@ -427,6 +429,7 @@ typing==3.7.4.3
 typing-extensions==4.6.3
     # via
     #   asgiref
+    #   kombu
     #   openai
     #   opentelemetry-sdk
     #   pydantic