diff --git a/django_scaffold/management/commands/__init__.py b/django_scaffold/management/commands/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/django_scaffold/management/commands/migrate.py b/django_scaffold/management/commands/migrate.py new file mode 100644 index 000000000..3402e2051 --- /dev/null +++ b/django_scaffold/management/commands/migrate.py @@ -0,0 +1,83 @@ +import logging +import time + +import redis_lock +from django.conf import settings +from django.core.management.base import BaseCommand, CommandError +from django.core.management.commands.migrate import Command as MigrateCommand +from django.db import connections +from django.db import transaction as django_transaction +from django.db.utils import IntegrityError, ProgrammingError + +from services.redis import get_redis_connection + +log = logging.getLogger(__name__) + +MIGRATION_LOCK_NAME = "djang-migrations-lock" + + +class MockLock: + def release(self): + pass + + +class Command(MigrateCommand): + """ + We need to override the migrate command to block on acquiring a lock in Redis. + Otherwise, concurrent worker and api deploys could attempt to run migrations + at the same time which is not safe. + This class is copied from `codecov-api` except it omits logic about faking + certain migrations. When the `legacy_migrations` app is moved to `shared` + and installed in `worker`, which is a prerequisite for core models, we can + delete this. + """ + + def _obtain_lock(self): + """ + In certain environments we might be running mutliple servers that will try and run the migrations at the same time. This is + not safe to do. So we have the command obtain a lock to try and run the migration. If it cannot get a lock, it will wait + until it is able to do so before continuing to run. We need to wait for the lock instead of hard exiting on seeing another + server running the migrations because we write code in such a way that the server expects for migrations to be applied before + new code is deployed (but the opposite of new db with old code is fine). + """ + # If we're running in a non-server environment, we don't need to worry about acquiring a lock + if settings.IS_DEV: + return MockLock() + + redis_connection = get_redis_connection() + lock = redis_lock.Lock( + redis_connection, MIGRATION_LOCK_NAME, expire=180, auto_renewal=True + ) + log.info("Trying to acquire migrations lock...") + acquired = lock.acquire(timeout=180) + + if not acquired: + return None + + return lock + + def handle(self, *args, **options): + log.info("Codecov is starting migrations...") + database = options["database"] + db_connection = connections[database] + options["run_syncdb"] = False + + lock = self._obtain_lock() + + # Failed to acquire lock due to timeout + if not lock: + log.error("Potential deadlock detected in api migrations.") + raise Exception("Failed to obtain lock for api migration.") + + try: + super().handle(*args, **options) + + # Autocommit is disabled in worker + django_transaction.commit(database) + except: + log.info("Codecov migrations failed.") + raise + else: + log.info("Codecov migrations succeeded.") + finally: + lock.release() diff --git a/django_scaffold/settings.py b/django_scaffold/settings.py index 5a3449154..473151a3b 100644 --- a/django_scaffold/settings.py +++ b/django_scaffold/settings.py @@ -9,7 +9,6 @@ ALLOWED_HOSTS = [] DATABASES["default"]["AUTOCOMMIT"] = False -DATABASES["default"]["ENGINE"] = "psqlextra.backend" if "timeseries" in DATABASES: DATABASES["timeseries"]["AUTOCOMMIT"] = False @@ -18,21 +17,10 @@ # Application definition INSTALLED_APPS = [ - "shared.django_apps.legacy_migrations", - "shared.django_apps.codecov_auth", - "shared.django_apps.core", - "shared.django_apps.reports", + "django_scaffold", # must be first to override migrate command "shared.django_apps.pg_telemetry", "shared.django_apps.ts_telemetry", "shared.django_apps.rollouts", - "shared.django_apps.user_measurements", - # Needed after installing user_measurements - "psqlextra", - "django.contrib.admin", - "django.contrib.contenttypes", - "django.contrib.auth", - # Needed for the manage.py commands after installing legacy, codecov_auth, core, reports apps - "django.contrib.messages", ] TELEMETRY_VANILLA_DB = "default" @@ -45,36 +33,9 @@ SKIP_RISKY_MIGRATION_STEPS = get_config("migrations", "skip_risky_steps", default=False) +MIDDLEWARE = [] -# Needed for makemigrations to work -MIDDLEWARE = [ - "django.contrib.auth.middleware.AuthenticationMiddleware", - "django.contrib.messages.middleware.MessageMiddleware", - "django.contrib.sessions.middleware.SessionMiddleware", - "django.middleware.csrf.CsrfViewMiddleware", -] - -# Needed for makemigrations to work -TEMPLATES = [ - { - "BACKEND": "django.template.backends.django.DjangoTemplates", - "DIRS": [], - "APP_DIRS": True, - "OPTIONS": { - "context_processors": [ - "django.template.context_processors.debug", - "django.contrib.auth.context_processors.auth", - "django.template.context_processors.request", - "django.contrib.messages.context_processors.messages", - ] - }, - } -] - -# Allows to use the pgpartition command -PSQLEXTRA_PARTITIONING_MANAGER = ( - "shared.django_apps.user_measurements.partitioning.manager" -) +TEMPLATES = [] # Password validation # https://docs.djangoproject.com/en/4.2/ref/settings/#auth-password-validators diff --git a/requirements.in b/requirements.in index b65dde1ab..a7d24ec25 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,4 @@ -https://github.com/codecov/shared/archive/a5c2ad51b22ea71cf7ce1d4af11ee0cadc7aab62.tar.gz#egg=shared +https://github.com/codecov/shared/archive/2b40636317fa9c242ca0ca130714302b3891fbd4.tar.gz#egg=shared https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem https://github.com/codecov/test-results-parser/archive/5515e960d5d38881036e9127f86320efca649f13.tar.gz#egg=test-results-parser boto3 diff --git a/requirements.txt b/requirements.txt index 3a87e354d..5abbc455b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.10 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile requirements.in @@ -363,7 +363,9 @@ requests==2.31.0 respx==0.20.2 # via -r requirements.in rfc3986[idna2008]==1.4.0 - # via httpx + # via + # httpx + # rfc3986 rsa==4.7.2 # via google-auth s3transfer==0.3.4 @@ -374,7 +376,7 @@ sentry-sdk==1.40.0 # via # -r requirements.in # shared -shared @ https://github.com/codecov/shared/archive/a5c2ad51b22ea71cf7ce1d4af11ee0cadc7aab62.tar.gz +shared @ https://github.com/codecov/shared/archive/2b40636317fa9c242ca0ca130714302b3891fbd4.tar.gz # via -r requirements.in six==1.15.0 # via @@ -427,6 +429,7 @@ typing==3.7.4.3 typing-extensions==4.6.3 # via # asgiref + # kombu # openai # opentelemetry-sdk # pydantic