From fafd35d0a32315edc565bb8a001fdafdcce68ffe Mon Sep 17 00:00:00 2001 From: Kevin Glisson Date: Mon, 24 Jun 2024 15:34:01 -0700 Subject: [PATCH] Making it more reliable --- .github/workflows/lint-migrations.yml | 35 +++---------------- .github/workflows/playwright.yml | 2 +- .github/workflows/python.yml | 4 +-- src/dispatch/database/revisions/core/env.py | 8 ++++- src/dispatch/database/revisions/tenant/env.py | 9 +++-- 5 files changed, 21 insertions(+), 37 deletions(-) diff --git a/.github/workflows/lint-migrations.yml b/.github/workflows/lint-migrations.yml index cbe9abc6a719..79d6f53daa5b 100644 --- a/.github/workflows/lint-migrations.yml +++ b/.github/workflows/lint-migrations.yml @@ -15,10 +15,10 @@ jobs: with: fetch-depth: 0 - name: Set up Python 3.11 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.11.2 - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} @@ -35,36 +35,9 @@ jobs: chmod +x eugene - name: Generate sql files run: | - # Navigate to the git repository root, if not already there - cd "$(git rev-parse --show-toplevel)" || exit - - # Perform git diff to check for changes in the specified directory - changed_files=$(git diff --name-only -- src/dispatch/database/revisions/tenant/versions) - - # Filter for newly added files (*.py) - new_files=$(echo "$changed_files" | grep -E "^src/dispatch/database/revisions/tenant/versions/2024-06-12_.*\.py$") - - # Check if there is exactly one new file and handle errors for multiple or no files - if [[ $(echo "$new_files" | wc -l) -ne 1 ]]; then - echo "Error: There should be exactly one new revision file. Found: $(echo "$new_files" | wc -l)" - exit 1 - fi - - new_file_path=$(echo "$new_files" | head -n 1) - new_revision_id=$(basename "$new_file_path" .py | cut -d'_' -f2) - - # Get the most recent revision ID excluding the newly added one - old_file=$(git ls-files src/dispatch/database/revisions/tenant/versions | grep -oP '2024-06-12_\K[^.]+(?=\.py)' | grep -v "$new_revision_id" | sort -rV | head -n 1) - - if [[ -z "$old_file" ]]; then - echo "Error: No old revision ID found" - exit 1 - fi - - old_revision_id=$(echo "$old_file") - dispatch database upgrade --revision-type tenant --revision="${old_revision_id}:${new_revision_id}" --sql + dispatch database upgrade --revision-type tenant --sql - name: Lint files - run: ./eugene lint --git-diff origin/main migration-scripts -f md --accept-failures > lint.md + run: ./eugene lint alembic_output.sql -f md --accept-failures > lint.md - name: Post Comment env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/playwright.yml b/.github/workflows/playwright.yml index 1a7897144757..bcf23b462174 100644 --- a/.github/workflows/playwright.yml +++ b/.github/workflows/playwright.yml @@ -31,7 +31,7 @@ jobs: - name: Check out Git repository uses: actions/checkout@v3 - name: Set up Python 3.11 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.11 - uses: actions/setup-node@v3 diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index ed9a547ebc82..2f652f531b7b 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -25,10 +25,10 @@ jobs: - name: Check out Git repository uses: actions/checkout@v2 - name: Set up Python 3.11 - uses: actions/setup-python@v2 + uses: actions/setup-python@v5 with: python-version: 3.11.2 - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.cache/pip key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }} diff --git a/src/dispatch/database/revisions/core/env.py b/src/dispatch/database/revisions/core/env.py index c001ddfbf068..364a1489ed63 100644 --- a/src/dispatch/database/revisions/core/env.py +++ b/src/dispatch/database/revisions/core/env.py @@ -1,4 +1,4 @@ -from alembic import context +from alembic import context, script from sqlalchemy import engine_from_config, pool from dispatch.logging import logging @@ -85,6 +85,10 @@ def run_migrations_offline(): # Get the URL from the config url = config.get_main_option("sqlalchemy.url") + alembic_script = script.ScriptDirectory.from_config(config) + + _, previous = list(alembic_script.walk_revisions(base='base', head='heads'))[:2] + # Set the migration options context.configure( url=url, @@ -92,6 +96,7 @@ def run_migrations_offline(): include_schemas=True, include_object=include_object, literal_binds=True, # Binds parameters with their string values + starting_rev=previous.revision, ) # Start a transaction and run migrations @@ -99,6 +104,7 @@ def run_migrations_offline(): context.run_migrations() + if context.is_offline_mode(): run_migrations_offline() else: diff --git a/src/dispatch/database/revisions/tenant/env.py b/src/dispatch/database/revisions/tenant/env.py index 8e3d98a96cad..d09b83cd5077 100644 --- a/src/dispatch/database/revisions/tenant/env.py +++ b/src/dispatch/database/revisions/tenant/env.py @@ -1,5 +1,5 @@ -import os -from alembic import context +from alembic import context, script + from sqlalchemy import engine_from_config, pool, inspect @@ -98,6 +98,10 @@ def run_migrations_offline(): # Get the URL from the config url = config.get_main_option("sqlalchemy.url") + alembic_script = script.ScriptDirectory.from_config(config) + + _, previous = list(alembic_script.walk_revisions(base='base', head='heads'))[:2] + # Set the migration options context.configure( url=url, @@ -105,6 +109,7 @@ def run_migrations_offline(): include_schemas=True, include_object=include_object, literal_binds=True, # Binds parameters with their string values + starting_rev=previous.revision, ) # Start a transaction and run migrations