diff --git a/requirements.in b/requirements.in index 6d144275c..c08834c4a 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,4 @@ -https://github.com/codecov/shared/archive/b48aec853deed52c19b0c3a44549787009861691.tar.gz#egg=shared +https://github.com/codecov/shared/archive/b653541785c677d6bc2ba438c628e1e645d521b9.tar.gz#egg=shared https://github.com/codecov/opentelem-python/archive/refs/tags/v0.0.4a1.tar.gz#egg=codecovopentelem https://github.com/codecov/test-results-parser/archive/5515e960d5d38881036e9127f86320efca649f13.tar.gz#egg=test-results-parser boto3 diff --git a/requirements.txt b/requirements.txt index 618336700..bfe503978 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with Python 3.11 +# This file is autogenerated by pip-compile with Python 3.9 # by the following command: # # pip-compile requirements.in @@ -116,6 +116,8 @@ ecdsa==0.18.0 # via tlslite-ng excel-base==1.0.4 # via django-excel-response2 +exceptiongroup==1.2.0 + # via pytest factory-boy==3.2.0 # via -r requirements.in faker==8.8.2 @@ -355,7 +357,9 @@ requests==2.31.0 respx==0.20.2 # via -r requirements.in rfc3986[idna2008]==1.4.0 - # via httpx + # via + # httpx + # rfc3986 rsa==4.7.2 # via google-auth s3transfer==0.3.4 @@ -366,7 +370,7 @@ sentry-sdk==1.40.0 # via # -r requirements.in # shared -shared @ https://github.com/codecov/shared/archive/b48aec853deed52c19b0c3a44549787009861691.tar.gz +shared @ https://github.com/codecov/shared/archive/b653541785c677d6bc2ba438c628e1e645d521b9.tar.gz # via -r requirements.in six==1.15.0 # via @@ -410,12 +414,16 @@ timestring @ https://github.com/codecov/timestring/archive/d37ceacc5954dff3b5bd2 # via -r requirements.in tlslite-ng==0.8.0b1 # via shared +tomli==2.0.1 + # via pytest tqdm==4.66.1 # via openai typing==3.7.4.3 # via shared typing-extensions==4.6.3 # via + # asgiref + # kombu # openai # opentelemetry-sdk # pydantic diff --git a/tasks/backfill_gh_app_installations.py b/tasks/backfill_gh_app_installations.py index c0ca420b7..7aa10dfe8 100644 --- a/tasks/backfill_gh_app_installations.py +++ b/tasks/backfill_gh_app_installations.py @@ -3,11 +3,17 @@ import shared.torngit as torngit from asgiref.sync import async_to_sync +from shared.config import get_config from sqlalchemy.orm.session import Session from app import celery_app from celery_config import backfill_gh_app_installations_name -from database.models.core import GithubAppInstallation, Owner, Repository +from database.models.core import ( + GITHUB_APP_INSTALLATION_DEFAULT_NAME, + GithubAppInstallation, + Owner, + Repository, +) from services.owner import get_owner_provider_service from tasks.base import BaseCodecovTask @@ -54,6 +60,28 @@ def add_repos_service_ids_from_provider( gh_app_installation.repository_service_ids = new_repo_service_ids db_session.commit() + def maybe_set_installation_to_all_repos( + self, + db_session: Session, + owner_service, + gh_app_installation: GithubAppInstallation, + ): + remote_gh_app_installation = async_to_sync( + owner_service.get_gh_app_installation + )(installation_id=gh_app_installation.installation_id) + repository_selection = remote_gh_app_installation.get( + "repository_selection", "" + ) + if repository_selection == "all": + gh_app_installation.repository_service_ids = None + db_session.commit() + log.info( + "Selection is set to all, no installation is needed", + extra=dict(ownerid=gh_app_installation.ownerid), + ) + return True + return False + def backfill_existing_gh_apps( self, db_session: Session, owner_ids: List[int] = None ): @@ -84,29 +112,19 @@ def backfill_existing_gh_apps( for gh_app_installation in gh_app_installations: # Check if gh app has 'all' repositories selected - installation_id = gh_app_installation.installation_id - owner = owners_dict[gh_app_installation.ownerid] ownerid = owner.ownerid owner_service = get_owner_provider_service( owner=owner, using_integration=True ) - - remote_gh_app_installation = async_to_sync( - owner_service.get_gh_app_installation - )(installation_id=installation_id) - repository_selection = remote_gh_app_installation.get( - "repository_selection", "" + is_selection_all = self.maybe_set_installation_to_all_repos( + db_session=db_session, + owner_service=owner_service, + gh_app_installation=gh_app_installation, ) - if repository_selection == "all": - gh_app_installation.repository_service_ids = None - db_session.commit() - log.info( - "Selection is set to all, no installation is needed", - extra=dict(ownerid=ownerid), - ) - else: + + if not is_selection_all: # Find and add all repos the gh app has access to self.add_repos_service_ids_from_provider( db_session=db_session, @@ -153,18 +171,28 @@ def backfill_owners_with_integration_without_gh_app( extra=dict(ownerid=ownerid), ) gh_app_installation = GithubAppInstallation( - owner=owner, installation_id=owner.integration_id + owner=owner, + installation_id=owner.integration_id, + app_id=get_config("github", "integration", "id"), + name=GITHUB_APP_INSTALLATION_DEFAULT_NAME, ) db_session.add(gh_app_installation) db_session.commit() - # Find and add all repos the gh app has access to - self.add_repos_service_ids_from_provider( + is_selection_all = self.maybe_set_installation_to_all_repos( db_session=db_session, - ownerid=ownerid, owner_service=owner_service, gh_app_installation=gh_app_installation, ) + + if not is_selection_all: + # Find and add all repos the gh app has access to + self.add_repos_service_ids_from_provider( + db_session=db_session, + ownerid=ownerid, + owner_service=owner_service, + gh_app_installation=gh_app_installation, + ) log.info("Successful backfill", extra=dict(ownerid=ownerid)) def run_impl( @@ -174,6 +202,10 @@ def run_impl( *args, **kwargs ): + log.info( + "Starting GH App backfill task", + ) + # Backfill gh apps we already have self.backfill_existing_gh_apps(db_session=db_session, owner_ids=owner_ids) diff --git a/tasks/tests/unit/test_backfill_gh_app_installations.py b/tasks/tests/unit/test_backfill_gh_app_installations.py index bf4d4ae99..0723d8bda 100644 --- a/tasks/tests/unit/test_backfill_gh_app_installations.py +++ b/tasks/tests/unit/test_backfill_gh_app_installations.py @@ -312,10 +312,11 @@ def test_backfill_with_both_types_of_owners( dbsession.add_all([owner, gh_app_installation, owner_without_app]) dbsession.commit() - # Mock fn return values - mock_repo_provider.get_gh_app_installation.return_value = { - "repository_selection": "all" - } + # Mock fn return values; 1st call "all", 2nd call "selected" + mock_repo_provider.get_gh_app_installation.side_effect = [ + {"repository_selection": "all"}, + {"repository_selection": "selected"}, + ] mock_repo_provider.list_repos_using_installation.return_value = mock_repos mocker.patch( f"tasks.backfill_gh_app_installations.get_owner_provider_service",