From b2d0e283e94690b442bd83642479ac1bb8c33cfc Mon Sep 17 00:00:00 2001 From: Ivar Flakstad <69173633+ivarflakstad@users.noreply.github.com> Date: Fri, 20 Dec 2024 18:24:33 +0100 Subject: [PATCH] Add option of specifying result upload repo --- .github/workflows/slack-report.yml | 15 ++++++++++++--- utils/notification_service.py | 23 ++++++++++++++++------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/slack-report.yml b/.github/workflows/slack-report.yml index ee2962ba89c37f..eb3294f392d94f 100644 --- a/.github/workflows/slack-report.yml +++ b/.github/workflows/slack-report.yml @@ -21,6 +21,13 @@ on: ci_event: required: true type: string + report_repo_id: + required: false + type: string + upload_report_summary: + required: false + type: boolean + default: false env: TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN: ${{ secrets.TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN }} @@ -47,6 +54,8 @@ jobs: CI_SLACK_CHANNEL_ID_DAILY: ${{ secrets.CI_SLACK_CHANNEL_ID_DAILY }} CI_SLACK_CHANNEL_DUMMY_TESTS: ${{ secrets.CI_SLACK_CHANNEL_DUMMY_TESTS }} SLACK_REPORT_CHANNEL: ${{ inputs.slack_report_channel }} + REPORT_REPO_ID: ${{ inputs.report_repo_id }} + UPLOAD_REPORT_SUMMARY: ${{ inputs.upload_report_summary }} ACCESS_REPO_INFO_TOKEN: ${{ secrets.ACCESS_REPO_INFO_TOKEN }} CI_EVENT: ${{ inputs.ci_event }} CI_SHA: ${{ github.sha }} @@ -70,7 +79,7 @@ jobs: with: name: ci_results_${{ inputs.job }} path: ci_results_${{ inputs.job }} - + - uses: actions/checkout@v4 - uses: actions/download-artifact@v4 - name: Send message to Slack for quantization workflow @@ -90,7 +99,7 @@ jobs: pip install huggingface_hub pip install slack_sdk pip show slack_sdk - python utils/notification_service_quantization.py "${{ inputs.quantization_matrix }}" + python utils/notification_service_quantization.py "${{ inputs.quantization_matrix }}" # Upload complete failure tables, as they might be big and only truncated versions could be sent to Slack. - name: Failure table artifacts @@ -98,4 +107,4 @@ jobs: uses: actions/upload-artifact@v4 with: name: ci_results_${{ inputs.job }} - path: ci_results_${{ inputs.job }} \ No newline at end of file + path: ci_results_${{ inputs.job }} diff --git a/utils/notification_service.py b/utils/notification_service.py index 6c9eab3a85387b..3581fd4fac3f7f 100644 --- a/utils/notification_service.py +++ b/utils/notification_service.py @@ -108,11 +108,13 @@ def __init__( ci_title: str, model_results: Dict, additional_results: Dict, - selected_warnings: List = None, + repo_id: str = "hf-internal-testing/transformers_daily_ci", + selected_warnings: Union[List, None] = None, prev_ci_artifacts=None, ): self.title = title self.ci_title = ci_title + self.repo_id = repo_id # Failures and success of the modeling tests self.n_model_success = sum(r["success"] for r in model_results.values()) @@ -533,11 +535,11 @@ def payload(self) -> str: commit_info = api.upload_file( path_or_fileobj=file_path, path_in_repo=f"{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_{job_name}/new_model_failures.txt", - repo_id="hf-internal-testing/transformers_daily_ci", + repo_id=self.repo_id, repo_type="dataset", token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None), ) - url = f"https://huggingface.co/datasets/hf-internal-testing/transformers_daily_ci/raw/{commit_info.oid}/{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_{job_name}/new_model_failures.txt" + url = f"https://huggingface.co/datasets/{self.repo_id}/raw/{commit_info.oid}/{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_{job_name}/new_model_failures.txt" # extra processing to save to json format new_failed_tests = {} @@ -560,7 +562,7 @@ def payload(self) -> str: _ = api.upload_file( path_or_fileobj=file_path, path_in_repo=f"{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_{job_name}/new_model_failures.json", - repo_id="hf-internal-testing/transformers_daily_ci", + repo_id=self.repo_id, repo_type="dataset", token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None), ) @@ -920,6 +922,8 @@ def prepare_reports(title, header, reports, to_truncate=True): if __name__ == "__main__": SLACK_REPORT_CHANNEL_ID = os.environ["SLACK_REPORT_CHANNEL"] + REPORT_REPO_ID = os.environ.get("REPORT_REPO_ID", "hf-internal-testing/transformers_daily_ci") + UPLOAD_REPORT_SUMMARY = os.environ.get("UPLOAD_REPORT_SUMMARY") == "true" # runner_status = os.environ.get("RUNNER_STATUS") # runner_env_status = os.environ.get("RUNNER_ENV_STATUS") @@ -1220,7 +1224,8 @@ def prepare_reports(title, header, reports, to_truncate=True): os.makedirs(os.path.join(os.getcwd(), f"ci_results_{job_name}")) target_workflow = "huggingface/transformers/.github/workflows/self-scheduled-caller.yml@refs/heads/main" - is_scheduled_ci_run = os.environ.get("CI_WORKFLOW_REF") == target_workflow + amd_target_workflow = "huggingface/transformers/.github/workflows/self-scheduled-amd-caller.yml@refs/heads/main" + is_scheduled_ci_run = os.environ.get("CI_WORKFLOW_REF") in [target_workflow, amd_target_workflow] # Only the model testing job is concerned: this condition is to avoid other jobs to upload the empty list as # results. @@ -1233,7 +1238,7 @@ def prepare_reports(title, header, reports, to_truncate=True): api.upload_file( path_or_fileobj=f"ci_results_{job_name}/model_results.json", path_in_repo=f"{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_{job_name}/model_results.json", - repo_id="hf-internal-testing/transformers_daily_ci", + repo_id=REPORT_REPO_ID, repo_type="dataset", token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None), ) @@ -1255,7 +1260,7 @@ def prepare_reports(title, header, reports, to_truncate=True): api.upload_file( path_or_fileobj=f"ci_results_{job_name}/{test_to_result_name[job]}_results.json", path_in_repo=f"{datetime.datetime.today().strftime('%Y-%m-%d')}/ci_results_{job_name}/{test_to_result_name[job]}_results.json", - repo_id="hf-internal-testing/transformers_daily_ci", + repo_id=REPORT_REPO_ID, repo_type="dataset", token=os.environ.get("TRANSFORMERS_CI_RESULTS_UPLOAD_TOKEN", None), ) @@ -1276,6 +1281,7 @@ def prepare_reports(title, header, reports, to_truncate=True): ci_title, model_results, additional_results, + repo_id=REPORT_REPO_ID, selected_warnings=selected_warnings, prev_ci_artifacts=prev_ci_artifacts, ) @@ -1284,3 +1290,6 @@ def prepare_reports(title, header, reports, to_truncate=True): if message.n_failures or (ci_event != "push" and not ci_event.startswith("Push CI (AMD)")): message.post() message.post_reply() + + # if UPLOAD_REPORT_SUMMARY + # message.upload_to_repo()