From bd89338fdcc706c29f456140749b46c779123381 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 16 Oct 2024 17:44:45 -0600 Subject: [PATCH 1/7] init --- .github/workflows/transfer_https.yaml | 38 +++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 .github/workflows/transfer_https.yaml diff --git a/.github/workflows/transfer_https.yaml b/.github/workflows/transfer_https.yaml new file mode 100644 index 0000000..b743694 --- /dev/null +++ b/.github/workflows/transfer_https.yaml @@ -0,0 +1,38 @@ +name: [PROOF OF CONCEPT - COPYURL HTTPS TO CLOUD STORAGE] + +env: + RUN_ID: ${{ github.run_id }} + # Type needs to be included as env variable! See https://rclone.org/docs/#config-file + RCLONE_CONFIG_SOURCE_TYPE: https + RCLONE_CONFIG_TARGET_TYPE: s3 + RCLONE_CONFIG_TARGET_ACCESS_KEY_ID: ${{ secrets.RCLONE_CONFIG_SOURCE_ACCESS_KEY_ID }} + RCLONE_CONFIG_TARGET_SECRET_ACCESS_KEY: ${{ secrets.RCLONE_CONFIG_SOURCE_SECRET_ACCESS_KEY}} + +on: + workflow_dispatch: + inputs: + url: + description: "url of the file to transfer" + required: true + +jobs: + list-bucket-and-copy: + runs-on: ubuntu-latest + steps: + - name: Setup Rclone + uses: AnimMouse/setup-rclone@v1 + with: + rclone_config: | + + [target] + type = s3 + provider = Ceph + endpoint = https://nyu1.osn.mghpcc.org + + disable_base64: true + + - run: > + rclone copyurl url "${{ github.event.inputs.url }}" + source:leap-pangeo-inbox/rclone-testing/ + --checksum --auto-filename + \ No newline at end of file From 31acd1318838bcc7c71fd21c5d8284473b85dc78 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 16 Oct 2024 17:46:07 -0600 Subject: [PATCH 2/7] lint --- .github/workflows/transfer_https.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/workflows/transfer_https.yaml b/.github/workflows/transfer_https.yaml index b743694..00d5726 100644 --- a/.github/workflows/transfer_https.yaml +++ b/.github/workflows/transfer_https.yaml @@ -1,4 +1,4 @@ -name: [PROOF OF CONCEPT - COPYURL HTTPS TO CLOUD STORAGE] +name: [rclone-https] env: RUN_ID: ${{ github.run_id }} @@ -35,4 +35,3 @@ jobs: rclone copyurl url "${{ github.event.inputs.url }}" source:leap-pangeo-inbox/rclone-testing/ --checksum --auto-filename - \ No newline at end of file From a91a0697bfe27a87b584029b6ab6b0d80d8cf999 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 16 Oct 2024 17:47:05 -0600 Subject: [PATCH 3/7] yaml lint --- .github/workflows/transfer_https.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/transfer_https.yaml b/.github/workflows/transfer_https.yaml index 00d5726..b4b5e2f 100644 --- a/.github/workflows/transfer_https.yaml +++ b/.github/workflows/transfer_https.yaml @@ -1,4 +1,4 @@ -name: [rclone-https] +name: rclone-https env: RUN_ID: ${{ github.run_id }} From e188dacb168f98decd7dce3ebba1e40edbf537d9 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 16 Oct 2024 17:48:52 -0600 Subject: [PATCH 4/7] tmp disable cat ci --- .github/workflows/catalog.yml | 230 +++++++++++++++++----------------- 1 file changed, 115 insertions(+), 115 deletions(-) diff --git a/.github/workflows/catalog.yml b/.github/workflows/catalog.yml index 2636c9a..e1097ab 100644 --- a/.github/workflows/catalog.yml +++ b/.github/workflows/catalog.yml @@ -1,115 +1,115 @@ -name: Catalog - -on: - pull_request: - branches: - workflow_dispatch: - repository_dispatch: - types: - - update-catalog - schedule: - - cron: "*/30 * * * *" # every 30 minutes. https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule - -permissions: - contents: write - pull-requests: write - -concurrency: - group: ${{ github.workflow }}-${{ github.ref }} - cancel-in-progress: true - -jobs: - validate-and-generate: - timeout-minutes: 30 - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - steps: - - name: Echo originating repository - if: github.event_name == 'repository_dispatch' - run: | - echo "Originating repository: ${{ github.event.client_payload.origin_repo }}" - - - uses: actions/checkout@v4 - - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - - name: Validate Feedstocks and Generate Catalog - uses: leap-stc/data-catalog-actions/leap-catalog@main - with: - generation-path: "catalog/input.yaml" - output-directory: "catalog/" - - - name: Upload artifacts - uses: actions/upload-artifact@v4 - with: - name: consolidated-web-catalog.json - path: catalog/output/consolidated-web-catalog.json - - update: - needs: validate-and-generate - runs-on: ubuntu-latest - defaults: - run: - shell: bash -l {0} - - environment: - name: ${{ github.event.pull_request.number }} - url: ${{ format('https://catalog-staging.leap.carbonplan.org/?catalog=https://raw.githubusercontent.com/{0}/{1}/catalog/output/consolidated-web-catalog.json', github.event.pull_request.head.repo.full_name, github.head_ref) }} - steps: - - uses: actions/checkout@v4 - with: - repository: ${{ github.event.pull_request.head.repo.full_name }} - ref: ${{ github.event.pull_request.head.ref }} - fetch-depth: 0 - ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} - - - name: Pull Remote Changes - run: | - git pull origin ${{ github.event.pull_request.head.ref }} --rebase --autostash - - - uses: actions/download-artifact@v4 - with: - name: consolidated-web-catalog.json - path: catalog/output - - - name: Commit and Push - run: | - git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" - git config --global user.name "github-actions[bot]" - python -m pip install pre-commit - pre-commit install - git add catalog/output/consolidated-web-catalog.json - # run pre-commit and return 0 to avoid failing the job - pre-commit run || true - git add catalog/output/consolidated-web-catalog.json || true - git commit -m "Auto-update consolidated-web-catalog.json" || true - - - name: Push changes - if: github.event_name == 'pull_request' - uses: ad-m/github-push-action@master - with: - # ssh: true - github_token: ${{ secrets.GITHUB_TOKEN }} - repository: ${{ github.event.pull_request.head.repo.full_name }} - branch: ${{ github.event.pull_request.head.ref }} - - - name: Create Pull Request - id: cpr - if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch') && github.ref == 'refs/heads/main' - uses: peter-evans/create-pull-request@v7 - with: - commit-message: "build: update catalog" - committer: GitHub - author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> - title: Update catalog - body: | - - Updates the consolidated web catalog for changes on `main` - - Auto-generated by [create-pull-request][1] - - [1]: https://github.com/peter-evans/create-pull-request - branch: update-consolidated-web-catalog - delete-branch: true +# name: Catalog + +# on: +# pull_request: +# branches: +# workflow_dispatch: +# repository_dispatch: +# types: +# - update-catalog +# schedule: +# - cron: "*/30 * * * *" # every 30 minutes. https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#schedule + +# permissions: +# contents: write +# pull-requests: write + +# concurrency: +# group: ${{ github.workflow }}-${{ github.ref }} +# cancel-in-progress: true + +# jobs: +# validate-and-generate: +# timeout-minutes: 30 +# runs-on: ubuntu-latest +# defaults: +# run: +# shell: bash -l {0} +# steps: +# - name: Echo originating repository +# if: github.event_name == 'repository_dispatch' +# run: | +# echo "Originating repository: ${{ github.event.client_payload.origin_repo }}" + +# - uses: actions/checkout@v4 + +# - uses: actions/setup-python@v5 +# with: +# python-version: "3.12" + +# - name: Validate Feedstocks and Generate Catalog +# uses: leap-stc/data-catalog-actions/leap-catalog@main +# with: +# generation-path: "catalog/input.yaml" +# output-directory: "catalog/" + +# - name: Upload artifacts +# uses: actions/upload-artifact@v4 +# with: +# name: consolidated-web-catalog.json +# path: catalog/output/consolidated-web-catalog.json + +# update: +# needs: validate-and-generate +# runs-on: ubuntu-latest +# defaults: +# run: +# shell: bash -l {0} + +# environment: +# name: ${{ github.event.pull_request.number }} +# url: ${{ format('https://catalog-staging.leap.carbonplan.org/?catalog=https://raw.githubusercontent.com/{0}/{1}/catalog/output/consolidated-web-catalog.json', github.event.pull_request.head.repo.full_name, github.head_ref) }} +# steps: +# - uses: actions/checkout@v4 +# with: +# repository: ${{ github.event.pull_request.head.repo.full_name }} +# ref: ${{ github.event.pull_request.head.ref }} +# fetch-depth: 0 +# ssh-key: ${{ secrets.SSH_PRIVATE_KEY }} + +# - name: Pull Remote Changes +# run: | +# git pull origin ${{ github.event.pull_request.head.ref }} --rebase --autostash + +# - uses: actions/download-artifact@v4 +# with: +# name: consolidated-web-catalog.json +# path: catalog/output + +# - name: Commit and Push +# run: | +# git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com" +# git config --global user.name "github-actions[bot]" +# python -m pip install pre-commit +# pre-commit install +# git add catalog/output/consolidated-web-catalog.json +# # run pre-commit and return 0 to avoid failing the job +# pre-commit run || true +# git add catalog/output/consolidated-web-catalog.json || true +# git commit -m "Auto-update consolidated-web-catalog.json" || true + +# - name: Push changes +# if: github.event_name == 'pull_request' +# uses: ad-m/github-push-action@master +# with: +# # ssh: true +# github_token: ${{ secrets.GITHUB_TOKEN }} +# repository: ${{ github.event.pull_request.head.repo.full_name }} +# branch: ${{ github.event.pull_request.head.ref }} + +# - name: Create Pull Request +# id: cpr +# if: (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || github.event_name == 'repository_dispatch') && github.ref == 'refs/heads/main' +# uses: peter-evans/create-pull-request@v7 +# with: +# commit-message: "build: update catalog" +# committer: GitHub +# author: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> +# title: Update catalog +# body: | +# - Updates the consolidated web catalog for changes on `main` +# - Auto-generated by [create-pull-request][1] + +# [1]: https://github.com/peter-evans/create-pull-request +# branch: update-consolidated-web-catalog +# delete-branch: true From 6344178676f609795ca3513a84f72cfdea507fa7 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 16 Oct 2024 17:51:02 -0600 Subject: [PATCH 5/7] wip --- .github/workflows/transfer_https.yaml | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/.github/workflows/transfer_https.yaml b/.github/workflows/transfer_https.yaml index b4b5e2f..3d81c7b 100644 --- a/.github/workflows/transfer_https.yaml +++ b/.github/workflows/transfer_https.yaml @@ -9,11 +9,12 @@ env: RCLONE_CONFIG_TARGET_SECRET_ACCESS_KEY: ${{ secrets.RCLONE_CONFIG_SOURCE_SECRET_ACCESS_KEY}} on: - workflow_dispatch: - inputs: - url: - description: "url of the file to transfer" - required: true + pull_request: + workflow_dispatch: + inputs: + url: + description: "url of the file to transfer" + required: true jobs: list-bucket-and-copy: From 4ed61a22dabe021452a1a3309b92371b3846efd5 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 16 Oct 2024 17:52:58 -0600 Subject: [PATCH 6/7] wip --- .github/workflows/transfer_https.yaml | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/.github/workflows/transfer_https.yaml b/.github/workflows/transfer_https.yaml index 3d81c7b..400139f 100644 --- a/.github/workflows/transfer_https.yaml +++ b/.github/workflows/transfer_https.yaml @@ -9,12 +9,11 @@ env: RCLONE_CONFIG_TARGET_SECRET_ACCESS_KEY: ${{ secrets.RCLONE_CONFIG_SOURCE_SECRET_ACCESS_KEY}} on: - pull_request: - workflow_dispatch: - inputs: - url: - description: "url of the file to transfer" - required: true + workflow_dispatch: + inputs: + inurl: + description: "input url" + required: true jobs: list-bucket-and-copy: @@ -33,6 +32,6 @@ jobs: disable_base64: true - run: > - rclone copyurl url "${{ github.event.inputs.url }}" + rclone copyurl url "${{ github.event.inputs.inurl }}" source:leap-pangeo-inbox/rclone-testing/ --checksum --auto-filename From 196114e42864fb40b06d175794374a022a7f5fb8 Mon Sep 17 00:00:00 2001 From: Raphael Hagen Date: Wed, 16 Oct 2024 17:53:12 -0600 Subject: [PATCH 7/7] wip --- .github/workflows/transfer_https.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/transfer_https.yaml b/.github/workflows/transfer_https.yaml index 400139f..73fa86d 100644 --- a/.github/workflows/transfer_https.yaml +++ b/.github/workflows/transfer_https.yaml @@ -16,7 +16,7 @@ on: required: true jobs: - list-bucket-and-copy: + transfer-http-to-osn: runs-on: ubuntu-latest steps: - name: Setup Rclone