full extraction with ms3 v2.4.0 (#52)

.github/workflows/helper.py

@@ -0,0 +1,82 @@
+import argparse
+import re
+import os
+def create_new_tag(tag, update_major):
+    if not (re.match(r'^v\d+\.\d+$', tag)):
+        raise Exception(f'tag: {tag} is not giving in the correct format e.i v0.0')
+
+    # Notice that this could make a tag version of three digits become two digits
+    # e.i 3.2.1 -> 3.3
+    digits_tags = (re.match(r'^v\d+\.\d+', tag)).group()[1::].split('.')
+    if len(digits_tags) != 2:
+        raise Exception(f'tag: {tag} must contain two version digits')
+
+    major_num = int(digits_tags[0])
+    minor_num = int(digits_tags[1])
+    if update_major:
+        print(f"Label detected to update major version")
+        major_num += 1
+        minor_num = 0
+    else:
+        minor_num += 1
+    return f"v{major_num}.{minor_num}"
+
+def store_tag(tag):
+    with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
+        print(f'new_tag={tag}', file=fh)
+
+def update_file_with_tag(f_name, old_tag, new_tag):
+    if os.path.isfile(f_name):
+        try:
+            with open(f_name, "r",encoding="utf-8") as f:
+                data = f.read()
+            data = data.replace(old_tag, new_tag)
+            with open(f_name, "w",encoding="utf-8") as f:
+                f.write(data)
+        except Exception as e:
+            print(e)
+    else:
+        print(f"Warning: {f_name} doest exist at the current path {os.getcwd()}")
+
+def main(args):
+    tag = args.tag
+    new_tag = "v2.0"
+    if not tag:
+        print(f"Warning: a latest release with a tag does not exist in current repository, starting from {new_tag}")
+    else:
+        new_tag = create_new_tag(tag,args.update_major_ver)
+        print(f"Repository with tag: {tag}, creating a new tag with: {new_tag}")
+        update_file_with_tag(".zenodo.json", tag, new_tag)
+        update_file_with_tag("CITATION.cff", tag, new_tag)
+        update_file_with_tag("README.md", tag, new_tag)
+    store_tag(new_tag)
+
+def run():
+    args = parser.parse_args()
+    main(args)
+
+
+def str_to_bool(value):
+    if value.lower() == "true":
+        return True
+    elif value.lower() == "false":
+        return False
+    else:
+        raise Exception(
+            f"Error: value {value} as argument is not accepted\n"
+            f"retry with true or false"
+        )
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument(
+        "--tag", type=str,
+        help="Require: latest tag",
+        required=True
+    )
+    parser.add_argument(
+        "--update_major_ver", type=str_to_bool,
+        help="Require: boolean to update the major tag number",
+        required=True
+    )
+    run()

.github/workflows/version_release.yml

@@ -0,0 +1,72 @@
+on:
+  pull_request:
+    types:
+      - closed
+
+jobs:
+  if_merged:
+    if: github.event.pull_request.merged == true
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Checkout corpus repository
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.MS3_BOT_TOKEN }}
+          ref: "${{ github.event.pull_request.base.ref }}"
+          submodules: recursive
+
+      - name: "Get latest tag version"
+        id: tag
+        continue-on-error: true
+        run: |
+            res=$(git tag -l --sort=-v:refname | grep --invert-match '\^' | head -n 1)
+            echo "tag_version=${res}" >> $GITHUB_OUTPUT
+        env:
+            GITHUB_TOKEN: ${{ secrets.MS3_BOT_TOKEN }}
+
+      - name: "Generate a new tag version"
+        id: generate_tag
+        run: |
+            major_in_PR="${{ contains(github.event.pull_request.labels.*.name, 'major_version')}}"
+            python .github/workflows/helper.py --tag "${{ steps.tag.outputs.tag_version }}" --update_major_ver "$major_in_PR"
+
+      - name: Setup Github credentials & push zenodo, citation and README changes
+        continue-on-error: true
+        run: |
+          git config --global user.name "ms3-bot"
+          git config --global user.email [email protected]
+          if [[ -f .zenodo.json ]]; then
+            git add .zenodo.json
+          fi
+          if [[ -f CITATION.cff ]]; then
+            git add CITATION.cff
+          fi
+          if [[ -f README.md ]]; then
+            git add README.md
+          fi          
+          git commit -m 'chore: files updated with tag: ${{ steps.generate_tag.outputs.new_tag }}'
+          git push
+
+      - name: "Create tag"
+        run: |
+          git tag -a "${{ steps.generate_tag.outputs.new_tag }}" -m "chore: files updated with tag: ${{ steps.generate_tag.outputs.new_tag }}"
+          git push origin "${{ steps.generate_tag.outputs.new_tag }}"
+
+      - name: "Get ms3 package & apply transform"
+        continue-on-error: true
+        run: |
+            pip install --upgrade pip
+            pip install ms3
+            ms3 transform -M -N -X -F -C -D
+
+      - uses: ncipollo/release-action@v1
+        with:
+          artifacts: "${{ github.event.repository.name }}.zip,\
+                      ${{ github.event.repository.name }}.datapackage.json,\
+                      ${{ github.event.repository.name }}.datapackage.errors"
+          body: "${{ github.event.pull_request.body }}"
+          name: "${{ github.event.pull_request.title }}"
+          tag: "${{ steps.generate_tag.outputs.new_tag }}"
+          makeLatest: "latest"

.pre-commit-config.yaml

@@ -0,0 +1,6 @@
+repos:
+-   repo: https://github.com/johentsch/ms3
+    rev: v2.4.0
+    hooks:
+    -   id: review
+        args: [-M, -N, -C, -X, -F, -D, -c LATEST_VERSION, --fail]