Merge pull request #67 from PrediktorAS/feature/test-coverage-integra…

…tion-functional-performance

Feature/test coverage integration functional performance

.coveragerc

@@ -26,3 +26,5 @@ exclude_lines =
     # Don't complain if non-runnable code isn't run:
     if 0:
     if __name__ == .__main__.:
+
+show_missing = True

.github/workflows/ci-cd-webhook.yml

@@ -0,0 +1,87 @@
+name: CI/CD and Webhook
+
+on: [push, pull_request]
+
+jobs:
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+    - uses: actions/checkout@v4
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: 3.12
+
+    - name: Create virtual environment
+      run: python -m venv .venv
+
+    - name: Activate virtual environment
+      run: source .venv/bin/activate
+
+    - name: Install dependencies
+      run: pip install -r requirements.txt
+
+    - name: Run pre-commit checks
+      run: pre-commit run --all-files
+
+  webhook:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Install Google Cloud CLI
+        run: |
+          echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | sudo tee -a /etc/apt/sources.list.d/google-cloud-sdk.list
+          sudo apt-get install apt-transport-https ca-certificates gnupg
+          curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | sudo apt-key --keyring /usr/share/keyrings/cloud.google.gpg add -
+          sudo apt-get update && sudo apt-get install google-cloud-cli
+
+      - name: Authenticate with Google Cloud
+        run: |
+          echo '${{ secrets.GCP_SA_KEY }}' > sa-key.json
+          gcloud auth activate-service-account sa-git-data-sources@tgs-prediktor-dev-598e.iam.gserviceaccount.com --key-file=sa-key.json
+          rm sa-key.json
+
+      - name: Generate and send identity token
+        run: |
+          IDENTITY_TOKEN=$(gcloud auth print-identity-token --audiences="https://us-central1-tgs-prediktor-dev-598e.cloudfunctions.net/git_webhook_to_bigquery")
+          WEBHOOK_URL="https://us-central1-tgs-prediktor-dev-598e.cloudfunctions.net/git_webhook_to_bigquery"
+
+          prepare_payload() {
+            local event_type="$1"
+            jq -n \
+              --arg event "$event_type" \
+              --arg repository "${{ github.repository }}" \
+              --arg url "https://github.com/${{ github.repository }}" \
+              --arg actor "${{ github.actor }}" \
+              --arg ref "${{ github.ref }}" \
+              --argjson payload "$2" \
+              '{
+                event: $event,
+                repository: { name: $repository, url: $url },
+                actor: $actor,
+                ref: $ref,
+                payload: $payload
+              }'
+          }
+
+          case "${{ github.event_name }}" in
+            push)
+              PAYLOAD=$(prepare_payload "push" "$(echo '${{ toJson(github.event) }}' | jq '{commits: .commits, before: .before, after: .after, pusher: .pusher}')")
+              ;;
+            pull_request)
+              PAYLOAD=$(prepare_payload "pull_request" "$(echo '${{ toJson(github.event.pull_request) }}' | jq '{number: .number, title: .title, state: .state, body: .body}')")
+              ;;
+            *)
+              PAYLOAD=$(prepare_payload "${{ github.event_name }}" "{}")
+              ;;
+          esac
+
+          curl -H "Authorization: Bearer $IDENTITY_TOKEN" \
+               -H "Content-Type: application/json" \
+               -d "$PAYLOAD" \
+               $WEBHOOK_URL

.github/workflows/codeql-analysis.yml

@@ -48,11 +48,11 @@ jobs:
         # If you wish to specify custom queries, you can do so here or in a config file.
         # By default, queries listed here will override any specified in a config file.
         # Prefix the list here with "+" to use these queries and those in the config file.
-        
+
         # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
         # queries: security-extended,security-and-quality
 
-        
+
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
@@ -61,7 +61,7 @@ jobs:
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
 
-    #   If the Autobuild fails above, remove it and uncomment the following three lines. 
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
     #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
 
     # - run: |

.github/workflows/pages.yml

@@ -21,4 +21,4 @@ jobs:
       uses: ad-m/github-push-action@master
       with:
         github_token: ${{ secrets.GITHUB_TOKEN }}
-        branch: gh-pages
+        branch: gh-pages

.github/workflows/tox.yml

@@ -77,4 +77,4 @@ jobs:
 
     - name: Test with tox
       run: |
-        tox
+        tox

.gitignore

@@ -58,4 +58,4 @@ venv*/
 # data folders
 *data*/
 *json
-*parquet
+*parquet

.pre-commit-config.yaml

@@ -0,0 +1,50 @@
+default_language_version:
+    python: python3.12
+
+repos:
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.6.0
+    hooks:
+    -   id: trailing-whitespace
+    -   id: end-of-file-fixer
+    -   id: check-yaml
+    -   id: check-added-large-files
+    -   id: requirements-txt-fixer
+    -   id: debug-statements
+    -   id: name-tests-test
+
+-   repo: https://github.com/PyCQA/docformatter
+    rev: v1.7.5
+    hooks:
+    -   id: docformatter
+        args: ["--in-place"]
+
+-   repo: https://github.com/PyCQA/autoflake
+    rev: v2.3.1
+    hooks:
+    -   id: autoflake
+        args: ["--remove-all-unused-imports", "--in-place"]
+
+-   repo: https://github.com/psf/black
+    rev: 24.4.2
+    hooks:
+      - id: black
+        args: ["--line-length", "79"]
+
+-   repo: https://github.com/PyCQA/flake8
+    rev: 6.1.0
+    hooks:
+    -   id: flake8
+        args: ["--ignore=E501,W503"]
+
+-   repo: https://github.com/PyCQA/bandit
+    rev: 1.7.5
+    hooks:
+    -   id: bandit
+        args: ["-x", "tests/*"]
+
+-   repo: https://github.com/pre-commit/mirrors-isort
+    rev: v5.9.0
+    hooks:
+    -   id: isort
+        args: ["--profile", "black", "--line-length", "79"]

README.md

@@ -40,35 +40,80 @@ source .venv/bin/activate
 pip install -r requirements.txt
 ```
 
+## Pre-commit Hooks
+
+To ensure code quality and maintain a consistent coding standard, pre-commit hooks have been added to the repository. All the pre-commit checks must be passed before committing your changes. Follow the instructions below to install and use them.
+
+4. Activate pre-commit hooks
+After installing the requirements, activate the pre-commit hooks by running:
+
+```
+pre-commit install
+```
+
+5. Run pre-commit hooks manually
+It's recommended to run the pre-commit hooks before making a commit to catch issues early. Use the following command to run all the hooks manually:
+
+```
+pre-commit run --all-files
+```
+
+6. Stage your changes
+After making the required changes and ensuring they pass the pre-commit hooks, add the changes to the staging area before creating a commit:
+
+```
+git add .
+```
+
+## isort for Import Sorting
+
+To maintain a clean and organized codebase, we use isort to automatically sort Python import statements according to PEP 8 standards.
+
+7. Run isort
+
+Run `isort` manually to sort the imports in the project using the command:
+
+```
+isort .
+```
+
+However, applying these changes could lead to conflicts. To verify the import order without making any modifications, use the following command:
+
+```
+isort . --check-only
+```
+
+The `isort` configuration is managed in the `pyproject.toml` file to ensure it integrates well with black and other tools.
+
+
 ## Run and build
 
-4. Run tests
+8. Run tests
 
 ```
 tox
 ```
 
-5. Do your changes
-   Add your changes and create a new PR to be approved.
+9. Make your changes and submit a new PR for approval.
 
-6. Build
+10. Build
 
 ```
 tox -e build
 ```
 
 ## Changes
 
-7. Please apply your changes. If they will facilitate the work of the person using pyPrediktorMapClient, especially the new features you've implemented, ensure that you describe your changes comprehensively and provide guidance in the README.md file under the chapter `Manual - How to Use` (check below).
+11. Please apply your changes. If they will facilitate the work of the person using pyPrediktorMapClient, especially the new features you've implemented, ensure that you describe your changes comprehensively and provide guidance in the README.md file under the chapter `Manual - How to Use` (check below).
 
-8. Commit your changes to a new branch, push and create a new pull request for review.
+12. Commit your changes to a new branch, push and create a new pull request for review.
 
 ## Publish on PyPi
 
-9. Open [https://pypi.org/](https://pypi.org/) and log in.
-10. Open [https://pypi.org/manage/account/](https://pypi.org/manage/account/) and generate a new API token but only if you don't have one already. Keep the API key on your local machine because once generated it will be visible only once. Delete API keys that are no longer used!
-11. In your code editor, open the root directory of the current project and clean the content of folder `dist`.
-12. Create a new tag and push it to the GitHub repository. For instance, if the latest tag is `0.6.7` the new tag should be `0.6.8`. Alternatively, if the changes are major, you can set the new tag to `0.7.0`.
+13. Open [https://pypi.org/](https://pypi.org/) and log in.
+14. Open [https://pypi.org/manage/account/](https://pypi.org/manage/account/) and generate a new API token but only if you don't have one already. Keep the API key on your local machine because once generated it will be visible only once. Delete API keys that are no longer used!
+15. In your code editor, open the root directory of the current project and clean the content of folder `dist`.
+16. Create a new tag and push it to the GitHub repository. For instance, if the latest tag is `0.6.7` the new tag should be `0.6.8`. Alternatively, if the changes are major, you can set the new tag to `0.7.0`.
 
 Use the following commands to create the tag and to publish it.
 
@@ -77,13 +122,13 @@ git tag 0.6.8
 git push origin 0.6.8
 ```
 
-13. Create a new build. Be aware that `tox.ini` file is configured in a way to get latest tag from the repository. That tag is going to be used to label the new build.
+17. Create a new build. Be aware that `tox.ini` file is configured in a way to get latest tag from the repository. That tag is going to be used to label the new build.
 
 ```
 tox -e build
 ```
 
-14. Be sure that twine is installed. If not, run the following command:
+18. Be sure that twine is installed. If not, run the following command:
 
 ```
 python3 -m pip install twine
@@ -95,19 +140,19 @@ or
 python -m pip install twine
 ```
 
-15. Publish the build on PyPi:
+19. Publish the build on PyPi:
 
 ```
 python3 -m twine upload -u __token__ -p API_KEY dist/*
 ```
 
 Replace API_KEY with the API key you generated earlier or a key you already have.
 
-16. Check if the new version has been release - [Release history](https://pypi.org/project/pyPrediktorMapClient/#history).
+20. Check if the new version has been release - [Release history](https://pypi.org/project/pyPrediktorMapClient/#history).
 
 Once that's done, we have to publish a new release in the GitHub repository.
 
-17. Open the list of released published on the [GitHub repository](https://github.com/PrediktorAS/pyPrediktorMapClient/releases). Draft a new release by pressing "Draft a new release".
+21. Open the list of released published on the [GitHub repository](https://github.com/PrediktorAS/pyPrediktorMapClient/releases). Draft a new release by pressing "Draft a new release".
 
 Use the newly created tag which in our example is 0.6.8. Add detailed descrption about the new changes.
 

docs/conf.py

@@ -8,9 +8,8 @@
 # serve to show the default.
 
 import os
-import sys
 import shutil
-import sphinx_rtd_theme
+import sys
 
 # -- Path setup --------------------------------------------------------------
 
@@ -44,7 +43,9 @@
 try:
     import sphinx
 
-    cmd_line = f"sphinx-apidoc --implicit-namespaces -f -o {output_dir} {module_dir}"
+    cmd_line = (
+        f"sphinx-apidoc --implicit-namespaces -f -o {output_dir} {module_dir}"
+    )
 
     args = cmd_line.split(" ")
     if tuple(sphinx.__version__.split(".")) >= ("1", "7"):
@@ -106,7 +107,9 @@
     version = ""
 
 if not version or version.lower() == "unknown":
-    version = os.getenv("READTHEDOCS_VERSION", "unknown")  # automatically set by RTD
+    version = os.getenv(
+        "READTHEDOCS_VERSION", "unknown"
+    )  # automatically set by RTD
 
 release = version