From 00aaf7d0c5f35b832a60f66e8dda9ff741414238 Mon Sep 17 00:00:00 2001
From: jtyoung84 <104453205+jtyoung84@users.noreply.github.com>
Date: Mon, 1 Jul 2024 17:51:42 -0700
Subject: [PATCH] build: adds workflows for gitflow release cycle

---
 .github/workflows/init.yml                    |  52 -------------
 .github/workflows/publish_main.yml            |  38 ++++++++++
 .../{test_and_lint.yml => run_dev_tests.yml}  |   4 +-
 .github/workflows/run_main_tests.yml          |  43 +++++++++++
 .github/workflows/tag_and_publish.yml         |  31 --------
 README.md                                     |   4 +-
 {doc_template => docs}/Makefile               |   0
 {doc_template => docs}/make.bat               |   0
 .../source/_static/dark-logo.svg              |   0
 .../source/_static/favicon.ico                | Bin
 .../source/_static/light-logo.svg             |   0
 docs/source/aind_data_access_api.rst          |  69 ++++++++++++++++++
 {doc_template => docs}/source/conf.py         |   7 ++
 {doc_template => docs}/source/index.rst       |   0
 docs/source/modules.rst                       |   7 ++
 src/aind_data_access_api/credentials.py       |   1 +
 src/aind_data_access_api/document_db.py       |   4 +
 src/aind_data_access_api/document_store.py    |   1 +
 src/aind_data_access_api/rds_tables.py        |   1 +
 19 files changed, 175 insertions(+), 87 deletions(-)
 delete mode 100644 .github/workflows/init.yml
 create mode 100644 .github/workflows/publish_main.yml
 rename .github/workflows/{test_and_lint.yml => run_dev_tests.yml} (94%)
 create mode 100644 .github/workflows/run_main_tests.yml
 delete mode 100644 .github/workflows/tag_and_publish.yml
 rename {doc_template => docs}/Makefile (100%)
 rename {doc_template => docs}/make.bat (100%)
 rename {doc_template => docs}/source/_static/dark-logo.svg (100%)
 rename {doc_template => docs}/source/_static/favicon.ico (100%)
 rename {doc_template => docs}/source/_static/light-logo.svg (100%)
 create mode 100644 docs/source/aind_data_access_api.rst
 rename {doc_template => docs}/source/conf.py (88%)
 rename {doc_template => docs}/source/index.rst (100%)
 create mode 100644 docs/source/modules.rst

diff --git a/.github/workflows/init.yml b/.github/workflows/init.yml
deleted file mode 100644
index 0336013..0000000
--- a/.github/workflows/init.yml
+++ /dev/null
@@ -1,52 +0,0 @@
-# Workflow runs only once when the template is first used.
-# File can be safely deleted after repo is initialized.
-name: Initialize repository
-on:
-  push:
-    branches:
-      - main
-
-jobs:
-  initialize-package:
-    name: Initialize the package
-    if: ${{github.event.repository.name != 'aind-library-template'}}
-    runs-on: ubuntu-latest
-    env:
-      REPO_NAME: ${{ github.event.repository.name }}
-      GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v3
-    - name: Rename package
-      run: |
-        pkg_name=$(echo "${REPO_NAME}" | tr - _)
-        current_description='description = "Prints messages to stdout. Simple boilerplate for libraries."'
-        new_description='description = "Generated from aind-library-template"'
-        readme_description='Template for a minimal, basic repository for an AIND library.'
-        new_readme_description='Generated from aind-library-template'
-        echo "Package Name ${pkg_name}"
-        mkdir src/${pkg_name}
-        touch src/${pkg_name}/__init__.py
-        echo '"""Init package"""' >> src/${pkg_name}/__init__.py
-        echo '__version__ = "0.0.0"' >> src/${pkg_name}/__init__.py
-        sed -i "s/aind_library_template/${pkg_name}/" pyproject.toml
-        sed -i "s/aind-library-template/${REPO_NAME}/" pyproject.toml
-        sed -i "s/aind_library_template/${pkg_name}/" doc_template/source/conf.py
-        sed -i "s/${current_description}/${new_description}/" pyproject.toml
-        sed -i "/pandas/d" pyproject.toml
-        sed -i "s/aind-library-template/${REPO_NAME}/" README.md
-        sed -i "s/${readme_description}/${new_readme_description}/" README.md
-    - name: Commit changes
-      uses: EndBug/add-and-commit@v9
-      with:
-        default_author: github_actions
-        message: "ci: version bump [skip actions]"
-        add: '["pyproject.toml", "README.md", "src/*", "doc_template/source/conf.py"]'
-        remove: '["-r src/aind_library_template", "tests/test_message_handler.py"]'
-    - name: Add first tag
-      run: |
-        git tag v0.0.0
-        git push origin v0.0.0
-    - name: Disable workflow
-      run: |
-        gh workflow disable -R $GITHUB_REPOSITORY "${{ github.workflow }}"
diff --git a/.github/workflows/publish_main.yml b/.github/workflows/publish_main.yml
new file mode 100644
index 0000000..c4a6b8e
--- /dev/null
+++ b/.github/workflows/publish_main.yml
@@ -0,0 +1,38 @@
+name: Tag and publish main
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  tag_and_publish:
+    name: Parse version
+    runs-on: ubuntu-latest
+    outputs:
+      pkg_version: ${{ steps.output_version.outputs.pkg_version }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Get version from file
+      run: |
+        pkg_name=$(grep -P 'version = \{attr = .*\}' pyproject.toml | grep -oP '\w+.__version__')
+        init_file="./src/${pkg_name//.__version__}/__init__.py"
+        pkg_version=$(grep -Po '[0-9]+\.[0-9]+\.[0-9]+' "$init_file")
+        echo "docker_tag=$pkg_version" >> "$GITHUB_ENV"
+    - name: Create git tag
+      run: |
+        git tag "v${{ env.docker_tag }}"
+    - name: Push git tag
+      run: git push origin "v${{ env.docker_tag }}"
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        pip install --upgrade setuptools wheel twine build
+        python -m build
+        twine check dist/*
+    - name: Publish on PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.AIND_PYPI_TOKEN }}
diff --git a/.github/workflows/test_and_lint.yml b/.github/workflows/run_dev_tests.yml
similarity index 94%
rename from .github/workflows/test_and_lint.yml
rename to .github/workflows/run_dev_tests.yml
index c8d832d..47d3c5e 100644
--- a/.github/workflows/test_and_lint.yml
+++ b/.github/workflows/run_dev_tests.yml
@@ -1,9 +1,9 @@
-name: Lint and run tests
+name: Run checks in dev
 
 on:
   pull_request:
     branches:
-      - main
+      - dev
 
 jobs:
   ci:
diff --git a/.github/workflows/run_main_tests.yml b/.github/workflows/run_main_tests.yml
new file mode 100644
index 0000000..53bf8c7
--- /dev/null
+++ b/.github/workflows/run_main_tests.yml
@@ -0,0 +1,43 @@
+name: Run checks in main and release
+
+on:
+  pull_request:
+    branches:
+      - '*release*'
+      - main
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.8', '3.9', '3.10' ]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: | 
+        python -m pip install -e .[dev]
+    - name: Run linter checks
+      run: flake8 . && interrogate --verbose .
+    - name: Run tests and coverage
+      run: coverage run -m unittest discover && coverage report
+  verify_version:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Check version incremented
+        run: |
+          pkg_name=$(grep -P 'version = \{attr = .*\}' pyproject.toml | grep -oP '\w+.__version__')
+          init_file="./src/${pkg_name//.__version__}/__init__.py"
+          pkg_version=$(grep -Po '[0-9]+\.[0-9]+\.[0-9]+' "$init_file")
+          latest_tag=$(git ls-remote --tags --refs --sort="v:refname" | tail -n1 | sed 's/.*\///')
+          echo "Checking pkg_version v$pkg_version and latest_tag $latest_tag"
+          if [ "$latest_tag" == "v$pkg_version" ]
+          then
+            exit 1
+          fi
+          echo "Versions are different"
diff --git a/.github/workflows/tag_and_publish.yml b/.github/workflows/tag_and_publish.yml
deleted file mode 100644
index 63af1a6..0000000
--- a/.github/workflows/tag_and_publish.yml
+++ /dev/null
@@ -1,31 +0,0 @@
-name: Tag and publish
-on:
-  push:
-    branches:
-      - main
-jobs:
-  tag:
-    uses: AllenNeuralDynamics/aind-github-actions/.github/workflows/tag.yml@main
-    secrets:
-      SERVICE_TOKEN: ${{ secrets.SERVICE_TOKEN }}
-# Uncomment the following step block to publish to PYPI.
-  publish:
-    needs: tag
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v3
-      - name: Pull latest changes
-        run: git pull origin main
-      - name: Set up Python 3.8
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.8
-      - name: Install dependencies
-        run: |
-          pip install --upgrade setuptools wheel twine build
-          python -m build
-          twine check dist/*
-      - name: Publish on PyPI
-        uses: pypa/gh-action-pypi-publish@release/v1
-        with:
-          password: ${{ secrets.AIND_PYPI_TOKEN }}
diff --git a/README.md b/README.md
index f68d25d..5258bfe 100644
--- a/README.md
+++ b/README.md
@@ -186,10 +186,10 @@ The table below, from [semantic release](https://github.com/semantic-release/sem
 ### Documentation
 To generate the rst files source files for documentation, run
 ```bash
-sphinx-apidoc -o doc_template/source/ src 
+sphinx-apidoc -o docs/source/ src
 ```
 Then to create the documentation HTML files, run
 ```bash
-sphinx-build -b html doc_template/source/ doc_template/build/html
+sphinx-build -b html docs/source/ docs/build/html
 ```
 More info on sphinx installation can be found [here](https://www.sphinx-doc.org/en/master/usage/installation.html).
diff --git a/doc_template/Makefile b/docs/Makefile
similarity index 100%
rename from doc_template/Makefile
rename to docs/Makefile
diff --git a/doc_template/make.bat b/docs/make.bat
similarity index 100%
rename from doc_template/make.bat
rename to docs/make.bat
diff --git a/doc_template/source/_static/dark-logo.svg b/docs/source/_static/dark-logo.svg
similarity index 100%
rename from doc_template/source/_static/dark-logo.svg
rename to docs/source/_static/dark-logo.svg
diff --git a/doc_template/source/_static/favicon.ico b/docs/source/_static/favicon.ico
similarity index 100%
rename from doc_template/source/_static/favicon.ico
rename to docs/source/_static/favicon.ico
diff --git a/doc_template/source/_static/light-logo.svg b/docs/source/_static/light-logo.svg
similarity index 100%
rename from doc_template/source/_static/light-logo.svg
rename to docs/source/_static/light-logo.svg
diff --git a/docs/source/aind_data_access_api.rst b/docs/source/aind_data_access_api.rst
new file mode 100644
index 0000000..78cde06
--- /dev/null
+++ b/docs/source/aind_data_access_api.rst
@@ -0,0 +1,69 @@
+aind\_data\_access\_api package
+===============================
+
+Submodules
+----------
+
+aind\_data\_access\_api.credentials module
+------------------------------------------
+
+.. automodule:: aind_data_access_api.credentials
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.document\_db module
+-------------------------------------------
+
+.. automodule:: aind_data_access_api.document_db
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.document\_db\_ssh module
+------------------------------------------------
+
+.. automodule:: aind_data_access_api.document_db_ssh
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.document\_store module
+----------------------------------------------
+
+.. automodule:: aind_data_access_api.document_store
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.models module
+-------------------------------------
+
+.. automodule:: aind_data_access_api.models
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.rds\_tables module
+------------------------------------------
+
+.. automodule:: aind_data_access_api.rds_tables
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.secrets module
+--------------------------------------
+
+.. automodule:: aind_data_access_api.secrets
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: aind_data_access_api
+   :members:
+   :undoc-members:
+   :show-inheritance:
diff --git a/doc_template/source/conf.py b/docs/source/conf.py
similarity index 88%
rename from doc_template/source/conf.py
rename to docs/source/conf.py
index 826d2f4..a5a88c5 100644
--- a/doc_template/source/conf.py
+++ b/docs/source/conf.py
@@ -9,7 +9,14 @@
 from os.path import abspath, dirname
 from pathlib import Path
 
+# Due to a bug, autodoc raises errors in certain situations with pydantic
+# A workaround is to import the problematic modules here
 from aind_data_access_api import __version__ as package_version
+from aind_data_access_api import (  # noqa: F401
+    document_db_ssh,
+    document_store,
+    rds_tables,
+)
 
 INSTITUTE_NAME = "Allen Institute for Neural Dynamics"
 
diff --git a/doc_template/source/index.rst b/docs/source/index.rst
similarity index 100%
rename from doc_template/source/index.rst
rename to docs/source/index.rst
diff --git a/docs/source/modules.rst b/docs/source/modules.rst
new file mode 100644
index 0000000..7d13cd8
--- /dev/null
+++ b/docs/source/modules.rst
@@ -0,0 +1,7 @@
+src
+===
+
+.. toctree::
+   :maxdepth: 4
+
+   aind_data_access_api
diff --git a/src/aind_data_access_api/credentials.py b/src/aind_data_access_api/credentials.py
index 8a8d0bd..fc8740b 100644
--- a/src/aind_data_access_api/credentials.py
+++ b/src/aind_data_access_api/credentials.py
@@ -37,6 +37,7 @@ def settings_customise_sources(
         """
         Method to pull configs from a variety sources, such as a file or aws.
         Arguments are required and set by pydantic.
+
         Parameters
         ----------
         settings_cls : Type[BaseSettings]
diff --git a/src/aind_data_access_api/document_db.py b/src/aind_data_access_api/document_db.py
index df63444..988b51c 100644
--- a/src/aind_data_access_api/document_db.py
+++ b/src/aind_data_access_api/document_db.py
@@ -238,6 +238,7 @@ def retrieve_data_asset_records(
     ) -> List[DataAssetRecord]:
         """
         Retrieve data asset records
+
         Parameters
         ----------
         filter_query : Optional[dict]
@@ -369,8 +370,10 @@ def upsert_list_of_records(
         """
         Upsert a list of records. There's a limit to the size of the
         request that can be sent, so we chunk the requests.
+
         Parameters
         ----------
+
         data_asset_records : List[DataAssetRecord]
           List of records to upsert into the DocDB database
         max_payload_size : int
@@ -384,6 +387,7 @@ def upsert_list_of_records(
         -------
         List[Response]
           A list of responses from the API Gateway.
+
         """
         if len(data_asset_records) == 0:
             return []
diff --git a/src/aind_data_access_api/document_store.py b/src/aind_data_access_api/document_store.py
index 8a537d8..1eb58a6 100644
--- a/src/aind_data_access_api/document_store.py
+++ b/src/aind_data_access_api/document_store.py
@@ -82,6 +82,7 @@ def retrieve_data_asset_records(
         Can add a query to filter the records. For example:
         retrieve_data_asset_records(query = {"subject.subject_id":"646253"})
         will retrieve records for that specific subject_id.
+
         Parameters
         ----------
         query : dict
diff --git a/src/aind_data_access_api/rds_tables.py b/src/aind_data_access_api/rds_tables.py
index 1256d94..5a2b5a1 100644
--- a/src/aind_data_access_api/rds_tables.py
+++ b/src/aind_data_access_api/rds_tables.py
@@ -151,6 +151,7 @@ def read_table(
     ) -> pd.DataFrame:
         """
         Import sql table as a pandas dataframe.
+
         Parameters
         ----------
         table_name : str