build: adds workflows for gitflow release cycle

.github/workflows/publish_main.yml

@@ -0,0 +1,38 @@
+name: Tag and publish main
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  tag_and_publish:
+    name: Parse version
+    runs-on: ubuntu-latest
+    outputs:
+      pkg_version: ${{ steps.output_version.outputs.pkg_version }}
+    steps:
+    - uses: actions/checkout@v3
+    - name: Get version from file
+      run: |
+        pkg_name=$(grep -P 'version = \{attr = .*\}' pyproject.toml | grep -oP '\w+.__version__')
+        init_file="./src/${pkg_name//.__version__}/__init__.py"
+        pkg_version=$(grep -Po '[0-9]+\.[0-9]+\.[0-9]+' "$init_file")
+        echo "docker_tag=$pkg_version" >> "$GITHUB_ENV"
+    - name: Create git tag
+      run: |
+        git tag "v${{ env.docker_tag }}"
+    - name: Push git tag
+      run: git push origin "v${{ env.docker_tag }}"
+    - name: Set up Python 3.8
+      uses: actions/setup-python@v2
+      with:
+        python-version: 3.8
+    - name: Install dependencies
+      run: |
+        pip install --upgrade setuptools wheel twine build
+        python -m build
+        twine check dist/*
+    - name: Publish on PyPI
+      uses: pypa/gh-action-pypi-publish@release/v1
+      with:
+        password: ${{ secrets.AIND_PYPI_TOKEN }}

.github/workflows/test_and_lint.yml → .github/workflows/run_dev_tests.yml

@@ -1,9 +1,9 @@
-name: Lint and run tests
+name: Run checks in dev
 
 on:
   pull_request:
     branches:
-      - main
+      - dev
 
 jobs:
   ci:

.github/workflows/run_main_tests.yml

@@ -0,0 +1,43 @@
+name: Run checks in main and release
+
+on:
+  pull_request:
+    branches:
+      - '*release*'
+      - main
+
+jobs:
+  ci:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [ '3.8', '3.9', '3.10' ]
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v3
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: | 
+        python -m pip install -e .[dev]
+    - name: Run linter checks
+      run: flake8 . && interrogate --verbose .
+    - name: Run tests and coverage
+      run: coverage run -m unittest discover && coverage report
+  verify_version:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+      - name: Check version incremented
+        run: |
+          pkg_name=$(grep -P 'version = \{attr = .*\}' pyproject.toml | grep -oP '\w+.__version__')
+          init_file="./src/${pkg_name//.__version__}/__init__.py"
+          pkg_version=$(grep -Po '[0-9]+\.[0-9]+\.[0-9]+' "$init_file")
+          latest_tag=$(git ls-remote --tags --refs --sort="v:refname" | tail -n1 | sed 's/.*\///')
+          echo "Checking pkg_version v$pkg_version and latest_tag $latest_tag"
+          if [ "$latest_tag" == "v$pkg_version" ]
+          then
+            exit 1
+          fi
+          echo "Versions are different"

README.md

@@ -186,10 +186,10 @@ The table below, from [semantic release](https://github.com/semantic-release/sem
 ### Documentation
 To generate the rst files source files for documentation, run
 ```bash
-sphinx-apidoc -o doc_template/source/ src 
+sphinx-apidoc -o docs/source/ src
 ```
 Then to create the documentation HTML files, run
 ```bash
-sphinx-build -b html doc_template/source/ doc_template/build/html
+sphinx-build -b html docs/source/ docs/build/html
 ```
 More info on sphinx installation can be found [here](https://www.sphinx-doc.org/en/master/usage/installation.html).

docs/source/aind_data_access_api.rst

@@ -0,0 +1,69 @@
+aind\_data\_access\_api package
+===============================
+
+Submodules
+----------
+
+aind\_data\_access\_api.credentials module
+------------------------------------------
+
+.. automodule:: aind_data_access_api.credentials
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.document\_db module
+-------------------------------------------
+
+.. automodule:: aind_data_access_api.document_db
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.document\_db\_ssh module
+------------------------------------------------
+
+.. automodule:: aind_data_access_api.document_db_ssh
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.document\_store module
+----------------------------------------------
+
+.. automodule:: aind_data_access_api.document_store
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.models module
+-------------------------------------
+
+.. automodule:: aind_data_access_api.models
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.rds\_tables module
+------------------------------------------
+
+.. automodule:: aind_data_access_api.rds_tables
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+aind\_data\_access\_api.secrets module
+--------------------------------------
+
+.. automodule:: aind_data_access_api.secrets
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
+Module contents
+---------------
+
+.. automodule:: aind_data_access_api
+   :members:
+   :undoc-members:
+   :show-inheritance:

doc_template/source/conf.py → docs/source/conf.py

@@ -9,7 +9,14 @@
 from os.path import abspath, dirname
 from pathlib import Path
 
+# Due to a bug, autodoc raises errors in certain situations with pydantic
+# A workaround is to import the problematic modules here
 from aind_data_access_api import __version__ as package_version
+from aind_data_access_api import (  # noqa: F401
+    document_db_ssh,
+    document_store,
+    rds_tables,
+)
 
 INSTITUTE_NAME = "Allen Institute for Neural Dynamics"
 

docs/source/modules.rst

@@ -0,0 +1,7 @@
+src
+===
+
+.. toctree::
+   :maxdepth: 4
+
+   aind_data_access_api

src/aind_data_access_api/credentials.py

@@ -37,6 +37,7 @@ def settings_customise_sources(
         """
         Method to pull configs from a variety sources, such as a file or aws.
         Arguments are required and set by pydantic.
+
         Parameters
         ----------
         settings_cls : Type[BaseSettings]

src/aind_data_access_api/document_db.py

@@ -238,6 +238,7 @@ def retrieve_data_asset_records(
     ) -> List[DataAssetRecord]:
         """
         Retrieve data asset records
+
         Parameters
         ----------
         filter_query : Optional[dict]
@@ -369,8 +370,10 @@ def upsert_list_of_records(
         """
         Upsert a list of records. There's a limit to the size of the
         request that can be sent, so we chunk the requests.
+
         Parameters
         ----------
+
         data_asset_records : List[DataAssetRecord]
           List of records to upsert into the DocDB database
         max_payload_size : int
@@ -384,6 +387,7 @@ def upsert_list_of_records(
         -------
         List[Response]
           A list of responses from the API Gateway.
+
         """
         if len(data_asset_records) == 0:
             return []

src/aind_data_access_api/document_store.py

@@ -82,6 +82,7 @@ def retrieve_data_asset_records(
         Can add a query to filter the records. For example:
         retrieve_data_asset_records(query = {"subject.subject_id":"646253"})
         will retrieve records for that specific subject_id.
+
         Parameters
         ----------
         query : dict

src/aind_data_access_api/rds_tables.py

@@ -151,6 +151,7 @@ def read_table(
     ) -> pd.DataFrame:
         """
         Import sql table as a pandas dataframe.
+
         Parameters
         ----------
         table_name : str