Install dependencies for tests based on package install (#951)

* Install dependencies for tests based on package install

Parsons dependencies can be installed in two different ways.
`pip install -r requirements.txt` is equivalent to pip install parsons
UNLESS the environment variable `PARSONS_LIMITED_DEPENDENCIES` is set,
in which case the dependencies are installed based on the explicit
configuration in `setup.py`.

There can be drift between what is defined in requirements.txt and
what is defined in `setup.py`.

These changes ensure that both sets of dependencies are used to run
tests, ensuring alignment between test environments and actual usage.

* Fix package dependencies when using PARSONS_LIMITED_DEPENDENCIES

.github/workflows/test-linux-windows.yml

@@ -14,6 +14,7 @@ jobs:
     strategy:
       matrix:
         python-version: ['3.8', '3.9', '3.10']
+        limited-dependencies: ['','TRUE']
         os: [ubuntu-latest]   # add in windows-latest to add windows testing
         include:
         - os: ubuntu-latest
@@ -39,7 +40,11 @@ jobs:
          ${{ runner.os }}-pip-
 
     - name: Install dependencies
-      run: python -m pip install -r requirements.txt
+      env:
+        PARSONS_LIMITED_DEPENDENCIES: ${{ matrix.limited-dependencies }}
+      run: |
+        python -m pip install .[all]
+        python -m pip install -r requirements-dev.txt
 
     - name: Run tests
       run: pytest -rf test/

.github/workflows/tests-mac.yml

@@ -12,6 +12,9 @@ env:
 
 jobs:
   build:
+    strategy:
+      matrix:
+        limited-dependencies: ['','TRUE']    
     runs-on: macos-latest
 
     steps:
@@ -31,7 +34,11 @@ jobs:
           mac-pip-
 
     - name: Install dependencies
-      run: python -m pip install -r requirements.txt
+      env:
+        PARSONS_LIMITED_DEPENDENCIES: ${{ matrix.limited-dependencies }}      
+      run: |
+        python -m pip install .[all]
+        python -m pip install -r requirements-dev.txt
 
     - name: Run tests
       run: TESTING=1 pytest -rf test/

requirements-dev.txt

@@ -0,0 +1,8 @@
+# Testing Requirements
+requests-mock==1.5.2
+flake8==4.0.1
+black==22.12.0
+testfixtures==6.18.5
+pytest==7.1.1
+pytest-datadir==1.3.0
+pytest-mock>=3.0.0

requirements.txt

@@ -40,15 +40,6 @@ SQLAlchemy==1.3.23
 requests_oauthlib==1.3.0
 bs4==0.0.1
 
-# Testing Requirements
-requests-mock==1.5.2
-flake8==4.0.1
-black==22.12.0
-testfixtures==6.18.5
-pytest==7.1.1
-pytest-datadir==1.3.0
-pytest-mock>=3.0.0
-
 # Stuff for TMC scripts
 # TODO Remove when we have a TMC-specific Docker image
 selenium==3.141.0

setup.py

@@ -14,15 +14,15 @@ def main():
             "simplejson",
         ]
         extras_require = {
-            "airtable": ["airtable-python-wrapper"],
+            "airtable": ["airtable-python-wrapper==0.13.0"],
             "alchemer": ["surveygizmo"],
             "azure": ["azure-storage-blob"],
             "box": ["boxsdk"],
             "braintree": ["braintree"],
             "catalist": ["paramiko"],
             "civis": ["civis"],
             "facebook": ["joblib", "facebook-business"],
-            "geocode": ["censusgeocode"],
+            "geocode": ["censusgeocode", "urllib3==1.26.18"],
             "github": ["PyGitHub"],
             "google": [
                 "apiclient",
@@ -37,6 +37,7 @@ def main():
             "mysql": ["mysql-connector-python", "SQLAlchemy"],
             "newmode": ["newmode"],
             "ngpvan": ["suds-py3"],
+            "mobilecommons": ["bs4"],
             "postgres": ["psycopg2-binary", "SQLAlchemy"],
             "redshift": ["boto3", "psycopg2-binary", "SQLAlchemy"],
             "s3": ["boto3"],