Test Template

.github/workflows/correctness.yaml

@@ -25,5 +25,3 @@ jobs:
         run: poetry run pylint ./tds
       - name: Pylint alembic
         run: poetry run pylint ./migrate
-      - name: Pytest
-        run: poetry run pytest

.github/workflows/publish.yaml

@@ -18,6 +18,9 @@ jobs:
   formatting:
     uses: ./.github/workflows/formatting.yaml
 
+  test:
+    uses: ./.github/workflows/test.yaml
+
   # Call the Tag Generator to generate an image tag to use
   tag-generator:
     uses: darpa-askem/.github/.github/workflows/tag-generator.yaml@main
@@ -27,9 +30,9 @@ jobs:
   # Build and Publish all targets associated with specified group
   bake:
     needs:
-  # Temorarily remove these to allow build while we fix testing
-  #      - correctness
-  #      - formatting
+      - correctness
+      - formatting
+      - test
       - tag-generator
     uses: darpa-askem/.github/.github/workflows/bake-publish.yml@main
     with:

.github/workflows/test.yaml

@@ -0,0 +1,25 @@
+---
+name: Test
+on:
+  workflow_call:
+  push:
+    branches: ['main']
+  pull_request:
+    branches: ['main']
+
+jobs:
+  test:
+    runs-on: ubuntu-22.04
+    steps:
+      - uses: actions/checkout@v1
+      - name: Set up test.
+        run: |
+          cp api.env.sample api.env
+          cp docker/overrides/docker-compose.testing.override.yml docker-compose.override.yml
+      - name: Run Test Suite
+        run: |
+          docker-compose --env-file api.env --profile tds-testing up --abort-on-container-exit --build
+
+      - name: Clean Up
+        if: always()
+        run: docker-compose --env-file api.env --profile tds-testing down

.gitignore

@@ -34,4 +34,5 @@ api.env
 # Minio
 .minio.sys
 data/askem-staging-data-service
-migrate/seeds/files/*.csv
+migrate/seeds/files/*.csv
+docker-compose.override.yml

api.env.sample

@@ -1,3 +1,4 @@
+TDS_HOST_PORT=8001
 SQL_URL=rdb
 SQL_PORT=5432
 SQL_USER=dev

docker-compose.yml

@@ -7,24 +7,6 @@ networks:
     driver: bridge
     name: data-annotation-stack
 services:
-  rdb:
-    container_name: data-service-rdb
-    image: "postgres:15.1"
-    ports:
-      - 8032:$SQL_PORT
-    environment:
-      - POSTGRES_PASSWORD=$SQL_PASSWORD
-      - POSTGRES_USER=$SQL_USER
-      - POSTGRES_DB=$SQL_DB
-    volumes:
-      - tds_data:/var/lib/postgresql/data
-    networks:
-      - data-api
-    healthcheck:
-      test: [ "CMD-SHELL", "pg_isready -U ${SQL_USER} -d ${SQL_DB}" ]
-      interval: 10s
-      timeout: 5s
-      retries: 5
   api:
     container_name: data-service-api
     build:
@@ -39,51 +21,8 @@ services:
     networks:
       - data-annotation-stack
       - data-api
-    depends_on:
-      migrations:
-        condition: service_completed_successfully
-      graphdb:
-        condition: service_started
-      minio:
-        condition: service_started
-    volumes:
-      - $PWD/tds:/api/tds
-      - $PWD/migrate:/api/migrate
-  graphdb:
-    build:
-      context: ./
-      dockerfile: docker/Dockerfile.neo4j
-    ports:
-      - 7474:7474
-      - 7687:7687
-    environment:
-      - NEO4J_AUTH=$NEO4J_AUTH
-      - NEO4J_dbms_memory_pagecache_size=512M
-    volumes:
-      - neo4j_data:/data
-    depends_on:
-      rdb:
-        condition: service_healthy
-      migrations:
-        condition: service_completed_successfully
-    networks:
-      - data-api
-  minio:
-    build:
-      context: ./
-      dockerfile: docker/Dockerfile.minio
-    environment:
-      MINIO_ROOT_USER: $MINIO_USER
-      MINIO_ROOT_PASSWORD: $MINIO_PWD
-    ports:
-      - 9000:9000
-      - 9001:9001
-    volumes:
-      - $PWD/data:/data
-    networks:
-      - data-api
-      - data-annotation-stack
   elasticsearch:
+    profiles: ["local", "tds-testing"]
     image: elasticsearch:${STACK_VERSION}
     environment:
       - discovery.type=single-node
@@ -100,42 +39,17 @@ services:
     healthcheck:
       test: curl http://localhost:9200/_cluster/health?wait_for_status=yellow || exit 1
       interval: 10s
-      timeout: 20s
-      retries: 5
-  kibana:
-    depends_on:
-      - elasticsearch
-    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
-    volumes:
-      - kibanadata:/usr/share/kibana/data
-    ports:
-      - ${KIBANA_PORT}:5601
-    environment:
-      - SERVERNAME=kibana
-      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
-      - ELASTICSEARCH_USERNAME=kibana_system
-      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
-    mem_limit: ${MEM_LIMIT}
-    networks:
-      - data-api
-      - data-annotation-stack
+      timeout: 10s
+      retries: 10
   migrations:
+    profiles: ["tds-api-only", "tds-api", "local"]
     container_name: data-service-migrations
     build:
       context: ./
       dockerfile: docker/Dockerfile.migrations
     entrypoint: /run_migrations.sh
     env_file:
       - api.env
-#    volumes:
-#      - $PWD/migrate:/migrate
-    depends_on:
-      rdb:
-        condition: service_healthy
-      elasticsearch:
-        condition: service_healthy
-      minio:
-        condition: service_started
     networks:
       - data-api
 volumes:

docker/Dockerfile

@@ -6,14 +6,10 @@ WORKDIR /api
 ADD poetry.lock poetry.lock
 ADD pyproject.toml pyproject.toml
 RUN poetry config virtualenvs.create false
-RUN poetry install --no-root
-COPY tests tests
-COPY tds tds 
+COPY tds tds
 COPY graph_relations.json graph_relations.json
+RUN mkdir /api/tests && touch /api/tests/test.py
+RUN poetry install --no-dev
 
-# Poetry complains if the README doesn't exist
-COPY README.md README.md
-COPY migrate migrate
-RUN poetry install --only-root
 EXPOSE 8000
 CMD ["poetry", "run", "tds"]

docker/Dockerfile.test

@@ -0,0 +1,17 @@
+FROM python:3.10
+RUN apt update 2> /dev/null
+RUN apt install -y postgresql postgresql-contrib sqlite3
+RUN pip install poetry
+WORKDIR /api
+ADD poetry.lock poetry.lock
+ADD pyproject.toml pyproject.toml
+RUN poetry config virtualenvs.create false
+COPY tests tests
+COPY tds tds
+COPY tds /tds_package/tds
+RUN find tds_package/tds/modules \! -name "model.py" -mindepth 2 -maxdepth 2 -print | xargs rm -rf
+COPY graph_relations.json graph_relations.json
+
+COPY migrate migrate
+COPY docker/scripts/tds_test.sh tds_test.sh
+RUN poetry install

docker/overrides/docker-compose.local.override.yml

@@ -0,0 +1,100 @@
+version: "3.9"
+services:
+  rdb:
+    profiles: [ "tds-api", "local" ]
+    container_name: data-service-rdb
+    image: "postgres:15.1"
+    ports:
+      - 8032:$SQL_PORT
+    environment:
+      - POSTGRES_PASSWORD=$SQL_PASSWORD
+      - POSTGRES_USER=$SQL_USER
+      - POSTGRES_DB=$SQL_DB
+    volumes:
+      - tds_data:/var/lib/postgresql/data
+    networks:
+      - data-api
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U ${SQL_USER} -d ${SQL_DB}" ]
+      interval: 10s
+      timeout: 5s
+      retries: 5
+  minio:
+    profiles: [ "local" ]
+    build:
+      context: ./
+      dockerfile: docker/Dockerfile.minio
+    environment:
+      MINIO_ROOT_USER: $MINIO_USER
+      MINIO_ROOT_PASSWORD: $MINIO_PWD
+    ports:
+      - 9000:9000
+      - 9001:9001
+    volumes:
+      - $PWD/data:/data
+    networks:
+      - data-api
+      - data-annotation-stack
+  graphdb:
+    profiles: [ "tds-api", "local" ]
+    build:
+      context: ./
+      dockerfile: docker/Dockerfile.neo4j
+    ports:
+      - 7474:7474
+      - 7687:7687
+    environment:
+      - NEO4J_AUTH=$NEO4J_AUTH
+      - NEO4J_dbms_memory_pagecache_size=512M
+    volumes:
+      - neo4j_data:/data
+    depends_on:
+      rdb:
+        condition: service_healthy
+      migrations:
+        condition: service_completed_successfully
+    networks:
+      - data-api
+  migrations:
+    depends_on:
+      rdb:
+        condition: service_healthy
+      elasticsearch:
+        condition: service_healthy
+      minio:
+        condition: service_started
+    volumes:
+      - $PWD/migrate:/migrate
+  api:
+    depends_on:
+      migrations:
+        condition: service_completed_successfully
+      graphdb:
+        condition: service_started
+      minio:
+        condition: service_started
+    volumes:
+      - $PWD/tests:/api/tests
+      - $PWD/tds:/api/tds
+      - $PWD/migrate:/api/migrate
+  elasticsearch:
+    volumes:
+      - elasticsearch_data:/usr/share/elasticsearch/data
+  kibana:
+    profiles: [ "local" ]
+    depends_on:
+      - elasticsearch
+    image: docker.elastic.co/kibana/kibana:${STACK_VERSION}
+    volumes:
+      - kibanadata:/usr/share/kibana/data
+    ports:
+      - ${KIBANA_PORT}:5601
+    environment:
+      - SERVERNAME=kibana
+      - ELASTICSEARCH_HOSTS=http://elasticsearch:9200
+      - ELASTICSEARCH_USERNAME=kibana_system
+      - ELASTICSEARCH_PASSWORD=${KIBANA_PASSWORD}
+    mem_limit: ${MEM_LIMIT}
+    networks:
+      - data-api
+      - data-annotation-stack

docker/overrides/docker-compose.testing.override.yml

@@ -0,0 +1,16 @@
+version: "3.9"
+services:
+  api:
+    container_name: data-service-api-testing
+    build:
+      context: ./
+      dockerfile: docker/Dockerfile.test
+    environment:
+      - NEO4J_ENABLED
+      - SQL_CONN_STR=sqlite:///test.db
+      - SEED_DATA=false
+      - SQL_NOW_STATEMENT=CURRENT_TIMESTAMP
+    depends_on:
+      elasticsearch:
+        condition: service_healthy
+    entrypoint: ["bash", "/api/tds_test.sh"]

docker/scripts/tds_test.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+
+alembic -c migrate/alembic.ini upgrade head
+python migrate/scripts/start_elasticsearch.py
+pytest tests/
+#touch /logger.log
+#tail -f /logger.log

migrate/env.py

@@ -38,6 +38,9 @@ def setup_context() -> str:
     url = config.get_main_option("sqlalchemy.url")
     url = re.sub(r"\${(.+?)}", lambda m: tokens[m.group(1)], url)
 
+    if os.getenv("SQL_CONN_STR", None):
+        url = os.getenv("SQL_CONN_STR")
+
     return url