Enable non-docker postgres (#1817)

* utilize docker compose --profile postgres, and POSTGRES_DOCKER var, to enable/disable using dockerized postgres

* update configuration.md

.github/workflows/cypress-tests.yml

@@ -27,7 +27,7 @@ jobs:
 
       - name: Build and start Docker containers
         run: |
-          docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env up -d --build
+          docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env --profile postgres up -d --build
 
       - name: Wait for application to be ready
         run: |
@@ -43,7 +43,7 @@ jobs:
             elapsed=$((current_time - start_time))
             if [ $elapsed -ge $timeout ]; then
               echo "Timeout waiting for application to be ready. Dumping logs:"
-              docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env logs
+              docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env --profile postgres logs
               exit 1
             fi
             sleep 5
@@ -72,4 +72,4 @@ jobs:
 
       - name: Stop Docker containers
         if: always()
-        run: docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env down
+        run: docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env --profile postgres down

.github/workflows/release-docker-images.yml

@@ -36,7 +36,7 @@ jobs:
           password: ${{ secrets.DOCKER_TOKEN }}
 
       - name: Build full project via docker-compose
-        run: docker compose build --parallel --build-arg GIT_HASH=${GITHUB_SHA:0:6}
+        run: docker compose --profile postgres build --parallel --build-arg GIT_HASH=${GITHUB_SHA:0:6}
 
       - name: Push images to Docker Hub
-        run: docker compose push --ignore-push-failures
+        run: docker compose --profile postgres push --ignore-push-failures

Makefile

@@ -8,23 +8,44 @@
 SHELL=/bin/bash
 E2E_RUN = cd e2e;
 
+define parse_env_value
+	$(shell grep -e ^$(1) ${ENV_FILE} | awk -F'[=]' '{gsub(/ /,"");print $$2}')
+endef
+
+define parse_env_bool
+  $(if $(filter false,$(1)),false,true)
+endef
+
+# Default environment and settings (dev)
 export ENV_FILE = .env
-export TAG = $(shell grep -e ^TAG ${ENV_FILE} | awk -F'[=]' '{gsub(/ /,""); print $$2}')
+export TAG = $(call parse_env_value,TAG)
 export GIT_HASH = $(shell git rev-parse --short HEAD)
+POSTGRES_DOCKER_RAW = $(shell echo $(call parse_env_value,POSTGRES_DOCKER) | tr '[:upper:]' '[:lower:]')
+export POSTGRES_DOCKER = $(call parse_env_bool,$(POSTGRES_DOCKER_RAW))
+
+# Default compose file args
 export COMPOSE_FILE_ARGS = -f docker-compose.yml -f docker-compose.dev.yml
+COMPOSE_FILE_ARGS += $(if $(POSTGRES_DOCKER),--profile postgres,)
+
+# Set up environment-specific values
+define setup_env
+	$(eval ENV_FILE = $(1))
+	$(eval TAG = $(call parse_env_value,TAG))
+	$(eval POSTGRES_DOCKER_RAW = $(shell echo $(call parse_env_value,POSTGRES_DOCKER) | tr '[:upper:]' '[:lower:]'))
+	$(eval POSTGRES_DOCKER = $(call parse_env_bool,$(POSTGRES_DOCKER_RAW)))
+	$(eval COMPOSE_FILE_ARGS = $(2))
+	$(eval COMPOSE_FILE_ARGS += $(if $(POSTGRES_DOCKER),--profile postgres,))
+endef
 
-PROD: ## Run in prod mode (e.g. `make PROD start`, etc.) using config in `prod.env`
-	$(eval ENV_FILE = prod.env)
-	$(eval TAG = $(shell grep -e ^TAG ${ENV_FILE} | awk -F'[=]' '{gsub(/ /,"");print $$2}'))
-	$(eval COMPOSE_FILE_ARGS = -f docker-compose.yml)
+PROD:
+	$(call setup_env,prod.env,-f docker-compose.yml)
 
-TEST: ## Run in test mode (e.g. `make TEST e2e-run`, etc.) using config in `test.env`
-	$(eval ENV_FILE = test.env)
-	$(eval TAG = $(shell grep -e ^TAG ${ENV_FILE} | awk -F'[=]' '{gsub(/ /,"");print $$2}'))
-	$(eval COMPOSE_FILE_ARGS = -f docker-compose.yml -f docker-compose.test.yml)
+TEST:
+	$(call setup_env,test.env,-f docker-compose.yml -f docker-compose.test.yml)
 
 echo_vars:
 	@echo ENV_FILE=${ENV_FILE}
+	@echo POSTGRES_DOCKER=${POSTGRES_DOCKER}
 	@echo TAG=${TAG}
 
 pull: echo_vars ## Pull most recent Docker container builds (nightlies)

README.md

@@ -77,7 +77,7 @@ cp example.env .env
 
 
 ```sh
-docker compose up --build
+docker compose --profile postgres up --build
 ```
 
 If you get a permission error, try running this command with `sudo`.
@@ -87,7 +87,7 @@ To avoid having to use `sudo` in the future (on a Linux or Windows machine with
 Once you've built the docker images, you can run without `--build`, which may be faster. Run
 
 ```sh
-docker compose up
+docker compose --profile postgres up
 ```
 
 or simply
@@ -103,16 +103,22 @@ If you have only changed configuration values in .env, you can recreate your con
 fully rebuilding them with `--force-recreate`. For example:
 
 ```sh
-docker compose down
-docker compose up --force-recreate
+docker compose --profile postgres down
+docker compose --profile postgres up --force-recreate
 ```
 
 To see what the environment of your containers is going to look like, run:
 
 ```sh
-docker compose convert
+docker compose --profile postgres convert
 ```
 
+#### Using a local or remote (non-docker) database
+
+Omit the `--profile postgres` flag to use a local or remote database. You will need to set the `DATABASE_URL` environment variable in your `.env` file to point to your database.
+
+When using `make` commands, setting POSTGRES_DOCKER to `true` or `false` will determine whether to automatically include `--profile postgres` when it calls out to `docker compose`.
+
 #### Production Mode Shortcuts
 
 The commands in the Makefile can be prefaced with PROD. If so, the "dev overlay" configuration in `docker-compose.dev.yml` will be ignored.
@@ -138,8 +144,8 @@ You can now test your setup by visiting `http://localhost:80/home`.
 Once the index page loads, you can create an account using the `/createuser` path.
 You'll be logged in right away; email validation is not required.
 
-When you're done working, you can end the process using `Ctrl+C`, or typing `docker compose down`
-if you are running in "detched mode".
+When you're done working, you can end the process using `Ctrl+C`, or typing `docker compose --profile postgres down`
+if you are running in "detached mode".
 
 ### Updating the system
 
@@ -174,7 +180,7 @@ However, if you are deploying in a high impact context and need help, please [re
 Once you've gotten [Polis running (as described above)](#-running-polis), you can enable developer conveniences by running
 
 ```bash
-docker compose -f docker-compose.yml -f docker-compose.dev.yml up
+docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up
 ```
 
 (run with `--build` if this is your first time running, or if you need to rebuild containers)
@@ -188,7 +194,7 @@ This enables:
 - etc.
 
 This command takes advantage of the `docker-compose.dev.yml` _overlay_ file, which layers the developer conveniences describe above into the base system, as described in the `docker-compose.yml` file.
-You can specify these `-f docker-compose.yml -f docker-compose.dev.yml` arguments for any `docker` command which you need to take advantage of these features (not just `docker compose up`).
+You can specify these `-f docker-compose.yml -f docker-compose.dev.yml` arguments for any `docker` command which you need to take advantage of these features (not just `docker compose --profile postgres up`).
 
 You can create your own `docker-compose.x.yml` file as an overlay and add or modify any values you need to differ
 from the defaults found in the `docker-compose.yml` file and pass it as the second argument to the `docker compose -f` command above.
@@ -219,7 +225,7 @@ git config --local include.path ../.gitconfig
 
 #### Running as a background process
 
-If you would like to run docker compose as a background process, run the `up` commands with the `--detach` flag, and use `docker compose down` to stop.
+If you would like to run docker compose as a background process, run the `up` commands with the `--detach` flag, and use `docker compose --profile postgres down` to stop.
 
 #### Using Docker Machine as your development environment
 

docker-compose.dev.yml

@@ -3,7 +3,7 @@
 # server, and exposing most ports. DO NOT USE THIS IN PRODUCTION.
 
 # This configuration does not work on its own and must be run together with docker-compose.yml,
-# e.g. `docker compose -f docker-compose.yml -f docker-compose.dev.yml up`.
+# e.g. `docker compose --profile postgres -f docker-compose.yml -f docker-compose.dev.yml up`.
 # (Note the usage in Makefile.)
 # For more information see https://docs.docker.com/compose/extends/
 
@@ -32,10 +32,10 @@ services:
     environment:
       CHOKIDAR_USEPOLLING: "true"
 
-  # postgres:
-  #   restart: no
-  #   ports:
-  #     - "${POSTGRES_PORT:-5432}:5432"
+  postgres:
+    restart: no
+    ports:
+      - "${POSTGRES_PORT:-5432}:5432"
 
   file-server:
     build:

docker-compose.test.yml

@@ -3,7 +3,7 @@
 # server. DO NOT USE THIS IN PRODUCTION.
 
 # This configuration does not work on its own and must be run together with docker-compose.yml,
-# e.g. `docker compose -f docker-compose.yml -f docker-compose.test.yml --env-file test.env up`.
+# e.g. `docker compose --profile postgres -f docker-compose.yml -f docker-compose.test.yml --env-file test.env up`.
 
 # For more information see https://docs.docker.com/compose/extends/
 

docker-compose.yml

@@ -1,24 +1,28 @@
-# Before running docker compose up for the first time,
+# Before running docker compose --profile postgres up for the first time,
 # first copy the env.example file to .env and fill in the values.
 # `cp env.example .env`
 # (edit values as needed)
 #
 # Then, build and run with:
-# `docker compose build`
-# `docker compose up`
+# `docker compose --profile postgres build`
+# `docker compose --profile postgres up`
 #
 # Subsequently you should only need to run:
-# `docker compose up`
+# `docker compose --profile postgres up`
 #
 # Force a full re-build with no cache from previous builds:
-# `docker compose build --no-cache`
-# `docker compose up`
+# `docker compose --profile postgres build --no-cache`
+# `docker compose --profile postgres up`
 #
 # If you only changed the .env file, you can do a quick rebuild:
-# `docker compose up --force-recreate`
+# `docker compose --profile postgres up --force-recreate`
 #
 # To stop, press CTRL+C (if not running in --detach mode), or run:
-# `docker compose down`
+# `docker compose --profile postgres down`
+#
+# omit the --profile postgres flag to run without the postgres service,
+# such as when using postgres on your host machine, or a remote server.
+# (by default, the Makefile will include --profile postgres, unless POSTGRES_DOCKER=false in your env)
 
 services:
   server:
@@ -34,7 +38,6 @@ services:
       labels:
         polis_tag: ${TAG:-dev}
     depends_on:
-      # - "postgres"
       - "file-server"
     networks:
       - "polis-net"
@@ -46,8 +49,6 @@ services:
 
   math:
     image: docker.io/compdem/polis-math:${TAG:-dev}
-    # depends_on:
-    #   - "postgres"
     build:
       context: ./math
       labels:
@@ -60,23 +61,25 @@ services:
     networks:
       - "polis-net"
 
-  # postgres:
-  #   image: docker.io/compdem/polis-postgres:${TAG:-dev}
-  #   restart: always
-  #   build:
-  #     context: ./server
-  #     dockerfile: Dockerfile-db
-  #     labels:
-  #       polis_tag: ${TAG:-dev}
-  #   environment:
-  #     - POSTGRES_DB=${POSTGRES_DB:?error}
-  #     - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?error}
-  #     - POSTGRES_USER=${POSTGRES_USER:?error}
-  #   networks:
-  #     - "polis-net"
-  #   volumes:
-  #     - "backups:/backups"
-  #     - "postgres:/var/lib/postgresql/data"
+  postgres:
+    image: docker.io/compdem/polis-postgres:${TAG:-dev}
+    restart: always
+    build:
+      context: ./server
+      dockerfile: Dockerfile-db
+      labels:
+        polis_tag: ${TAG:-dev}
+    environment:
+      - POSTGRES_DB=${POSTGRES_DB:?error}
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:?error}
+      - POSTGRES_USER=${POSTGRES_USER:?error}
+    networks:
+      - "polis-net"
+    volumes:
+      - "backups:/backups"
+      - "postgres:/var/lib/postgresql/data"
+    profiles:
+      - postgres
 
   nginx-proxy:
     image: docker.io/compdem/polis-nginx-proxy:${TAG:-dev}
@@ -120,9 +123,9 @@ volumes:
   backups:
     labels:
       polis_tag: ${TAG:-dev}
-  # postgres:
-  #   labels:
-  #     polis_tag: ${TAG:-dev}
+  postgres:
+    labels:
+      polis_tag: ${TAG:-dev}
   server-logs:
     labels:
       polis_tag: ${TAG:-dev}

docs/configuration.md

@@ -62,12 +62,19 @@ If you are deploying to a custom domain (not `pol.is`) then you need to update b
 ### Database
 
 - **`READ_ONLY_DATABASE_URL`** (optional) Database replica for reads.
+
+#### Required when using the docker postgres service (optional)
+
 - **`POSTGRES_DB`** database name (e.g. `polis-dev`)
 - **`POSTGRES_HOST`** database host (e.g. `postgres:5432` if using docker compose, `localhost:5432` if using local db)
 - **`POSTGRES_PASSWORD`** database password
 - **`POSTGRES_PORT`** typically 5432
 - **`POSTGRES_USER`** typically `postgres`. Any username will be used by the docker container to create a db user.
+
+#### Required in all cases
+
 - **`DATABASE_URL`** should be the combination of above values, `postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}`
+- **`POSTGRES_DOCKER`** Set to `false` if using a postgres database outside of docker. Defaults to `true`. Read by Makefile.
 
 ### Docker Concerns
 

docs/migrations.md

@@ -12,7 +12,7 @@ But if we update the database schema after your initial provisioning of your ser
   - Please submit an issue if you'd like to work on enabling backups through Docker Compose.
 - Your database data is stored on a docker volume, which means that it will
   persist even when you destroy all your docker containers. Be mindful of this.
-  - You can remove ALL volumes defined within a `docker-compose` file via: `docker compose down --volumes`
+  - You can remove ALL volumes defined within a `docker-compose` file via: `docker compose --profile postgres down --volumes`
   - You can remove ONE volume via `docker volume ls` and `docker volume rm <name>`
 - SQL migrations can be found in [`server/postgres/migrations/`][] of this
   repo.
@@ -24,7 +24,7 @@ For example, if we add the migration file
 host system:
 
 ```sh
-docker compose exec postgres psql --username postgres --dbname polis-dev --file=/docker-entrypoint-initdb.d/000001_update_pwreset_table.sql
+docker compose --profile postgres exec postgres psql --username postgres --dbname polis-dev --file=/docker-entrypoint-initdb.d/000001_update_pwreset_table.sql
 ```
 
 You can also run a local .sql file on a postgres container instance with this syntax:

example.env

@@ -20,12 +20,16 @@ SERVER_LOG_LEVEL=
 ###### DATABASE ######
 # Optional DB replica for reads:
 READ_ONLY_DATABASE_URL=
+# Required for using dockerized postgres (ie --profile postgres).
 POSTGRES_DB=polis-dev
 POSTGRES_PORT=5432
 POSTGRES_HOST=postgres:${POSTGRES_PORT}
 POSTGRES_USER=postgres
 POSTGRES_PASSWORD=oiPorg3Nrz0yqDLE
+# Always required. Replace with your own database URL if not using dockerized postgres.
 DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@${POSTGRES_HOST}/${POSTGRES_DB}
+# Makefile will read this to determine if the database is running in a docker container.
+POSTGRES_DOCKER=true
 
 
 ###### DOCKER CONCERNS ######

math/README.md

@@ -15,7 +15,7 @@ This allows you to connect to and evaluate code from within the running math wor
 To use this infrastructure, run the following from the root of this repository:
 
 ```sh
-docker compose -f docker-compose.yml -f docker-compose.dev.yml up
+docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up
 ```
 
 The first time you run this (or if you've edited any of the docker files, or certain other parts of the system), you may need to run this command with the `--build` flag.
@@ -59,7 +59,7 @@ There are also a number of commands which can be run, either locally or with `do
 * `clojure -M:run full` - run a full system (poller plus auxiliary task processing)
 * etc.
 
-If you are exporting data, you will need to run `docker compose` here with the same `-f docker-compose.yml -f docker-compose.dev.yml` options as described above, so that the directory (volume) mounting in the dev configuration file will mirror the generated files over onto your local filesystem.
+If you are exporting data, you will need to run `docker compose` here with the same `-f docker-compose.yml -f docker-compose.dev.yml --profile postgres` options as described above, so that the directory (volume) mounting in the dev configuration file will mirror the generated files over onto your local filesystem.
 
 ## Tests
 

server/README.md

@@ -49,9 +49,9 @@ flavors of node.
 
   Another popular option is to run the database in docker, and perhaps other services as well, while running this
   API server locally (not docker). Ensure that the postgres port is published from your docker container. This will be
-  the default behavior if you run `docker compose -f docker-compose.yml -f docker-compose.dev.yml up postgres` from the
+  the default behavior if you run `docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up postgres` from the
   root folder of the polis project. To run everything but the API server in this fashion you can use
-  `docker compose -f docker-compose.yml -f docker-compose.dev.yml up math postgres file-server maildev`. In this case
+  `docker compose -f docker-compose.yml -f docker-compose.dev.yml --profile postgres up math postgres file-server maildev`. In this case
   the polis-dev database should be accessible at the default DATABASE_URL seen in server/example.env.
 
 3\. Connect to the new database then run the migrations in its shell. You can skip this step if you built the