Merge branch 'edge' into make-psql-shell

.github/workflows/bundlewatch.yml

@@ -47,15 +47,13 @@ jobs:
       working-directory: client-admin
       run: |
         npm install
-        cp polis.config.template.js polis.config.js
         npm run build:prod
         mv dist/static/js/admin_bundle.*.js dist/static/js/admin_bundle.xxxxxxxx.js
 
     - name: "Install & Build: client-participation"
       working-directory: client-participation
       run: |
         npm install
-        cp polis.config.template.js polis.config.js
         npm run deploy:prod
         # So directory stays consistent between builds for comparison.
         mv dist/cached/* dist/cached/xxxxxxxxx

.github/workflows/cypress-tests.yml

@@ -60,7 +60,7 @@ jobs:
         run: cp e2e/cypress/fixtures/html/embed.html client-admin/public/embed.html
 
       - name: Serve app via docker-compose
-        run: docker-compose up --detach
+        run: docker compose up --detach
 
       - name: "Run Cypress tests: default"
         uses: cypress-io/github-action@v2
@@ -104,12 +104,12 @@ jobs:
 
       - name: Check status of containers
         if: failure()
-        run: docker-compose ps
+        run: docker compose ps
 
       - name: 'Check logs: server'
         if: failure()
-        run: docker-compose logs server
+        run: docker compose logs server
 
       - name: 'Check logs: nginx-proxy'
         if: failure()
-        run: docker-compose logs nginx-proxy
+        run: docker compose logs nginx-proxy

.github/workflows/release-docker-images.yml

@@ -36,7 +36,7 @@ jobs:
           password: ${{ secrets.DOCKER_TOKEN }}
 
       - name: Build full project via docker-compose
-        run: docker-compose build --parallel --build-arg GIT_HASH=${GITHUB_SHA:0:6}
+        run: docker compose build --parallel --build-arg GIT_HASH=${GITHUB_SHA:0:6}
 
       - name: Push images to Docker Hub
-        run: docker-compose push --ignore-push-failures
+        run: docker compose push --ignore-push-failures

.gitignore

@@ -1 +1,2 @@
+.env
 prod.env

.vscode/polis.code-workspace

@@ -0,0 +1,47 @@
+{
+	"folders": [
+		{
+      "name": "polis-root",
+			"path": ".."
+		},
+    {
+      "name": "client-admin",
+      "path": "../client-admin"
+    },
+    {
+      "name": "client-participation",
+      "path": "../client-participation"
+    },
+    {
+      "name": "client-report",
+      "path": "../client-report"
+    },
+    {
+      "name": "e2e",
+      "path": "../e2e"
+    },
+    {
+      "name": "file-server",
+      "path": "../file-server"
+    },
+    {
+      "name": "math",
+      "path": "../math"
+    },
+    {
+      "name": "server",
+      "path": "../server"
+    },
+  ],
+  "settings": {
+    "jest.disabledWorkspaceFolders": [
+      "polis-root",
+      "client-admin",
+      "client-participation",
+      "client-report",
+      "e2e",
+      "file-server",
+      "math"
+    ]
+  }
+}

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+  "jest.autoRun": "off",
+}

Makefile

@@ -17,18 +17,18 @@ PROD: ## Run in prod mode (e.g. `make PROD start`, etc.)
 	$(eval TAG = $(shell grep -e ^TAG ${ENV_FILE} | awk -F'[=]' '{gsub(/ /,"");print $$2}'))
 	$(eval COMPOSE_FILE_ARGS = -f docker-compose.yml)
 
-echo_vars: 
+echo_vars:
 	@echo ENV_FILE=${ENV_FILE}
 	@echo TAG=${TAG}
 
 pull: echo_vars ## Pull most recent Docker container builds (nightlies)
-	docker-compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} pull
+	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} pull
 
 start: echo_vars ## Start all Docker containers
-	docker-compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} up
+	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} up
 
 stop: echo_vars ## Stop all Docker containers
-	docker-compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} down
+	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} down
 
 rm-containers: echo_vars ## Remove Docker containers where (polis_tag="${TAG}")
 	@echo 'removing filtered containers (polis_tag="${TAG}")'
@@ -54,14 +54,14 @@ hash: ## Show current short hash
 	@echo Git hash: ${GIT_HASH}
 
 start-rebuild: echo_vars ## Start all Docker containers, [re]building as needed
-	docker-compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} up --build
+	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} up --build
 
 start-FULL-REBUILD: echo_vars stop rm-ALL ## Remove and restart all Docker containers, volumes, and images where (polis_tag="${TAG}")
-	docker-compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} build --no-cache
-	docker-compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} down
-	docker-compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} up --build
-	docker-compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} down
-	docker-compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} up --build
+	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} build --no-cache
+	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} down
+	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} up --build
+	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} down
+	docker compose ${COMPOSE_FILE_ARGS} --env-file ${ENV_FILE} up --build
 
 e2e-install: e2e/node_modules ## Install Cypress E2E testing tools
 	$(E2E_RUN) npm install

README.md

@@ -13,12 +13,12 @@ For a detailed methods paper, see [Polis: Scaling Deliberation by Mapping High D
    [docker-image-builds]: https://hub.docker.com/u/compdem
    [e2e-tests]: https://github.com/compdemocracy/polis/actions?query=workflow%3A%22E2E+Tests%22
 
-<br/>
+---
 
-
-### 🎈 🪁 Start here! 🪁 🎈
+## 🎈 🪁 Start here! 🪁 🎈
 
 If you're interested in using or contributing to Polis, please see the following:
+
 - [📚 **knowledge base**][knowledge-base]: for a comprehensive wiki to help you understand and use the system
 - [🌐 **main deployment**](https://pol.is): the main deployment of Polis is at <https://pol.is>, and is
   free to use for nonprofits and government
@@ -31,32 +31,40 @@ If you're interested in using or contributing to Polis, please see the following
    [issues]: https://github.com/compdemocracy/polis/issues
    [board]: https://github.com/compdemocracy/polis/projects/1
    [beta-board]: https://github.com/compdemocracy/polis/projects/1
-   [contributing]: /CONTRIBUTING.md#how-we-work
    [discussions]: https://github.com/compdemocracy/polis/discussions
    [hello]: mailto:[email protected]
 
 If you're trying to set up a Polis deployment or development environment, then please read the rest of this document 👇 ⬇️ 👇
 
+---
 
+## ⚡ Running Polis
 
-</br></br></br>
+Polis comes with Docker infrastructure for running a complete system, whether for a [production deployment](#-production-deployment) or a [development environment](#-development-tooling) (details for each can be found in later sections of this document).
+As a consequence, the only prerequisite to running Polis is that you install a recent `docker` (and Docker Desktop if you are on Mac or Windows).
 
+If you aren't able to use Docker for some reason, the various Dockerfiles found in subdirectories (`math`, `server`, `*-client`) of this repository _can_ be used as a reference for how you'd set up a system manually.
+If you're interested in doing the legwork to support alternative infrastructure, please [let us know in an issue](https://github.com/compdemocracy.org/issues).
 
+### Quick Start
 
-## ⚡ Running Polis
+```sh
+cp example.env .env
+make start
+```
 
-Polis comes with Docker infrastructure for running a complete system, whether for a [production deployment](#-production-deployment) or a [development environment](#-development-tooling) (details for each can be found in later sections of this document).
-As a consequence, the only prerequisite to running Polis is that you install a recent `docker` (and Docker Desktop if you are on Mac).
+That should run docker compose with the development overlay (see below) and default configuration values.
 
-If you aren't able to use Docker for some reason, the various `Dockerfile`s found in subdirectories (`math`, `server`, `*-client`) of this repository _can_ be used as a reference for how you'd set up a system manually.
-If you're interested in doing the legwork to support alternative infrastructure, please [let us know in an issue](https://github.com/compdemocracy.org/issues).
+Visit `localhost:80/createuser` and get started.
 
 ### Docker & Docker Compose
 
 Newer versions of `docker` have `docker compose` built in as a subcommand.
 If you are using an older version (and don't want to upgrade), you'll need to separately install `docker-compose`, and use that instead in the instructions that follow.
 Note however that the newer `docker compose` command is required to [take advantage of Docker Swarm](/docs/scaling#docker-compose-over-docker-swarm) as a scaling option.
 
+Many convenient commands are found in the Makefile. Run `make help` for a list of available commands.
+
 ### Building and running the containers
 
 First clone the repository, then navigate via command line to the root directory and run the following command to build and run the docker containers.
@@ -67,16 +75,36 @@ docker compose up --build
 
 If you get a permission error, try running this command with `sudo`.
 If this fixes the problem, sudo will be necessary for all other commands as well.
-To avoid having to use `sudo` in the future (on a Linux or Windows machine with WSL), you can follow setup instruction here: <https://docs.docker.com/engine/install/linux-postinstall/>.
+To avoid having to use `sudo` in the future (on a Linux or Windows machine with WSL), [you can follow setup instructions here.](https://docs.docker.com/engine/install/linux-postinstall/)
 
-Once you've built the docker images, you can run without `--build`, which may be faster.
-Simply:
+Once you've built the docker images, you can run without `--build`, which may be faster. Run
 
 ```sh
 docker compose up
 ```
 
-Any time you want to _rebuild_ the images, just reaffix `--build` when you run.
+or simply
+
+```sh
+make start
+```
+
+Any time you want to _rebuild_ the images, just reaffix `--build` when you run. Another way to
+easily rebuild and start your containers is with `make start-rebuild`.
+
+If you have only changed configuration values in .env, you can reacreate your containers without
+fully rebuilding them with `--force-recreate`. For example:
+
+```sh
+docker compose down
+docker compose up --force-recreate
+```
+
+To see what the environment of your containers is going to look like, run:
+
+```sh
+docker compose convert
+```
 
 ### Testing out your instance
 
@@ -85,38 +113,36 @@ You can now test your setup by visiting `http://localhost:80/home`.
 Once the index page loads, you can create an account using the `/createuser` path.
 You'll be logged in right away; email validation is not required.
 
-When you're done working, you can end the process using `Ctrl+C`.
+When you're done working, you can end the process using `Ctrl+C`, or typing `docker compose down`
+if you are running in "detched mode".
 
 ### Updating the system
 
 If you want to update the system, you may need to handle the following:
-* [⬆️ Run database migrations](docs/migrations.md), if there are new such
-* Update docker images by running with `--build` if there have been changes to the Dockerfiles
-  * consider using `--no-cache` if you'd like to rebuild from scratch, but note that this will take much longer
-
 
+- [⬆️ Run database migrations](docs/migrations.md), if there are new such
+- Update docker images by running with `--build` if there have been changes to the Dockerfiles
+  - consider using `--no-cache` if you'd like to rebuild from scratch, but note that this will take much longer
 
-</br>
+---
 
 ## 🚀 Production deployment
 
 While the commands above will get a functional Polis system up and running, additional steps must be taken to properly configure, secure and scale the system.
 In particular
 
-* [⚙️ Configure the system](docs/configuration.md), esp:
-  * the domain name you'll be serving from
-  * enable and add API keys for 3rd party services (e.g. automatic comment translation, spam filtering, etc)
-* [🔏 Set up SSL/HTTPS](docs/ssl.md), to keep the site secure
-* [📈 Scale](docs/scaling.md) for large or many concurrent conversations
+- [⚙️ Configure the system](docs/configuration.md), esp:
+  - the domain name you'll be serving from
+  - enable and add API keys for 3rd party services (e.g. automatic comment translation, spam filtering, etc)
+- [🔏 Set up SSL/HTTPS](docs/ssl.md), to keep the site secure
+- [📈 Scale](docs/scaling.md) for large or many concurrent conversations
 
-#### Support
+### Support
 
 We encourage you to take advantage of the public channels above for support setting up a deployment.
 However, if you are deploying in a high impact context and need help, please [reach out to us][hello]
 
-</br>
-
-
+---
 
 ## 💻 Development tooling
 
@@ -129,27 +155,40 @@ docker compose -f docker-compose.yml -f docker-compose.dev.yml up
 (run with `--build` if this is your first time running, or if you need to rebuild containers)
 
 This enables:
-* Live code reloading and static type checking of the server code
-* A nREPL connection port open for connecting to the running math process
-* Ports open for connecting directly to the database container
-* Live code reloading for the client repos (in process)
-* etc.
+
+- Live code reloading and static type checking of the server code
+- A nREPL connection port open for connecting to the running math process
+- Ports open for connecting directly to the database container
+- Live code reloading for the client repos (in process)
+- etc.
 
 This command takes advantage of the `docker-compose.dev.yml` _overlay_ file, which layers the developer conveniences describe above into the base system, as described in the `docker-compose.yml` file.
 You can specify these `-f docker-compose.yml -f docker-compose.dev.yml` arguments for any `docker` command which you need to take advantage of these features (not just `docker compose up`).
 
+You can create your own `docker-compose.x.yml` file as an overlay and add or modify any values you need to differ
+from the defaults found in the `docker-compose.yml` file and pass it as the second argument to the `docker compose -f` command above.
+
 ### Testing
 
 We use Cypress for automated, end-to-end browser testing for PRs on GitHub (see badge above).
 Please see [`e2e/README.md`](/e2e/README.md) for more information on running these tests locally.
 
 ### Miscellaneous & troubleshooting
 
+#### Docker Problems
+
+A lot of issues might be resolved by killing all docker containers and/or restarting docker entirely. If that doesn't
+work, this will wipe all of your polis containers and volumes and completely rebuild them:
+
+`make start-FULL-REBUILD`
+
+see also `make help` for additional commands that might be useful.
+
 #### Git Configuration
 
 Due to past file re-organizations, you may find the following git configuration helpful for looking at history:
 
-```
+```sh
 git config --local include.path ../.gitconfig
 ```
 
@@ -166,10 +205,12 @@ If your development machine is having trouble handling all of the docker contain
 Sometimes npm/docker get in a weird state, especially with native libs, and fail to recover gracefully.
 You may get a message like `Error: Cannot find module .... bcrypt`.
 
-If this happens to you, try following the instructions here: 
+If this happens to you, try
+[following the instructions here.](https://github.com/compdemocracy/polis/issues/1391)
 
-https://github.com/compdemocracy/polis/issues/1391
+#### Issues with Apple Silicon (M1 & M2) chips
 
+You may find it necessary to install some dependencies, namely nodejs and postgres stuff, in a [Rosetta terminal](https://support.apple.com/en-us/HT211861). Create an issue or reach out if you are having strange build issues on Apple computers.
 
 ## ©️  License