More documentation

README.md

@@ -8,6 +8,44 @@
 - A node version manager, like [nvm](https://github.com/nvm-sh/nvm) or [n](https://github.com/tj/n)
 - [Keybase](https://keybase.io/) - [Lists keybase URL](keybase://team/cautionyourblast.fcdo/config/dev/lists)
 
+### Quickstart guide (for lists development)
+
+**1. Make form json changes if necessary**
+
+You do not need to do this if you only need to run the find or list-management app. You also do not need to do this if you are running the entire application via docker-compose.
+
+If you are running the form runner via docker, but are running lists locally, then you will need to change the output within [./docker/apply/forms-jsons](./docker/apply/forms-json)
+   to target host.docker.internal:3000, instead of lists:3000.
+
+```
+      "outputConfiguration": {
+        "url": "http://host.docker.internal:3000/ingest/funeralDirectors"
+      }
+```
+
+**2. Setup the test database file**
+
+For ARM (e.g. M1, M2) processors you will need to change the Dockerfile in `docker/db` to use the correct PostGIS image.
+
+To use test data, you must have the PGP keys to decrypt the data. 
+1. Download keybase://team/cautionyourblast.fcdo/config/dev/lists/pgp/.env
+1. Load the env vars from .env into the shell `$ set -o allexport; source .env; set +o allexport;`
+1. Build the container via docker compose `$ docker compose build postgres`
+
+**3. Start the databases and applications**
+
+```sh
+docker compose up postgres redis apply
+```
+
+**4. Start the lists app**
+1. Download the environment variables from [keybase://team/cautionyourblast.fcdo/config/dev/lists/.env](keybase://team/cautionyourblast.fcdo/config/dev/lists/pgp/.env) into the root of this project
+1. If you need to test form submissions locally, you will need to authenticate your shell with AWS
+1. Set your node version to 18 
+1. Install dependencies `npm install`
+1. Run the prisma migrations `npm run prisma:deploy`
+1. Start the application `npm run dev`
+
 
 ## Architecture
 
@@ -40,13 +78,13 @@ to target host.docker.internal:3000, instead of lists:3000.
 
 ```
       "outputConfiguration": {
-        "url": "http://host.docker.internal:3000/ingest/funeralDirectors"
+        "url": "http://lists:3000/ingest/funeralDirectors"
       }
 ```
 
 To start the form runner
 ```sh
-$ docker compose up apply
+docker compose up apply
 ```
 
 By default, it will start on port 3001. It will be accessible from your local machine at localhost:3001.
@@ -86,7 +124,7 @@ Compose will start the following:
   - If you are using Apple Silicon (M1, M2), change `docker/db/Dockerfile` to use `FROM gangstead/postgis:13-3.1-arm`
   - If you have access to keybase and the B64 encrypted PGP keys, you may use the test data. See the Dockerfile for more details
 1. `Redis`: The Redis database, accessible on [http://localhost:6379](http://localhost:6379)
-1. `Apply`: The form runner, accessible on [http://localhost:3001](http://localhost:3001)
+1. `Apply`: The form runner, accessible on [http://localhost:3001](http://localhost:3001), or [http://localhost:3000/application](http://localhost:3000/application)
 1. `Lists`: The lists server, accessible on [http://localhost:3000](http://localhost:3000)
 
 Note: See `docker-compose.yml` file for respective usernames and passwords.
@@ -163,7 +201,6 @@ Webpack will watch `/src` folder and will rebuild when changes occur, then Nodem
 
 For code styling and formatting we are using:
 
-- [https://standardjs.com/](https://standardjs.com)
 - [eslint-config-standard-with-typescript](https://www.npmjs.com/package/eslint-config-standard-with-typescript)
 - [https://prettier.io/](https://prettier.io/)
 
@@ -199,3 +236,6 @@ This project uses Conventional Commits to version the package correctly and gene
 You can generate valid commit messages by running `npm run commit` and following the instructions on your terminal window. Windows users should use the Bash terminal from the Windows Subsystem for Linux to run this.
 
 All commit messages are run through a validator and any invalid commit messages will be rejected.
+
+## Other guides
+- 

docker-compose.yml

@@ -5,6 +5,11 @@ services:
     build:
       context: ./docker/db
       dockerfile: Dockerfile
+      args:
+        - PGP_SECRET=${PGP_SECRET}
+        - PGP_PUB=${PGP_PUB}
+        - PGP_OTRUST=${PGP_OTRUST}
+
     # if you are on Mac M1 please use the image below
     # see https://github.com/docker/for-mac/issues/5122
     # image: "gangstead/postgis:13-3.1-arm"
@@ -130,6 +135,7 @@ services:
     build:
       context: "docker/apply"
       dockerfile: Dockerfile
+
     environment:
       NODE_ENV: test
     ports:

docker/db/Dockerfile

@@ -1,7 +1,12 @@
-FROM postgis/postgis
-#FROM gangstead/postgis:13-3.1-arm
+#FROM postgis/postgis
+# If you are using an ARM device (e.g. Apple Silicon M1, M2), use the following line instead
+FROM gangstead/postgis:13-3.1-arm
+
+# These secrets are stored in keybase://team/cautionyourblast.fcdo/config/dev/lists/pgp.
+# You can replace the next 3 lines with the contents of keybase://team/cautionyourblast.fcdo/config/dev/lists/pgp/copy_into_docker.txt
+# Or load the environment variables into your shell.
+
 
-# These secrets are stored in keybase://team/cautionyourblast.fcdo/config/dev/lists/pgp
 ARG PGP_SECRET="NO PGP SECRET KEY SET"
 ARG PGP_PUB="NO PGP PUBLIC KEY SET"
 ARG PGP_OTRUST="NO PGP OWNER TRUST DB SET"
@@ -10,6 +15,9 @@ ENV POSTGRES_USER="master"
 ENV POSTGRES_DB=lists
 ENV POSTGRES_USER=master
 ENV POSTGRES_PASSWORD=postgrespass
+ENV PGP_SECRET=$PGP_SECRET
+ENV PGP_PUB=$PGP_PUB
+ENV PGP_OTRUST=$PGP_OTRUST
 
 COPY test_data.sql.zip.gpg /docker-entrypoint-initdb.d/
 COPY postgresql-local.conf postgresql-local.conf
@@ -20,7 +28,7 @@ RUN echo $PGP_SECRET | base64 --decode | gpg --import && \
     echo $PGP_OTRUST | gpg --import-ownertrust
 
 ### If you are unable to decrypt these files, you may have an invalid or outdated key and need to regenerate them
-### You can also comment out the following line to skip loading test data
+### You can also comment out the following line to skip  test data
 RUN cd /docker-entrypoint-initdb.d && gpg -o 00-test_data.sql -d test_data.sql.zip.gpg
 
 CMD docker-entrypoint.sh -c 'shared_buffers=256MB' -c 'max_connections=200' -c config_file=postgresql-local.conf

docs/README.md

@@ -9,3 +9,4 @@
 - [Fixing local data issues](fixing-local-data-issues.md)
 - [Scheduler process(es)](scheduler.md)
 - [Update dev with prod data](update-prod-data.md)
+- [Adding a new list type](adding-new-list-type.md)

docs/adding-new-list-type.md

@@ -0,0 +1,95 @@
+# Adding a new list type
+
+## Creating and processing a new form (apply)
+
+### Creating a new form 
+**1. Create a new form using [XGovFormBuilder](https://github.com/XGovFormBuilder/digital-form-builder).**
+
+It is recommended that you start from an existing form first, like [lawyers.json](./docker/apply/forms-json/lawyers.json).
+This is so common fields are handled the same across all forms, and easier to render on the find application later.
+
+Common fields used across lawyers, funeral directors and translators and interpreters are 
+- `contactName`
+- `organisationName`
+- `regions`
+- `size`
+- `address.firstLine`
+- `address.secondLine`
+- `city`
+- `postCode`
+- `addressCountry`
+- `emailAddress`
+- `publicEmailAddress`
+- `publishEmail`
+- `phoneNumber`
+- `contactPhoneNumber`
+- `declaration`
+- `representedBritishNationals`
+- `speakEnglish`
+
+**2. In the `metadata` property, add the camel cased type of form**  
+
+```json5
+{ 
+  //..
+  metadata: {
+    type: "notaries" // or "funeralDirectors"
+  }
+}
+
+```
+
+**3. Change the outputConfiguration url to `/ingest/<type>`**
+
+Add the new file to the `forms-json` directory**
+
+### Processing the new form (apply)
+## Add the deserialisers 
+After the user submits the data, it will go to the newly configured lists endpoint `http://lists:3000/ingest/<type>`.
+
+This will be handled by the [ingestPostController](./src/server/components/lists/controllers/ingest/ingestPostController.ts)
+
+IngestPostController will then attempt to deserialise the data into a ListItem.
+
+1. Add the new service type to [./lists/src/shared/types.ts](./lists/src/shared/types.ts). This will help identify where code may need to be changed to accommodate the new type
+1. Add the service name to [serviceName](./src/server/utils/service-name.ts)
+1. Add a custom deserialiser, which converts XGovFormBuilder data type into the Lists type.
+   1. The [baseDeserialiser](src/server/models/listItem/providers/deserialisers/index.ts) will flatten the object
+   1. Create a new file, `<ServiceType>.deserialiser.ts`, with a function named `<serviceType>Deserialiser`. This function can make any additional conversions or override the base deserialiser's output
+   1. Each deserialiser needs to add `country` from `addressCountry`
+1. Add a new key to the [DESERIALISER](./src/server/models/listItem/providers/deserialisers/index.ts) Record
+
+## Find application
+
+The express router for the find/* endpoints can be found at [./src/server/components/lists/find/router.ts](./src/server/components/lists/find/router.ts)
+
+If the service requires additional filtering on a new field, you will need to add a new route, the matching views, input sanitisation, the actual query to find the list items, and the results page. 
+
+The find router uses the Post, Redirect, Get pattern. Post the form; redirect to either the same page if there was an error or to the next page; Get the resource. The Post handler should not render anything. 
+
+### Add the start page for the new service 
+When the user lands on `/find/<type>`, we will show them the `notice.njk` view. This is rendered by the `/find/:serviceType` route.
+
+1. Create a new directory and notice.njk file `./src/server/views/lists/find/<service-type>/notice.njk`
+
+Copying from an existing is recommended. Don't forget to replace any page titles or service specific text.
+
+### Add the different filtering questions
+
+1. Create the pages for your new filtering questions in `./src/server/views/lists/find/<service-type>/<question>.njk`
+2. Create the new routes for them in [./src/server/components/lists/find/router.ts](./src/server/components/lists/find/router.ts)
+   1. A get route handler will be needed to render the page
+   2. A post route handler will be needed to handle the form submission
+      1. The post handler needs to sanitise the user's input
+      2. store it to session
+      3. and redirect to the next page (or to the same page if there was an error)
+         1. use `req.flash` to store the error message temporarily
+3. Ensure the POST handler that should precede your new page, redirects to your new correct page
+4. Render the answers box (grey box on righthand side) by adding `./lists/partials/<service-type>/answers-box.njk`
+
+
+### Add the results page and query the database
+1. Add a results page `./src/server/views/lists/find/<service-type>/results.njk` 
+2. Add a function `find<serviceType>PerCountry` in [src/server/models/listItem/providers/<serviceType>](./src/server/models/listItem/providers/) - This will query the database
+3. Add a function `search<serviceType>` in [./src/server/components/lists/searches](./src/server/components/lists/searches) - This will parse the session data and pass it onto `find<serviceType>perCountry`
+4. Determine which fields to render and create a partial for it. See [./src/server/views/lists/partials/funeral-directors/funeral-directors-results-list.njk](./src/server/views/lists/partials/funeral-directors/funeral-directors-results-list.njk) as an example

src/server/components/lists/controllers/ingest/ingestPostController.ts

@@ -7,6 +7,7 @@ import { createConfirmationLink, getServiceTypeName } from "server/components/li
 import { sendApplicationConfirmationEmail } from "server/services/govuk-notify";
 import { logger } from "server/services/logger";
 import type { ListItemJsonData } from "server/models/listItem/providers/deserialisers/types";
+import { isCybDev, isLocalHost, isSmokeTest } from "server/config";
 
 export async function ingestPostController(req: Request, res: Response): Promise<void> {
   const serviceType = getServiceTypeName(req.params.serviceType) as ServiceType;
@@ -49,6 +50,9 @@ export async function ingestPostController(req: Request, res: Response): Promise
     }
 
     const confirmationLink = createConfirmationLink(req, reference);
+    if (isLocalHost || isCybDev || isSmokeTest) {
+      logger.info(`Generated confirmation link: ${confirmationLink}`);
+    }
 
     await sendApplicationConfirmationEmail(contactName, email, typeName, country.name, confirmationLink);