diff --git a/README.md b/README.md index d7772ead9..1f03fefa7 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,44 @@ - A node version manager, like [nvm](https://github.com/nvm-sh/nvm) or [n](https://github.com/tj/n) - [Keybase](https://keybase.io/) - [Lists keybase URL](keybase://team/cautionyourblast.fcdo/config/dev/lists) +### Quickstart guide (for lists development) + +**1. Make form json changes if necessary** + +You do not need to do this if you only need to run the find or list-management app. You also do not need to do this if you are running the entire application via docker-compose. + +If you are running the form runner via docker, but are running lists locally, then you will need to change the output within [./docker/apply/forms-jsons](./docker/apply/forms-json) + to target host.docker.internal:3000, instead of lists:3000. + +``` + "outputConfiguration": { + "url": "http://host.docker.internal:3000/ingest/funeralDirectors" + } +``` + +**2. Setup the test database file** + +For ARM (e.g. M1, M2) processors you will need to change the Dockerfile in `docker/db` to use the correct PostGIS image. + +To use test data, you must have the PGP keys to decrypt the data. +1. Download keybase://team/cautionyourblast.fcdo/config/dev/lists/pgp/.env +1. Load the env vars from .env into the shell `$ set -o allexport; source .env; set +o allexport;` +1. Build the container via docker compose `$ docker compose build postgres` + +**3. Start the databases and applications** + +```sh +docker compose up postgres redis apply +``` + +**4. Start the lists app** +1. Download the environment variables from [keybase://team/cautionyourblast.fcdo/config/dev/lists/.env](keybase://team/cautionyourblast.fcdo/config/dev/lists/pgp/.env) into the root of this project +1. If you need to test form submissions locally, you will need to authenticate your shell with AWS +1. Set your node version to 18 +1. Install dependencies `npm install` +1. Run the prisma migrations `npm run prisma:deploy` +1. Start the application `npm run dev` + ## Architecture @@ -40,13 +78,13 @@ to target host.docker.internal:3000, instead of lists:3000. ``` "outputConfiguration": { - "url": "http://host.docker.internal:3000/ingest/funeralDirectors" + "url": "http://lists:3000/ingest/funeralDirectors" } ``` To start the form runner ```sh -$ docker compose up apply +docker compose up apply ``` By default, it will start on port 3001. It will be accessible from your local machine at localhost:3001. @@ -86,7 +124,7 @@ Compose will start the following: - If you are using Apple Silicon (M1, M2), change `docker/db/Dockerfile` to use `FROM gangstead/postgis:13-3.1-arm` - If you have access to keybase and the B64 encrypted PGP keys, you may use the test data. See the Dockerfile for more details 1. `Redis`: The Redis database, accessible on [http://localhost:6379](http://localhost:6379) -1. `Apply`: The form runner, accessible on [http://localhost:3001](http://localhost:3001) +1. `Apply`: The form runner, accessible on [http://localhost:3001](http://localhost:3001), or [http://localhost:3000/application](http://localhost:3000/application) 1. `Lists`: The lists server, accessible on [http://localhost:3000](http://localhost:3000) Note: See `docker-compose.yml` file for respective usernames and passwords. @@ -163,7 +201,6 @@ Webpack will watch `/src` folder and will rebuild when changes occur, then Nodem For code styling and formatting we are using: -- [https://standardjs.com/](https://standardjs.com) - [eslint-config-standard-with-typescript](https://www.npmjs.com/package/eslint-config-standard-with-typescript) - [https://prettier.io/](https://prettier.io/) @@ -199,3 +236,6 @@ This project uses Conventional Commits to version the package correctly and gene You can generate valid commit messages by running `npm run commit` and following the instructions on your terminal window. Windows users should use the Bash terminal from the Windows Subsystem for Linux to run this. All commit messages are run through a validator and any invalid commit messages will be rejected. + +## Other guides +- diff --git a/docker-compose.yml b/docker-compose.yml index 23d5c9046..79b9ee7c2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -5,6 +5,11 @@ services: build: context: ./docker/db dockerfile: Dockerfile + args: + - PGP_SECRET=${PGP_SECRET} + - PGP_PUB=${PGP_PUB} + - PGP_OTRUST=${PGP_OTRUST} + # if you are on Mac M1 please use the image below # see https://github.com/docker/for-mac/issues/5122 # image: "gangstead/postgis:13-3.1-arm" @@ -130,6 +135,7 @@ services: build: context: "docker/apply" dockerfile: Dockerfile + environment: NODE_ENV: test ports: diff --git a/docker/db/Dockerfile b/docker/db/Dockerfile index 5de623370..2a5429da2 100644 --- a/docker/db/Dockerfile +++ b/docker/db/Dockerfile @@ -1,7 +1,12 @@ -FROM postgis/postgis -#FROM gangstead/postgis:13-3.1-arm +#FROM postgis/postgis +# If you are using an ARM device (e.g. Apple Silicon M1, M2), use the following line instead +FROM gangstead/postgis:13-3.1-arm + +# These secrets are stored in keybase://team/cautionyourblast.fcdo/config/dev/lists/pgp. +# You can replace the next 3 lines with the contents of keybase://team/cautionyourblast.fcdo/config/dev/lists/pgp/copy_into_docker.txt +# Or load the environment variables into your shell. + -# These secrets are stored in keybase://team/cautionyourblast.fcdo/config/dev/lists/pgp ARG PGP_SECRET="NO PGP SECRET KEY SET" ARG PGP_PUB="NO PGP PUBLIC KEY SET" ARG PGP_OTRUST="NO PGP OWNER TRUST DB SET" @@ -10,6 +15,9 @@ ENV POSTGRES_USER="master" ENV POSTGRES_DB=lists ENV POSTGRES_USER=master ENV POSTGRES_PASSWORD=postgrespass +ENV PGP_SECRET=$PGP_SECRET +ENV PGP_PUB=$PGP_PUB +ENV PGP_OTRUST=$PGP_OTRUST COPY test_data.sql.zip.gpg /docker-entrypoint-initdb.d/ COPY postgresql-local.conf postgresql-local.conf @@ -20,7 +28,7 @@ RUN echo $PGP_SECRET | base64 --decode | gpg --import && \ echo $PGP_OTRUST | gpg --import-ownertrust ### If you are unable to decrypt these files, you may have an invalid or outdated key and need to regenerate them -### You can also comment out the following line to skip loading test data +### You can also comment out the following line to skip test data RUN cd /docker-entrypoint-initdb.d && gpg -o 00-test_data.sql -d test_data.sql.zip.gpg CMD docker-entrypoint.sh -c 'shared_buffers=256MB' -c 'max_connections=200' -c config_file=postgresql-local.conf diff --git a/docs/README.md b/docs/README.md index f6aa29232..424be1b18 100644 --- a/docs/README.md +++ b/docs/README.md @@ -9,3 +9,4 @@ - [Fixing local data issues](fixing-local-data-issues.md) - [Scheduler process(es)](scheduler.md) - [Update dev with prod data](update-prod-data.md) +- [Adding a new list type](adding-new-list-type.md) diff --git a/docs/adding-new-list-type.md b/docs/adding-new-list-type.md new file mode 100644 index 000000000..3aaa2e067 --- /dev/null +++ b/docs/adding-new-list-type.md @@ -0,0 +1,95 @@ +# Adding a new list type + +## Creating and processing a new form (apply) + +### Creating a new form +**1. Create a new form using [XGovFormBuilder](https://github.com/XGovFormBuilder/digital-form-builder).** + +It is recommended that you start from an existing form first, like [lawyers.json](./docker/apply/forms-json/lawyers.json). +This is so common fields are handled the same across all forms, and easier to render on the find application later. + +Common fields used across lawyers, funeral directors and translators and interpreters are +- `contactName` +- `organisationName` +- `regions` +- `size` +- `address.firstLine` +- `address.secondLine` +- `city` +- `postCode` +- `addressCountry` +- `emailAddress` +- `publicEmailAddress` +- `publishEmail` +- `phoneNumber` +- `contactPhoneNumber` +- `declaration` +- `representedBritishNationals` +- `speakEnglish` + +**2. In the `metadata` property, add the camel cased type of form** + +```json5 +{ + //.. + metadata: { + type: "notaries" // or "funeralDirectors" + } +} + +``` + +**3. Change the outputConfiguration url to `/ingest/`** + +Add the new file to the `forms-json` directory** + +### Processing the new form (apply) +## Add the deserialisers +After the user submits the data, it will go to the newly configured lists endpoint `http://lists:3000/ingest/`. + +This will be handled by the [ingestPostController](./src/server/components/lists/controllers/ingest/ingestPostController.ts) + +IngestPostController will then attempt to deserialise the data into a ListItem. + +1. Add the new service type to [./lists/src/shared/types.ts](./lists/src/shared/types.ts). This will help identify where code may need to be changed to accommodate the new type +1. Add the service name to [serviceName](./src/server/utils/service-name.ts) +1. Add a custom deserialiser, which converts XGovFormBuilder data type into the Lists type. + 1. The [baseDeserialiser](src/server/models/listItem/providers/deserialisers/index.ts) will flatten the object + 1. Create a new file, `.deserialiser.ts`, with a function named `Deserialiser`. This function can make any additional conversions or override the base deserialiser's output + 1. Each deserialiser needs to add `country` from `addressCountry` +1. Add a new key to the [DESERIALISER](./src/server/models/listItem/providers/deserialisers/index.ts) Record + +## Find application + +The express router for the find/* endpoints can be found at [./src/server/components/lists/find/router.ts](./src/server/components/lists/find/router.ts) + +If the service requires additional filtering on a new field, you will need to add a new route, the matching views, input sanitisation, the actual query to find the list items, and the results page. + +The find router uses the Post, Redirect, Get pattern. Post the form; redirect to either the same page if there was an error or to the next page; Get the resource. The Post handler should not render anything. + +### Add the start page for the new service +When the user lands on `/find/`, we will show them the `notice.njk` view. This is rendered by the `/find/:serviceType` route. + +1. Create a new directory and notice.njk file `./src/server/views/lists/find//notice.njk` + +Copying from an existing is recommended. Don't forget to replace any page titles or service specific text. + +### Add the different filtering questions + +1. Create the pages for your new filtering questions in `./src/server/views/lists/find//.njk` +2. Create the new routes for them in [./src/server/components/lists/find/router.ts](./src/server/components/lists/find/router.ts) + 1. A get route handler will be needed to render the page + 2. A post route handler will be needed to handle the form submission + 1. The post handler needs to sanitise the user's input + 2. store it to session + 3. and redirect to the next page (or to the same page if there was an error) + 1. use `req.flash` to store the error message temporarily +3. Ensure the POST handler that should precede your new page, redirects to your new correct page +4. Render the answers box (grey box on righthand side) by adding `./lists/partials//answers-box.njk` + + +### Add the results page and query the database +1. Add a results page `./src/server/views/lists/find//results.njk` +2. Add a function `findPerCountry` in [src/server/models/listItem/providers/](./src/server/models/listItem/providers/) - This will query the database +3. Add a function `search` in [./src/server/components/lists/searches](./src/server/components/lists/searches) - This will parse the session data and pass it onto `findperCountry` +4. Determine which fields to render and create a partial for it. See [./src/server/views/lists/partials/funeral-directors/funeral-directors-results-list.njk](./src/server/views/lists/partials/funeral-directors/funeral-directors-results-list.njk) as an example diff --git a/src/server/components/lists/controllers/ingest/ingestPostController.ts b/src/server/components/lists/controllers/ingest/ingestPostController.ts index 521d1fcc2..4ad5674d2 100644 --- a/src/server/components/lists/controllers/ingest/ingestPostController.ts +++ b/src/server/components/lists/controllers/ingest/ingestPostController.ts @@ -7,6 +7,7 @@ import { createConfirmationLink, getServiceTypeName } from "server/components/li import { sendApplicationConfirmationEmail } from "server/services/govuk-notify"; import { logger } from "server/services/logger"; import type { ListItemJsonData } from "server/models/listItem/providers/deserialisers/types"; +import { isCybDev, isLocalHost, isSmokeTest } from "server/config"; export async function ingestPostController(req: Request, res: Response): Promise { const serviceType = getServiceTypeName(req.params.serviceType) as ServiceType; @@ -49,6 +50,9 @@ export async function ingestPostController(req: Request, res: Response): Promise } const confirmationLink = createConfirmationLink(req, reference); + if (isLocalHost || isCybDev || isSmokeTest) { + logger.info(`Generated confirmation link: ${confirmationLink}`); + } await sendApplicationConfirmationEmail(contactName, email, typeName, country.name, confirmationLink);