diff --git a/adminSiteServer/app.test.tsx b/adminSiteServer/app.test.tsx index 106d871ee3b..192442aeda8 100644 --- a/adminSiteServer/app.test.tsx +++ b/adminSiteServer/app.test.tsx @@ -1,9 +1,51 @@ -import { OwidAdminApp } from "./app.js" +import { OwidAdminApp } from "./appClass.js" import { jest } from "@jest/globals" +import { logInAsUser } from "./authentication.js" +import knex, { Knex } from "knex" +import { dbTestConfig } from "../db/tests/dbTestConfig.js" +import sqlFixtures from "sql-fixtures" +import { + TransactionCloseMode, + knexReadWriteTransaction, + setKnexInstance, +} from "../db/db.js" +import { cleanTestDb } from "../db/tests/testHelpers.js" jest.setTimeout(10000) // wait for up to 10s for the app server to start +let testKnexInstance: Knex | undefined = undefined +let serverKnexInstance: Knex | undefined = undefined -describe(OwidAdminApp, () => { +beforeAll(async () => { + const dataSpec = { + users: [ + { + email: "admin@example.com", + fullName: "Admin", + password: "admin", + createdAt: new Date(), + updatedAt: new Date(), + }, + ], + } + testKnexInstance = knex(dbTestConfig) + serverKnexInstance = knex(dbTestConfig) + await cleanTestDb(testKnexInstance) + + const fixturesCreator = new sqlFixtures(testKnexInstance) + await fixturesCreator.create(dataSpec) + setKnexInstance(serverKnexInstance!) +}) + +afterAll((done: any) => { + // We leave the user in the database for other tests to use + // For other cases it is good to drop any rows created in the test + void Promise.allSettled([ + testKnexInstance?.destroy(), + serverKnexInstance?.destroy(), + ]).then(() => done()) +}) + +describe("OwidAdminApp", () => { const app = new OwidAdminApp({ isDev: true, gitCmsDir: "", quiet: true }) it("should be able to create an app", () => { @@ -12,7 +54,33 @@ describe(OwidAdminApp, () => { it("should be able to start the app", async () => { await app.startListening(8765, "localhost") + console.error("Server started") expect(app.server).toBeTruthy() + const _ = await knexReadWriteTransaction( + async (trx) => { + console.error("Transaction started") + const cookieId = await logInAsUser({ + email: "admin@example.com", + id: 1, + }) + console.error("Logged in") + const bla = await fetch( + "http://localhost:8765/admin/nodeVersion", + { + headers: { cookie: `sessionid=${cookieId.id}` }, + } + ) + console.error("fetched") + expect(bla.status).toBe(200) + const text = await bla.text() + console.error("text", text) + expect(text).toBe("v18.16.1") + }, + TransactionCloseMode.Close, + testKnexInstance + ) + console.error("Transaction done") await app.stopListening() + console.error("Server stopped") }) }) diff --git a/adminSiteServer/app.tsx b/adminSiteServer/app.tsx index 3eee4db8ce5..b45978b2367 100644 --- a/adminSiteServer/app.tsx +++ b/adminSiteServer/app.tsx @@ -1,268 +1,10 @@ -import React from "react" -import { simpleGit } from "simple-git" -import express, { NextFunction } from "express" -require("express-async-errors") // todo: why the require? -import cookieParser from "cookie-parser" -import "reflect-metadata" -import http from "http" -import Bugsnag from "@bugsnag/js" -import BugsnagPluginExpress from "@bugsnag/plugin-express" +import { GIT_CMS_DIR } from "../gitCms/GitCmsConstants.js" import { ADMIN_SERVER_HOST, ADMIN_SERVER_PORT, - BAKED_BASE_URL, - BUGSNAG_NODE_API_KEY, ENV, } from "../settings/serverSettings.js" -import * as db from "../db/db.js" -import * as wpdb from "../db/wpdb.js" -import { IndexPage } from "./IndexPage.js" -import { - authCloudflareSSOMiddleware, - authMiddleware, -} from "./authentication.js" -import { apiRouter } from "./apiRouter.js" -import { testPageRouter } from "./testPageRouter.js" -import { adminRouter } from "./adminRouter.js" -import { renderToHtmlPage } from "../serverUtils/serverUtil.js" - -import { publicApiRouter } from "./publicApiRouter.js" -import { mockSiteRouter } from "./mockSiteRouter.js" -import { GIT_CMS_DIR } from "../gitCms/GitCmsConstants.js" -import { GdocsContentSource } from "@ourworldindata/utils" -import OwidGdocPage from "../site/gdocs/OwidGdocPage.js" -import { getAndLoadGdocById } from "../db/model/Gdoc/GdocFactory.js" - -interface OwidAdminAppOptions { - gitCmsDir: string - isDev: boolean - quiet?: boolean -} - -export class OwidAdminApp { - constructor(options: OwidAdminAppOptions) { - this.options = options - } - - app = express() - private options: OwidAdminAppOptions - - private async getGitCmsBranchName() { - const git = simpleGit({ - baseDir: this.options.gitCmsDir, - binary: "git", - maxConcurrentProcesses: 1, - }) - const branches = await git.branchLocal() - const gitCmsBranchName = await branches.current - return gitCmsBranchName - } - - private gitCmsBranchName = "" - - server?: http.Server - async stopListening() { - if (!this.server) return - - this.server.close() - } - - async startListening(adminServerPort: number, adminServerHost: string) { - this.gitCmsBranchName = await this.getGitCmsBranchName() - let bugsnagMiddleware - - const { app } = this - - if (BUGSNAG_NODE_API_KEY) { - Bugsnag.start({ - apiKey: BUGSNAG_NODE_API_KEY, - context: "admin-server", - plugins: [BugsnagPluginExpress], - autoTrackSessions: false, - }) - bugsnagMiddleware = Bugsnag.getPlugin("express") - // From the docs: "this must be the first piece of middleware in the - // stack. It can only capture errors in downstream middleware" - if (bugsnagMiddleware) app.use(bugsnagMiddleware.requestHandler) - } - - // since the server is running behind a reverse proxy (nginx), we need to "trust" - // the X-Forwarded-For header in order to get the real request IP - // https://expressjs.com/en/guide/behind-proxies.html - app.set("trust proxy", true) - - // Parse cookies https://github.com/expressjs/cookie-parser - app.use(cookieParser()) - - app.use(express.urlencoded({ extended: true, limit: "50mb" })) - - app.use("/admin/login", authCloudflareSSOMiddleware) - - // Require authentication (only for /admin requests) - app.use(authMiddleware) - - app.use("/assets", express.static("dist/assets")) - app.use("/fonts", express.static("public/fonts")) - - app.use("/api", publicApiRouter.router) - app.use("/assets-admin", express.static("dist/assets-admin")) - app.use("/admin/api", apiRouter.router) - app.use("/admin/test", testPageRouter) - app.use("/admin/storybook", express.static(".storybook/build")) - app.use("/admin", adminRouter) - - // Default route: single page admin app - app.get("/admin/*", async (req, res) => { - res.send( - renderToHtmlPage( - - ) - ) - }) - - // Public preview of a Gdoc document - app.get("/gdocs/:id/preview", async (req, res) => { - try { - // TODO: this transaction is only RW because somewhere inside it we fetch images - await db.knexReadWriteTransaction(async (knex) => { - const gdoc = await getAndLoadGdocById( - knex, - req.params.id, - GdocsContentSource.Gdocs - ) - - res.set("X-Robots-Tag", "noindex") - res.send( - renderToHtmlPage( - - ) - ) - }) - } catch (error) { - console.error("Error fetching gdoc preview", error) - res.status(500).json({ - error: { message: String(error), status: 500 }, - }) - } - }) - - // From the docs: "this handles any errors that Express catches. This - // needs to go before other error handlers. BugSnag will call the `next` - // error handler if it exists. - if (bugsnagMiddleware) app.use(bugsnagMiddleware.errorHandler) - - // todo: we probably always want to have this, and can remove the isDev - if (this.options.isDev) app.use("/", mockSiteRouter) - - // Give full error messages, including in production - app.use(this.errorHandler) - - await this.connectToDatabases() - - this.server = await this.listenPromise( - app, - adminServerPort, - adminServerHost - ) - this.server.timeout = 5 * 60 * 1000 // Increase server timeout for long-running uploads - - if (!this.options.quiet) - console.log( - `owid-admin server started on http://${adminServerHost}:${adminServerPort}` - ) - } - - // Server.listen does not seem to have an async/await form yet. - // https://github.com/expressjs/express/pull/3675 - // https://github.com/nodejs/node/issues/21482 - private listenPromise( - app: express.Express, - adminServerPort: number, - adminServerHost: string - ): Promise { - return new Promise((resolve) => { - const server = app.listen(adminServerPort, adminServerHost, () => { - resolve(server) - }) - }) - } - - errorHandler = async ( - err: any, - req: express.Request, - res: express.Response, - // keep `next` because Express only passes errors to handlers with 4 parameters. - // eslint-disable-next-line @typescript-eslint/no-unused-vars - next: NextFunction - ) => { - if (!res.headersSent) { - res.status(err.status || 500) - res.send({ - error: { - message: err.stack || err, - status: err.status || 500, - }, - }) - } else { - res.write( - JSON.stringify({ - error: { - message: err.stack || err, - status: err.status || 500, - }, - }) - ) - res.end() - } - } - - connectToDatabases = async () => { - try { - const _ = db.knexInstance() - } catch (error) { - // grapher database is in fact required, but we will not fail now in case it - // comes online later - if (!this.options.quiet) { - console.error(error) - console.warn( - "Could not connect to grapher database. Continuing without DB..." - ) - } - } - - if (wpdb.isWordpressDBEnabled) { - try { - await wpdb.singleton.connect() - } catch (error) { - if (!this.options.quiet) { - console.error(error) - console.warn( - "Could not connect to Wordpress database. Continuing without Wordpress..." - ) - } - } - } else if (!this.options.quiet) { - console.log( - "WORDPRESS_DB_NAME is not configured -- continuing without Wordpress DB" - ) - } - - if (!wpdb.isWordpressAPIEnabled && !this.options.quiet) { - console.log( - "WORDPRESS_API_URL is not configured -- continuing without Wordpress API" - ) - } - } -} +import { OwidAdminApp } from "./appClass.js" if (!module.parent) void new OwidAdminApp({ diff --git a/adminSiteServer/appClass.tsx b/adminSiteServer/appClass.tsx new file mode 100644 index 00000000000..92ce41aa936 --- /dev/null +++ b/adminSiteServer/appClass.tsx @@ -0,0 +1,267 @@ +import React from "react" +import { simpleGit } from "simple-git" +import express, { NextFunction } from "express" +require("express-async-errors") // todo: why the require +import cookieParser from "cookie-parser" +import "reflect-metadata" +import http from "http" +import Bugsnag from "@bugsnag/js" +import BugsnagPluginExpress from "@bugsnag/plugin-express" +import { + BAKED_BASE_URL, + BUGSNAG_NODE_API_KEY, +} from "../settings/serverSettings.js" +import * as db from "../db/db.js" +import * as wpdb from "../db/wpdb.js" +import { IndexPage } from "./IndexPage.js" +import { + authCloudflareSSOMiddleware, + authMiddleware, +} from "./authentication.js" +import { apiRouter } from "./apiRouter.js" +import { testPageRouter } from "./testPageRouter.js" +import { adminRouter } from "./adminRouter.js" +import { renderToHtmlPage } from "../serverUtils/serverUtil.js" + +import { publicApiRouter } from "./publicApiRouter.js" +import { mockSiteRouter } from "./mockSiteRouter.js" +import { GdocsContentSource } from "@ourworldindata/utils" +import OwidGdocPage from "../site/gdocs/OwidGdocPage.js" +import { getAndLoadGdocById } from "../db/model/Gdoc/GdocFactory.js" + +interface OwidAdminAppOptions { + gitCmsDir: string + isDev: boolean + quiet?: boolean +} + +export class OwidAdminApp { + constructor(options: OwidAdminAppOptions) { + this.options = options + } + + app = express() + private options: OwidAdminAppOptions + + private async getGitCmsBranchName() { + const git = simpleGit({ + baseDir: this.options.gitCmsDir, + binary: "git", + maxConcurrentProcesses: 1, + }) + const branches = await git.branchLocal() + const gitCmsBranchName = await branches.current + return gitCmsBranchName + } + + private gitCmsBranchName = "" + + server?: http.Server + async stopListening() { + if (!this.server) return + + this.server.close() + } + + async startListening(adminServerPort: number, adminServerHost: string) { + console.error("preparing to listen") + this.gitCmsBranchName = await this.getGitCmsBranchName() + let bugsnagMiddleware + + const { app } = this + + if (BUGSNAG_NODE_API_KEY) { + Bugsnag.start({ + apiKey: BUGSNAG_NODE_API_KEY, + context: "admin-server", + plugins: [BugsnagPluginExpress], + autoTrackSessions: false, + }) + bugsnagMiddleware = Bugsnag.getPlugin("express") + // From the docs: "this must be the first piece of middleware in the + // stack. It can only capture errors in downstream middleware" + if (bugsnagMiddleware) app.use(bugsnagMiddleware.requestHandler) + } + + // since the server is running behind a reverse proxy (nginx), we need to "trust" + // the X-Forwarded-For header in order to get the real request IP + // https://expressjs.com/en/guide/behind-proxies.html + app.set("trust proxy", true) + + // Parse cookies https://github.com/expressjs/cookie-parser + app.use(cookieParser()) + + app.use(express.urlencoded({ extended: true, limit: "50mb" })) + + app.use("/admin/login", authCloudflareSSOMiddleware) + + // Require authentication (only for /admin requests) + app.use(authMiddleware) + + app.use("/assets", express.static("dist/assets")) + app.use("/fonts", express.static("public/fonts")) + app.use("/assets-admin", express.static("dist/assets-admin")) + + app.use("/api", publicApiRouter.router) + app.use("/admin/api", apiRouter.router) + app.use("/admin/test", testPageRouter) + app.use("/admin/storybook", express.static(".storybook/build")) + app.use("/admin", adminRouter) + + // Default route: single page admin app + app.get("/admin/*", async (req, res) => { + res.send( + renderToHtmlPage( + + ) + ) + }) + + // Public preview of a Gdoc document + app.get("/gdocs/:id/preview", async (req, res) => { + try { + // TODO: this transaction is only RW because somewhere inside it we fetch images + await db.knexReadWriteTransaction(async (knex) => { + const gdoc = await getAndLoadGdocById( + knex, + req.params.id, + GdocsContentSource.Gdocs + ) + + res.set("X-Robots-Tag", "noindex") + res.send( + renderToHtmlPage( + + ) + ) + }) + } catch (error) { + console.error("Error fetching gdoc preview", error) + res.status(500).json({ + error: { message: String(error), status: 500 }, + }) + } + }) + + // From the docs: "this handles any errors that Express catches. This + // needs to go before other error handlers. BugSnag will call the `next` + // error handler if it exists. + if (bugsnagMiddleware) app.use(bugsnagMiddleware.errorHandler) + + // todo: we probably always want to have this, and can remove the isDev + if (this.options.isDev) app.use("/", mockSiteRouter) + + // Give full error messages, including in production + app.use(this.errorHandler) + + await this.connectToDatabases() + + console.error("setting up listening") + + this.server = await this.listenPromise( + app, + adminServerPort, + adminServerHost + ) + console.error("listening") + this.server.timeout = 5 * 60 * 1000 // Increase server timeout for long-running uploads + + if (!this.options.quiet) + console.log( + `owid-admin server started on http://${adminServerHost}:${adminServerPort}` + ) + } + + // Server.listen does not seem to have an async/await form yet. + // https://github.com/expressjs/express/pull/3675 + // https://github.com/nodejs/node/issues/21482 + private listenPromise( + app: express.Express, + adminServerPort: number, + adminServerHost: string + ): Promise { + return new Promise((resolve) => { + const server = app.listen(adminServerPort, adminServerHost, () => { + resolve(server) + }) + }) + } + + errorHandler = async ( + err: any, + req: express.Request, + res: express.Response, + // keep `next` because Express only passes errors to handlers with 4 parameters. + // eslint-disable-next-line @typescript-eslint/no-unused-vars + next: NextFunction + ) => { + if (!res.headersSent) { + res.status(err.status || 500) + res.send({ + error: { + message: err.stack || err, + status: err.status || 500, + }, + }) + } else { + res.write( + JSON.stringify({ + error: { + message: err.stack || err, + status: err.status || 500, + }, + }) + ) + res.end() + } + } + + connectToDatabases = async () => { + try { + console.error("connecting to db") + const _ = db.knexInstance() + console.error("connected") + } catch (error) { + // grapher database is in fact required, but we will not fail now in case it + // comes online later + if (!this.options.quiet) { + console.error(error) + console.warn( + "Could not connect to grapher database. Continuing without DB..." + ) + } + } + + if (wpdb.isWordpressDBEnabled) { + try { + await wpdb.singleton.connect() + } catch (error) { + if (!this.options.quiet) { + console.error(error) + console.warn( + "Could not connect to Wordpress database. Continuing without Wordpress..." + ) + } + } + } else if (!this.options.quiet) { + console.log( + "WORDPRESS_DB_NAME is not configured -- continuing without Wordpress DB" + ) + } + + if (!wpdb.isWordpressAPIEnabled && !this.options.quiet) { + console.log( + "WORDPRESS_API_URL is not configured -- continuing without Wordpress API" + ) + } + } +} diff --git a/adminSiteServer/authentication.tsx b/adminSiteServer/authentication.tsx index 31e6317bf41..2ecc2908cd4 100644 --- a/adminSiteServer/authentication.tsx +++ b/adminSiteServer/authentication.tsx @@ -201,7 +201,7 @@ function saltedHmac(salt: string, value: string): string { return hmac.digest("hex") } -async function logInAsUser(user: DbPlainUser) { +export async function logInAsUser(user: Pick) { const sessionId = randomstring.generate() const sessionJson = JSON.stringify({ diff --git a/adminSiteServer/sql-fixtures.d.ts b/adminSiteServer/sql-fixtures.d.ts new file mode 100644 index 00000000000..cc3717771a5 --- /dev/null +++ b/adminSiteServer/sql-fixtures.d.ts @@ -0,0 +1 @@ +declare module "sql-fixtures" diff --git a/db/db.ts b/db/db.ts index a89bddf2eed..695be96c6e8 100644 --- a/db/db.ts +++ b/db/db.ts @@ -24,10 +24,12 @@ export const closeTypeOrmAndKnexConnections = async (): Promise => { let _knexInstance: Knex | undefined = undefined -export const knexInstance = (): Knex => { - if (_knexInstance) return _knexInstance +export function setKnexInstance(knexInstance: Knex): void { + _knexInstance = knexInstance +} - _knexInstance = knex({ +const getNewKnexInstance = (): Knex => { + return knex({ client: "mysql", connection: { host: GRAPHER_DB_HOST, @@ -44,6 +46,12 @@ export const knexInstance = (): Knex => { }, }, }) +} + +export const knexInstance = (): Knex => { + if (_knexInstance) return _knexInstance + + _knexInstance = getNewKnexInstance() registerExitHandler(async () => { if (_knexInstance) await _knexInstance.destroy() diff --git a/db/tests/basic.test.ts b/db/tests/basic.test.ts index 41fb23ec1ec..ff0fee4c520 100644 --- a/db/tests/basic.test.ts +++ b/db/tests/basic.test.ts @@ -19,6 +19,7 @@ import { DbRawChart, UsersTableName, } from "@ourworldindata/types" +import { cleanTestDb, sleep } from "./testHelpers.js" let knexInstance: Knex | undefined = undefined @@ -35,6 +36,7 @@ beforeAll(async () => { ], } knexInstance = knex(dbTestConfig) + await cleanTestDb(knexInstance) const fixturesCreator = new sqlFixtures(knexInstance) await fixturesCreator.create(dataSpec) @@ -53,25 +55,17 @@ test("it can query a user created in fixture via TypeORM", async () => { .where({ email: "admin@example.com" }) .first() expect(user).toBeTruthy() - expect(user.id).toBe(1) + expect(user.id).toBe(2) expect(user.email).toBe("admin@example.com") }) -function sleep(time: number, value: any): Promise { - return new Promise((resolve) => { - setTimeout(() => { - return resolve(value) - }, time) - }) -} - test("timestamps are automatically created and updated", async () => { await knexReadWriteTransaction( async (trx) => { const chart: DbInsertChart = { config: "{}", lastEditedAt: new Date(), - lastEditedByUserId: 1, + lastEditedByUserId: 2, is_indexable: 0, } await trx.table(ChartsTableName).insert(chart) @@ -144,7 +138,7 @@ test("knex interface", async () => { }) // Use the update helper method - await updateUser(trx, 2, { isSuperuser: 1 }) + await updateUser(trx, 3, { isSuperuser: 1 }) // Check results after update and insert const afterUpdate = await trx @@ -183,7 +177,7 @@ test("knex interface", async () => { ]) expect(usersFromRawQueryWithInClauseAsArray.length).toBe(2) - await deleteUser(trx, 2) + await deleteUser(trx, 3) }, TransactionCloseMode.KeepOpen, knexInstance diff --git a/db/tests/testHelpers.ts b/db/tests/testHelpers.ts new file mode 100644 index 00000000000..e1627d6d290 --- /dev/null +++ b/db/tests/testHelpers.ts @@ -0,0 +1,16 @@ +import { Knex } from "knex" + +export async function cleanTestDb( + knexInstance: Knex +): Promise { + await knexInstance.raw("DELETE FROM users") + await knexInstance.raw("DELETE from charts") +} + +export function sleep(time: number, value: any): Promise { + return new Promise((resolve) => { + setTimeout(() => { + return resolve(value) + }, time) + }) +} diff --git a/jest.config.js b/jest.config.js index d393f03e580..b205ede88ea 100644 --- a/jest.config.js +++ b/jest.config.js @@ -7,6 +7,7 @@ module.exports = { testPathIgnorePatterns: [ ".jsdom.test.", "/itsJustJavascript/db/tests", + "/itsJustJavascript/adminSiteServer/app.test.(jsx|js)", ], testMatch: ["**/*.test.(jsx|js)"], modulePathIgnorePatterns: ["/wordpress/"], diff --git a/jest.db.config.js b/jest.db.config.js index 62058e09448..9e643ea1a89 100644 --- a/jest.db.config.js +++ b/jest.db.config.js @@ -4,7 +4,10 @@ module.exports = { { displayName: { name: "db", color: "blue" }, testEnvironment: "node", - testMatch: ["/itsJustJavascript/db/tests/**/*.test.js"], + testMatch: [ + "/itsJustJavascript/db/tests/**/*.test.js", + "/itsJustJavascript/adminSiteServer/*.test.(jsx|js)", + ], modulePathIgnorePatterns: ["/wordpress/"], }, ],