Merge branch 'master' into search-evaluation-algolia

owid · May 1, 2024 · 44b44d7 · 44b44d7
2 parents c637c92 + dff12bf
commit 44b44d7
Show file tree

Hide file tree

Showing 333 changed files with 13,880 additions and 10,251 deletions.
diff --git a/.bundlewatch.config.json b/.bundlewatch.config.json
@@ -1,12 +1,12 @@
 {
     "files": [
         {
-            "path": "./dist/assets/common.mjs",
-            "maxSize": "2.2MB"
+            "path": "./dist/assets/owid.mjs",
+            "maxSize": "2.4MB"
         },
         {
-            "path": "./dist/assets/owid.mjs",
-            "maxSize": "450KB"
+            "path": "./dist/assets/owid.css",
+            "maxSize": "430KB"
         }
     ],
     "defaultCompression": "none"

diff --git a/.env.example-full b/.env.example-full
@@ -42,6 +42,8 @@ IMAGE_HOSTING_R2_SECRET_ACCESS_KEY=''  # optional
 
 OPENAI_API_KEY=''
 
+GRAPHER_DYNAMIC_THUMBNAIL_URL=''  # optional; can set this to https://ourworldindata.org/grapher/thumbnail to use the live thumbnail worker
+
 # enable search (readonly)
 ALGOLIA_ID=''   # optional
 ALGOLIA_SEARCH_KEY=''  # optional

diff --git a/.eslintrc.cjs b/.eslintrc.cjs
@@ -1,3 +1,5 @@
+const path = require('node:path');
+
 module.exports = {
     extends: [
         "plugin:react/recommended",
@@ -12,7 +14,7 @@ module.exports = {
         ecmaFeatures: {
             jsx: true,
         },
-        project: "./tsconfig.eslint.json",
+        project: path.join(__dirname, "tsconfig.eslint.json"),
         ecmaVersion: "latest",
     },
     overrides: [
@@ -41,8 +43,13 @@ module.exports = {
         "@typescript-eslint/prefer-for-of": "warn",
         "@typescript-eslint/restrict-plus-operands": "warn",
         eqeqeq: "warn",
+        // Turn off import rules covered by typescript-eslint.
+        // https://typescript-eslint.io/troubleshooting/performance-troubleshooting/#eslint-plugin-import
+        "import/named": "off",
         "import/namespace": "off",
-        "import/no-named-as-default-member": "off", // probably makes sense to enable this at some point
+        "import/default": "off",
+        "import/no-named-as-default-member": "off",
+        "import/no-unresolved": "off",
         "no-console": ["warn", { allow: ["warn", "error"] }],
         "prefer-const": ["warn", { destructuring: "all" }],
         "react/display-name": "warn",
@@ -51,6 +58,7 @@ module.exports = {
         "react/no-render-return-value": "warn",
         "react/no-unescaped-entities": ["warn", { forbid: [">", "}"] }],
         "react/prop-types": "warn",
+        "@typescript-eslint/no-floating-promises": "error",
     },
     settings: {
         "import/resolver": {

diff --git a/.github/workflows/svg-compare-all-views.yml b/.github/workflows/svg-compare-all-views.yml
@@ -61,27 +61,6 @@ jobs:
                   RM_ON_ERROR: ${{ env.ORIGINAL_BRANCH_NAME == 'master' && '' || '--rm-on-error' }}
               run: node --enable-source-maps itsJustJavascript/devTools/svgTester/verify-graphs.js -i owid-grapher-svgs/configs -o owid-grapher-svgs/all-views/svg -r owid-grapher-svgs/all-views/svg --ids-from-file owid-grapher-svgs/most-viewed-charts.txt --all-views $RM_ON_ERROR > compare-result
 
-            # If the last step failed we want to commit all changed svgs and push them to the new branch on the owid-grapher-svgs repo
-            - uses: stefanzweifel/git-auto-commit-action@v5
-              if: ${{ steps.run-verify-graphs.outcome == 'failure' }}
-              with:
-                  repository: ./owid-grapher-svgs/
-                  branch: ${{ env.PUSH_BRANCH_NAME }}
-                  push_options: "--force"
-                  commit_message: Automated commit with svg differences (all views) triggered by commit https://github.com/owid/owid-grapher/commit/${{github.sha}}
-
-            # Merge differences into owid-grapher-svgs master if we are on master in owid-grapher
-            - name: Merge into owid-grapher-svgs master
-              if: ${{ steps.run-verify-graphs.outcome == 'failure' && env.ORIGINAL_BRANCH_NAME == 'master' }}
-              working-directory: owid-grapher-svgs
-              run: |
-                  git config user.name "${GITHUB_ACTOR}"
-                  git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
-                  git checkout master
-                  git pull origin master
-                  git merge -s ort -X theirs -m 'Automated merge commit with svg differences (all views) triggered by commit https://github.com/owid/owid-grapher/commit/${{github.sha}}' ${{ env.PUSH_BRANCH_NAME }}
-                  git push --force origin master
-
             # The action fails if there were any errors.
             - name: Fail with error message if we had errors
               if: ${{ steps.run-verify-graphs.outputs.num_errors > 0 }}

diff --git a/.github/workflows/svg-compare.yml b/.github/workflows/svg-compare.yml
@@ -60,27 +60,6 @@ jobs:
                   RM_ON_ERROR: ${{ env.ORIGINAL_BRANCH_NAME == 'master' && '' || '--rm-on-error' }}
               run: node --enable-source-maps itsJustJavascript/devTools/svgTester/verify-graphs.js -i owid-grapher-svgs/configs -o owid-grapher-svgs/svg -r owid-grapher-svgs/svg $RM_ON_ERROR > compare-result
 
-            # If the last step failed we want to commit all changed svgs and push them to the new branch on the owid-grapher-svgs repo
-            - uses: stefanzweifel/git-auto-commit-action@v4
-              if: ${{ steps.run-verify-graphs.outcome == 'failure' }}
-              with:
-                  repository: ./owid-grapher-svgs/
-                  branch: ${{ env.PUSH_BRANCH_NAME }}
-                  push_options: "--force"
-                  commit_message: Automated commit with svg differences triggered by commit https://github.com/owid/owid-grapher/commit/${{github.sha}}
-
-            # Merge differences into owid-grapher-svgs master if we are on master in owid-grapher
-            - name: Merge into owid-grapher-svgs master
-              if: ${{ steps.run-verify-graphs.outcome == 'failure' && env.ORIGINAL_BRANCH_NAME == 'master' }}
-              working-directory: owid-grapher-svgs
-              run: |
-                  git config user.name "${GITHUB_ACTOR}"
-                  git config user.email "${GITHUB_ACTOR}@users.noreply.github.com"
-                  git checkout master
-                  git pull origin master
-                  git merge -s ort -X theirs -m 'Automated merge commit with svg differences triggered by commit https://github.com/owid/owid-grapher/commit/${{github.sha}}' ${{ env.PUSH_BRANCH_NAME }}
-                  git push --force origin master
-
             # The action fails if there were any errors.
             - name: Fail with error message if we had errors
               if: ${{ steps.run-verify-graphs.outputs.num_errors > 0 }}

diff --git a/.husky/pre-commit b/.husky/pre-commit
@@ -0,0 +1 @@
+npx lint-staged
diff --git a/.vscode/extensions.json b/.vscode/extensions.json
@@ -1,7 +1,6 @@
 {
     "recommendations": [
         "esbenp.prettier-vscode",
-        "dbaeumer.vscode-eslint",
-        "xdebug.php-debug"
+        "dbaeumer.vscode-eslint"
     ]
 }
diff --git a/.vscode/launch.json b/.vscode/launch.json
@@ -129,4 +129,4 @@
             "port": 9000
         }
     ]
-}
+}
diff --git a/.vscode/settings.json b/.vscode/settings.json
@@ -20,5 +20,8 @@
     "javascript.preferences.importModuleSpecifierEnding": "js",
     "[sql]": {
         "editor.defaultFormatter": "inferrinizzard.prettier-sql-vscode"
-    }
-}
+    },
+    "Prettier-SQL.keywordCase": "upper",
+    "Prettier-SQL.SQLFlavourOverride": "mysql",
+    "Prettier-SQL.expressionWidth": 80
+}
diff --git a/.yarn/releases/yarn-4.0.0.cjs b/.yarn/releases/yarn-4.0.0.cjs
diff --git a/.yarn/releases/yarn-4.1.1.cjs b/.yarn/releases/yarn-4.1.1.cjs
diff --git a/.yarnrc.yml b/.yarnrc.yml
@@ -3,9 +3,9 @@ compressionLevel: mixed
 enableGlobalCache: false
 
 logFilters:
-    - code: YN0013 # A package cannot be found in the cache for the given package and will be fetched from its remote location.
+    - code: YN0013
       level: discard
 
 nodeLinker: node-modules
 
-yarnPath: .yarn/releases/yarn-4.0.0.cjs
+yarnPath: .yarn/releases/yarn-4.1.1.cjs
diff --git a/Makefile b/Makefile
@@ -24,27 +24,28 @@ help:
 	@echo 'Available commands:'
 	@echo
 	@echo '  GRAPHER ONLY'
-	@echo '  make up                start dev environment via docker-compose and tmux'
-	@echo '  make down              stop any services still running'
-	@echo '  make refresh           (while up) download a new grapher snapshot and update MySQL'
-	@echo '  make refresh.pageviews (while up) download and load pageviews from the private datasette instance'
-	@echo '  make migrate           (while up) run any outstanding db migrations'
-	@echo '  make test              run full suite (except db tests) of CI checks including unit tests'
-	@echo '  make dbtest            run db test suite that needs a running mysql db'
-	@echo '  make svgtest           compare current rendering against reference SVGs'
+	@echo '  make up                     start dev environment via docker-compose and tmux'
+	@echo '  make down                   stop any services still running'
+	@echo '  make refresh                (while up) download a new grapher snapshot and update MySQL'
+	@echo '  make refresh.pageviews      (while up) download and load pageviews from the private datasette instance'
+	@echo '  make migrate                (while up) run any outstanding db migrations'
+	@echo '  make test                   run full suite (except db tests) of CI checks including unit tests'
+	@echo '  make dbtest                 run db test suite that needs a running mysql db'
+	@echo '  make svgtest                compare current rendering against reference SVGs'
 	@echo
 	@echo '  GRAPHER + WORDPRESS (staff-only)'
-	@echo '  make up.full           start dev environment via docker-compose and tmux'
-	@echo '  make down.full         stop any services still running'
-	@echo '  make refresh.wp        download a new wordpress snapshot and update MySQL'
-	@echo '  make refresh.full      do a full MySQL update of both wordpress and grapher'
-	@echo '  make sync-images       sync all images from the remote master'
-	@echo '  make reindex			reindex (or initialise) search in Algolia'
-	@echo '  make bench.search      run search benchmarks'
+	@echo '  make up.full                start dev environment via docker-compose and tmux'
+	@echo '  make down.full              stop any services still running'
+	@echo '  make refresh.wp             download a new wordpress snapshot and update MySQL'
+	@echo '  make refresh.full           do a full MySQL update of both wordpress and grapher'
+	@echo '  make sync-images            sync all images from the remote master'
+	@echo '  make update.chart-entities  update the charts_x_entities join table'
+	@echo '  make reindex                reindex (or initialise) search in Algolia'
+  @echo '  make bench.search           run search benchmarks'
 	@echo
 	@echo '  OPS (staff-only)'
-	@echo '  make deploy            Deploy your local site to production'
-	@echo '  make stage             Deploy your local site to staging'
+	@echo '  make deploy                 Deploy your local site to production'
+	@echo '  make stage                  Deploy your local site to staging'
 	@echo
 
 up: export DEBUG = 'knex:query'
@@ -60,7 +61,7 @@ up: require create-if-missing.env ../owid-content tmp-downloads/owid_metadata.sq
 	@echo '==> Starting dev environment'
 	@mkdir -p logs
 	tmux new-session -s grapher \
-		-n docker 'docker-compose -f docker-compose.grapher.yml up' \; \
+		-n docker 'docker compose -f docker-compose.grapher.yml up' \; \
 			set remain-on-exit on \; \
 		set-option -g default-shell $(SCRIPT_SHELL) \; \
 		new-window -n admin \
@@ -114,7 +115,7 @@ up.full: require create-if-missing.env.full ../owid-content wordpress/.env tmp-d
 
 	@echo '==> Starting dev environment'
 	tmux new-session -s grapher \
-		-n docker 'docker-compose -f docker-compose.full.yml up' \; \
+		-n docker 'docker compose -f docker-compose.full.yml up' \; \
 			set remain-on-exit on \; \
 		set-option -g default-shell $(SCRIPT_SHELL) \; \
 		new-window -n admin \
@@ -135,7 +136,7 @@ up.full: require create-if-missing.env.full ../owid-content wordpress/.env tmp-d
 
 migrate:
 	@echo '==> Running DB migrations'
-	yarn && yarn buildTsc && yarn runDbMigrations
+	rm -rf itsJustJavascript && yarn && yarn buildLerna && yarn buildTsc && yarn runDbMigrations
 
 refresh.full: refresh refresh.wp sync-images
 
@@ -178,15 +179,15 @@ sync-images.preflight-check:
 
 down:
 	@echo '==> Stopping services'
-	docker-compose -f docker-compose.grapher.yml down
+	docker compose -f docker-compose.grapher.yml down
 
 down.full:
 	@echo '==> Stopping services'
-	docker-compose -f docker-compose.full.yml down
+	docker compose -f docker-compose.full.yml down
 
 require:
 	@echo '==> Checking your local environment has the necessary commands...'
-	@which docker-compose >/dev/null 2>&1 || (echo "ERROR: docker-compose is required."; exit 1)
+	@which docker >/dev/null 2>&1 || (echo "ERROR: docker compose is required."; exit 1)
 	@which yarn >/dev/null 2>&1 || (echo "ERROR: yarn is required."; exit 1)
 	@which tmux >/dev/null 2>&1 || (echo "ERROR: tmux is required."; exit 1)
 	@which finger >/dev/null 2>&1 || (echo "ERROR: finger is required."; exit 1)
@@ -348,12 +349,16 @@ itsJustJavascript: node_modules
 	yarn run tsc -b
 	touch $@
 
+update.chart-entities: itsJustJavascript
+	@echo '==> Updating chart entities table'
+	node --enable-source-maps itsJustJavascript/baker/updateChartEntities.js --all
+
 reindex: itsJustJavascript
 	@echo '==> Reindexing search in Algolia'
 	node --enable-source-maps itsJustJavascript/baker/algolia/configureAlgolia.js
 	node --enable-source-maps itsJustJavascript/baker/algolia/indexToAlgolia.js
 	node --enable-source-maps itsJustJavascript/baker/algolia/indexChartsToAlgolia.js
-	node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorersToAlgolia.js
+	node --enable-source-maps itsJustJavascript/baker/algolia/indexExplorerViewsToAlgolia.js
 
 bench.search: itsJustJavascript
 	@echo '==> Running search benchmarks'

diff --git a/README.md b/README.md
@@ -70,6 +70,8 @@ Much of our code is based around [reactive programming](https://en.wikipedia.org
 
 All non-WordPress code is written in [TypeScript](https://www.typescriptlang.org/).
 
+If you want to enable pre-commit hooks, run `yarn husky`.
+
 [Visual Studio Code](https://code.visualstudio.com/) is recommended for autocompletion and other awesome editor analysis features enabled by static typing.
 
 ## Why did we start this project?

diff --git a/adminSiteClient/Admin.tsx b/adminSiteClient/Admin.tsx
@@ -163,6 +163,7 @@ export class Admin {
     }
 
     @action.bound private removeRequest(request: Promise<Response>): void {
+        // eslint-disable-next-line @typescript-eslint/no-floating-promises
         this.currentRequests = this.currentRequests.filter(
             (req) => req !== request
         )

diff --git a/adminSiteClient/AdminApp.tsx b/adminSiteClient/AdminApp.tsx
@@ -13,6 +13,7 @@ import { DatasetEditPage } from "./DatasetEditPage.js"
 import { VariablesAnnotationPage } from "./VariablesAnnotationPage.js"
 import { SourceEditPage } from "./SourceEditPage.js"
 import { RedirectsIndexPage } from "./RedirectsIndexPage.js"
+import SiteRedirectsIndexPage from "./SiteRedirectsIndexPage"
 import { TagEditPage } from "./TagEditPage.js"
 import { TagsIndexPage } from "./TagsIndexPage.js"
 import { PostsIndexPage } from "./PostsIndexPage.js"
@@ -23,7 +24,6 @@ import { DeployStatusPage } from "./DeployStatusPage.js"
 import { ExplorerTagsPage } from "./ExplorerTagsPage.js"
 import { SuggestedChartRevisionApproverPage } from "./SuggestedChartRevisionApproverPage.js"
 import { SuggestedChartRevisionListPage } from "./SuggestedChartRevisionListPage.js"
-import { SuggestedChartRevisionImportPage } from "./SuggestedChartRevisionImportPage.js"
 import { BulkDownloadPage } from "./BulkDownloadPage.js"
 import {
     BrowserRouter as Router,
@@ -249,6 +249,11 @@ export class AdminApp extends React.Component<{
                                 path="/redirects"
                                 component={RedirectsIndexPage}
                             />
+                            <Route
+                                exact
+                                path="/site-redirects"
+                                component={SiteRedirectsIndexPage}
+                            />
                             <Route
                                 exact
                                 path="/tags/:tagId"
@@ -314,11 +319,6 @@ export class AdminApp extends React.Component<{
                                 path="/suggested-chart-revisions"
                                 component={SuggestedChartRevisionListPage}
                             />
-                            <Route
-                                exact
-                                path="/suggested-chart-revisions/import"
-                                component={SuggestedChartRevisionImportPage}
-                            />
                             <Route
                                 exact
                                 path="/suggested-chart-revisions/review"

diff --git a/adminSiteClient/AdminSidebar.tsx b/adminSiteClient/AdminSidebar.tsx
@@ -114,7 +114,12 @@ export const AdminSidebar = (): JSX.Element => (
             </li>
             <li>
                 <Link to="/redirects">
-                    <FontAwesomeIcon icon={faArrowRight} /> Redirects
+                    <FontAwesomeIcon icon={faArrowRight} /> Chart Redirects
+                </Link>
+            </li>
+            <li>
+                <Link to="/site-redirects">
+                    <FontAwesomeIcon icon={faArrowRight} /> Site Redirects
                 </Link>
             </li>
             <li>
-Original file line number
+Diff line change
@@ Expand Up / @@ -129,4 +129,4 @@ @@
                 "port": 9000
             }
         ]
-    }
+    }