Merge remote-tracking branch 'origin/master' into explorer-name-stand…

…ardization

Makefile

@@ -56,6 +56,7 @@ up: require create-if-missing.env ../owid-content tmp-downloads/owid_metadata.sq
 	yarn run tsc -b
 
 	@echo '==> Starting dev environment'
+	@mkdir -p logs
 	tmux new-session -s grapher \
 		-n docker 'docker-compose -f docker-compose.grapher.yml up' \; \
 			set remain-on-exit on \; \
@@ -83,6 +84,7 @@ up.devcontainer: create-if-missing.env.devcontainer tmp-downloads/owid_metadata.
 	yarn run tsc -b
 
 	@echo '==> Starting dev environment'
+	@mkdir -p logs
 	tmux new-session -s grapher \
 		-n admin \
 			'devTools/docker/wait-for-mysql.sh && yarn run tsc-watch -b --onSuccess "yarn startAdminServer"' \; \

adminSiteClient/AdminSidebar.tsx

@@ -5,7 +5,6 @@ import {
     faChartBar,
     faFile,
     faTable,
-    faTruckFast,
     faSkullCrossbones,
     faPen,
     faDatabase,
@@ -19,9 +18,10 @@ import {
     faSatelliteDish,
     faCodeBranch,
     faDownload,
+    faHatWizard,
 } from "@fortawesome/free-solid-svg-icons"
 
-import { FASTTRACK_URL } from "../settings/clientSettings.js"
+import { ETL_WIZARD_URL } from "../settings/clientSettings.js"
 
 export const AdminSidebar = (): JSX.Element => (
     <aside className="AdminSidebar">
@@ -56,12 +56,12 @@ export const AdminSidebar = (): JSX.Element => (
 
             <li>
                 <a
-                    href={FASTTRACK_URL}
+                    href={ETL_WIZARD_URL}
                     target="_blank"
                     rel="noopener"
                     title="Tailscale required"
                 >
-                    <FontAwesomeIcon icon={faTruckFast} /> Fast-track
+                    <FontAwesomeIcon icon={faHatWizard} /> Wizard
                 </a>
             </li>
             <li>

adminSiteClient/ChartEditorPage.tsx

@@ -304,7 +304,10 @@ export class ChartEditorPage
         this.fetchRefs()
         this.fetchRedirects()
         this.fetchPageviews()
-        this.fetchTopics()
+
+        // (2024-02-15) Disabled due to slow query performance
+        // https://github.com/owid/owid-grapher/issues/3198
+        // this.fetchTopics()
     }
 
     disposers: IReactionDisposer[] = []

adminSiteClient/gdocsDeploy.ts

@@ -1,12 +1,11 @@
+import { isEqual, omit } from "@ourworldindata/utils"
 import {
     OwidGdoc,
     OwidGdocBaseInterface,
     OwidGdocPostContent,
-    isEqual,
-    omit,
     OwidGdocDataInsightContent,
     OwidGdocType,
-} from "@ourworldindata/utils"
+} from "@ourworldindata/types"
 import { GDOC_DIFF_OMITTABLE_PROPERTIES } from "./GdocsDiff.js"
 import { GDOCS_DETAILS_ON_DEMAND_ID } from "../settings/clientSettings.js"
 
@@ -52,6 +51,7 @@ export const checkIsLightningUpdate = (
         breadcrumbs: true,
         errors: true,
         linkedCharts: true,
+        linkedIndicators: true,
         linkedDocuments: true,
         relatedCharts: true,
         revisionId: true,

baker/GDriveImagesBaker.tsx

@@ -52,24 +52,29 @@ export const bakeDriveImages = async (bakedSiteDir: string) => {
                         headers: {
                             "If-None-Match": existingEtag,
                         },
+                    }).then((response) => {
+                        if (response.status === 304) {
+                            // Image has not been modified, skip without logging
+                            return response
+                        } else if (response.ok) {
+                            // Log fetched images if it was success but wasn't 304
+                            console.log(
+                                `Fetching image ${image.filename} from ${remoteFilePath} using etag ${existingEtag}...`
+                            )
+                            return response
+                        } else {
+                            // If the response status is 404, throw an error to trigger retry
+                            const msg = `Fetching image failed: ${response.status} ${response.statusText} ${response.url}`
+                            console.log(msg)
+                            throw new Error(msg)
+                        }
                     }),
                 { maxRetries: 5, exponentialBackoff: true, initialDelay: 1000 }
             )
 
             // Image has not been modified, skip
             if (response.status === 304) {
                 return
-            } else {
-                // Log fetched images, this should be pretty rare as most images should return 304
-                console.log(
-                    `Fetching image ${image.filename} from ${remoteFilePath} using etag ${existingEtag}...`
-                )
-            }
-
-            if (!response.ok) {
-                throw new Error(
-                    `Fetching image failed: ${response.status} ${response.statusText} ${response.url}`
-                )
             }
 
             let buffer = Buffer.from(await response.arrayBuffer())

baker/GrapherBaker.tsx

@@ -7,22 +7,12 @@ import {
     urlToSlug,
     without,
     deserializeJSONFromHTML,
-    OwidVariableDataMetadataDimensions,
     uniq,
-    JsonError,
     keyBy,
-    DimensionProperty,
-    OwidVariableWithSource,
     mergePartialGrapherConfigs,
-    OwidChartDimensionInterface,
     compact,
-    OwidGdocPostInterface,
     merge,
-    EnrichedFaq,
-    FaqEntryData,
-    FaqDictionary,
     partition,
-    ImageMetadata,
 } from "@ourworldindata/utils"
 import {
     getRelatedArticles,
@@ -45,13 +35,25 @@ import * as db from "../db/db.js"
 import { glob } from "glob"
 import { isPathRedirectedToExplorer } from "../explorerAdminServer/ExplorerRedirects.js"
 import { getPostEnrichedBySlug } from "../db/model/Post.js"
-import { ChartTypeName, GrapherInterface } from "@ourworldindata/types"
+import {
+    JsonError,
+    GrapherInterface,
+    OwidVariableDataMetadataDimensions,
+    DimensionProperty,
+    OwidVariableWithSource,
+    OwidChartDimensionInterface,
+    OwidGdocPostInterface,
+    EnrichedFaq,
+    FaqEntryData,
+    FaqDictionary,
+    ImageMetadata,
+} from "@ourworldindata/types"
 import workerpool from "workerpool"
 import ProgressBar from "progress"
 import {
     getVariableData,
-    getVariableMetadata,
     getMergedGrapherConfigForVariable,
+    getVariableOfDatapageIfApplicable,
 } from "../db/model/Variable.js"
 import { getDatapageDataV2, getDatapageGdoc } from "../datapage/Datapage.js"
 import { Image } from "../db/model/Image.js"
@@ -60,57 +62,25 @@ import { logErrorAndMaybeSendToBugsnag } from "../serverUtils/errorLog.js"
 import { parseFaqs } from "../db/model/Gdoc/rawToEnriched.js"
 import { GdocPost } from "../db/model/Gdoc/GdocPost.js"
 import { getShortPageCitation } from "../site/gdocs/utils.js"
-import { isEmpty } from "lodash"
 import { getSlugForTopicTag, getTagToSlugMap } from "./GrapherBakingUtils.js"
 
 const renderDatapageIfApplicable = async (
     grapher: GrapherInterface,
     isPreviewing: boolean,
     imageMetadataDictionary?: Record<string, Image>
 ) => {
-    // If we have a single Y variable and that one has a schema version >= 2,
-    // meaning it has the metadata to render a datapage, AND if the metadata includes
-    // text for at least one of the description* fields or titlePublic, then we show the datapage
-    // based on this information.
-    const yVariableIds = grapher
-        .dimensions!.filter((d) => d.property === DimensionProperty.y)
-        .map((d) => d.variableId)
-    const xVariableIds = grapher
-        .dimensions!.filter((d) => d.property === DimensionProperty.x)
-        .map((d) => d.variableId)
-    // Make a data page for single indicator indicator charts.
-    // For scatter plots we want to only show a data page if it has no X variable mapped, which
-    // is a special case where time is the X axis. Marimekko charts are the other chart that uses
-    // the X dimension but there we usually map population on X which should not prevent us from
-    // showing a data page.
-    if (
-        yVariableIds.length === 1 &&
-        (grapher.type !== ChartTypeName.ScatterPlot ||
-            xVariableIds.length === 0)
-    ) {
-        const variableId = yVariableIds[0]
-        const variableMetadata = await getVariableMetadata(variableId)
-
-        if (
-            variableMetadata.schemaVersion !== undefined &&
-            variableMetadata.schemaVersion >= 2 &&
-            (!isEmpty(variableMetadata.descriptionShort) ||
-                !isEmpty(variableMetadata.descriptionProcessing) ||
-                !isEmpty(variableMetadata.descriptionKey) ||
-                !isEmpty(variableMetadata.descriptionFromProducer) ||
-                !isEmpty(variableMetadata.presentation?.titlePublic))
-        ) {
-            return await renderDataPageV2({
-                variableId,
-                variableMetadata,
-                isPreviewing: isPreviewing,
-                useIndicatorGrapherConfigs: false,
-                pageGrapher: grapher,
-                imageMetadataDictionary,
-            })
-        }
-    }
-    return undefined
+    const variable = await getVariableOfDatapageIfApplicable(grapher)
+
+    if (!variable) return undefined
+
+    return await renderDataPageV2({
+        variableId: variable.id,
+        variableMetadata: variable.metadata,
+        isPreviewing: isPreviewing,
+        useIndicatorGrapherConfigs: false,
+        pageGrapher: grapher,
+        imageMetadataDictionary,
+    })
 }
 
 /**