🎉 (entity selector) sort by external indicators

owid · Apr 9, 2024 · 85b3c66 · 85b3c66
1 parent a5a8afc
commit 85b3c66
Show file tree

Hide file tree

Showing 6 changed files with 284 additions and 60 deletions.
diff --git a/packages/@ourworldindata/grapher/src/core/Grapher.tsx b/packages/@ourworldindata/grapher/src/core/Grapher.tsx
@@ -118,8 +118,7 @@ import {
     DEFAULT_GRAPHER_CONFIG_SCHEMA,
     DEFAULT_GRAPHER_WIDTH,
     DEFAULT_GRAPHER_HEIGHT,
-    getVariableDataRoute,
-    getVariableMetadataRoute,
+    isPopulationVariableId,
     DEFAULT_GRAPHER_FRAME_PADDING,
     DEFAULT_GRAPHER_ENTITY_TYPE,
     DEFAULT_GRAPHER_ENTITY_TYPE_PLURAL,
@@ -129,6 +128,7 @@ import {
     GRAPHER_LOADED_EVENT_NAME,
     GRAPHER_DRAWER_ID,
 } from "../core/GrapherConstants"
+import { loadDataAndMetadataForVariable } from "./loadVariable"
 import Cookies from "js-cookie"
 import {
     ChartDimension,
@@ -253,21 +253,9 @@ async function loadVariablesDataSite(
     variableIds: number[],
     dataApiUrl: string
 ): Promise<MultipleOwidVariableDataDimensionsMap> {
-    const loadVariableDataPromises = variableIds.map(async (variableId) => {
-        const dataPromise = fetch(getVariableDataRoute(dataApiUrl, variableId))
-        const metadataPromise = fetch(
-            getVariableMetadataRoute(dataApiUrl, variableId)
-        )
-        const [dataResponse, metadataResponse] = await Promise.all([
-            dataPromise,
-            metadataPromise,
-        ])
-        if (!dataResponse.ok) throw new Error(dataResponse.statusText)
-        if (!metadataResponse.ok) throw new Error(metadataResponse.statusText)
-        const data = await dataResponse.json()
-        const metadata = await metadataResponse.json()
-        return { data, metadata }
-    })
+    const loadVariableDataPromises = variableIds.map((variableId) =>
+        loadDataAndMetadataForVariable(variableId, dataApiUrl)
+    )
     const variablesData: OwidVariableDataMetadataDimensions[] =
         await Promise.all(loadVariableDataPromises)
     const variablesDataMap = new Map(
@@ -1591,12 +1579,6 @@ export class Grapher
         )
             columnSlugs.push(colorColumnSlug)
 
-        const isPopulationVariableId = (id: string): boolean =>
-            id === "525709" || // "Population (historical + projections), Gapminder, HYDE & UN"
-            id === "525711" || // "Population (historical estimates), Gapminder, HYDE & UN"
-            id === "597929" || // "Population (various sources, 2023.1)"
-            id === "597930" // "Population (various sources, 2023.1)"
-
         if (xColumnSlug !== undefined) {
             // exclude population variable if it's used as the x dimension in a marimekko
             if (!isMarimekko || !isPopulationVariableId(xColumnSlug))

diff --git a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts
@@ -50,29 +50,11 @@ export const ThereWasAProblemLoadingThisChart = `There was a problem loading thi
 
 export const WorldEntityName = "World"
 
-export const getVariableDataRoute = (
-    dataApiUrl: string,
-    variableId: number
-): string => {
-    if (dataApiUrl.includes("v1/indicators/")) {
-        // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.data.json
-        return `${dataApiUrl}${variableId}.data.json`
-    } else {
-        throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`)
-    }
-}
-
-export const getVariableMetadataRoute = (
-    dataApiUrl: string,
-    variableId: number
-): string => {
-    if (dataApiUrl.includes("v1/indicators/")) {
-        // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.metadata.json
-        return `${dataApiUrl}${variableId}.metadata.json`
-    } else {
-        throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`)
-    }
-}
+export const isPopulationVariableId = (id: string): boolean =>
+    id === "525709" || // "Population (historical + projections), Gapminder, HYDE & UN"
+    id === "525711" || // "Population (historical estimates), Gapminder, HYDE & UN"
+    id === "597929" || // "Population (various sources, 2023.1)"
+    id === "597930" // "Population (various sources, 2023.1)"
 
 export enum Patterns {
     noDataPattern = "noDataPattern",

diff --git a/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts b/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts
@@ -9,6 +9,7 @@ import {
     OwidColumnDef,
     LegacyGrapherInterface,
     OwidVariableDimensions,
+    OwidVariableDataMetadataDimensions,
 } from "@ourworldindata/types"
 import {
     OwidTable,
@@ -738,3 +739,78 @@ const annotationsToMap = (annotations: string): Map<string, string> => {
     })
     return entityAnnotationsMap
 }
+
+/**
+ * Loads a single variable into an OwidTable.
+ */
+export function buildVariableTable(
+    variable: OwidVariableDataMetadataDimensions
+): OwidTable {
+    const entityMeta = variable.metadata.dimensions.entities.values
+    const entityMetaById: OwidEntityKey = Object.fromEntries(
+        entityMeta.map((entity) => [entity.id.toString(), entity])
+    )
+
+    // Base column defs, present in all OwidTables
+    const baseColumnDefs: Map<ColumnSlug, CoreColumnDef> = new Map()
+    StandardOwidColumnDefs.forEach((def) => {
+        baseColumnDefs.set(def.slug, def)
+    })
+
+    const columnDefs = new Map(baseColumnDefs)
+
+    // Time column
+    const timeColumnDef = timeColumnDefFromOwidVariable(variable.metadata)
+    columnDefs.set(timeColumnDef.slug, timeColumnDef)
+
+    // Value column
+    const valueColumnDef = columnDefFromOwidVariable(variable.metadata)
+    // Because database columns can contain mixed types, we want to avoid
+    // parsing for Grapher data until we fix that.
+    valueColumnDef.skipParsing = true
+    columnDefs.set(valueColumnDef.slug, valueColumnDef)
+
+    // Column values
+
+    const times = timeColumnValuesFromOwidVariable(
+        variable.metadata,
+        variable.data
+    )
+    const entityIds = variable.data.entities ?? []
+    const entityNames = entityIds.map(
+        // if entityMetaById[id] does not exist, then we don't have entity
+        // from variable metadata in MySQL. This can happen because we take
+        // data from S3 and metadata from MySQL. After we unify it, it should
+        // no longer be a problem
+        (id) => entityMetaById[id]?.name ?? id.toString()
+    )
+    // see comment above about entityMetaById[id]
+    const entityCodes = entityIds.map((id) => entityMetaById[id]?.code)
+
+    // If there is a conversionFactor, apply it.
+    let values = variable.data.values || []
+    const conversionFactor = valueColumnDef.display?.conversionFactor
+    if (conversionFactor !== undefined) {
+        values = values.map((value) =>
+            isNumber(value) ? value * conversionFactor : value
+        )
+
+        // If a non-int conversion factor is applied to an integer column,
+        // we end up with a numeric column.
+        if (
+            valueColumnDef.type === ColumnTypeNames.Integer &&
+            !isInteger(conversionFactor)
+        )
+            valueColumnDef.type = ColumnTypeNames.Numeric
+    }
+
+    const columnStore: { [key: string]: any[] } = {
+        [OwidTableSlugs.entityId]: entityIds,
+        [OwidTableSlugs.entityCode]: entityCodes,
+        [OwidTableSlugs.entityName]: entityNames,
+        [timeColumnDef.slug]: times,
+        [valueColumnDef.slug]: values,
+    }
+
+    return new OwidTable(columnStore, Array.from(columnDefs.values()))
+}
diff --git a/packages/@ourworldindata/grapher/src/core/loadVariable.ts b/packages/@ourworldindata/grapher/src/core/loadVariable.ts
@@ -0,0 +1,44 @@
+import { OwidVariableDataMetadataDimensions } from "@ourworldindata/types"
+
+export const getVariableDataRoute = (
+    dataApiUrl: string,
+    variableId: number
+): string => {
+    if (dataApiUrl.includes("v1/indicators/")) {
+        // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.data.json
+        return `${dataApiUrl}${variableId}.data.json`
+    } else {
+        throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`)
+    }
+}
+
+export const getVariableMetadataRoute = (
+    dataApiUrl: string,
+    variableId: number
+): string => {
+    if (dataApiUrl.includes("v1/indicators/")) {
+        // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.metadata.json
+        return `${dataApiUrl}${variableId}.metadata.json`
+    } else {
+        throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`)
+    }
+}
+
+export async function loadDataAndMetadataForVariable(
+    variableId: number,
+    dataApiUrl: string
+): Promise<OwidVariableDataMetadataDimensions> {
+    const dataPromise = fetch(getVariableDataRoute(dataApiUrl, variableId))
+    const metadataPromise = fetch(
+        getVariableMetadataRoute(dataApiUrl, variableId)
+    )
+    const [dataResponse, metadataResponse] = await Promise.all([
+        dataPromise,
+        metadataPromise,
+    ])
+    if (!dataResponse.ok) throw new Error(dataResponse.statusText)
+    if (!metadataResponse.ok) throw new Error(metadataResponse.statusText)
+    const data = await dataResponse.json()
+    const metadata = await metadataResponse.json()
+    return { data, metadata }
+}