🎉 (entity selector) sort by external indicators

adminSiteClient/EditorBasicTab.tsx

@@ -13,7 +13,12 @@ import {
     EntitySelectionMode,
     StackMode,
 } from "@ourworldindata/types"
-import { DimensionSlot, WorldEntityName } from "@ourworldindata/grapher"
+import {
+    DimensionSlot,
+    WorldEntityName,
+    CONTINENTS_INDICATOR_ID,
+    POPULATION_INDICATOR_ID_USED_IN_ADMIN,
+} from "@ourworldindata/grapher"
 import {
     DimensionProperty,
     moveArrayItemToIndex,
@@ -317,7 +322,7 @@ export class EditorBasicTab extends React.Component<{ editor: ChartEditor }> {
             )
             if (!hasColor)
                 grapher.addDimension({
-                    variableId: 123, // "Countries Continents"
+                    variableId: CONTINENTS_INDICATOR_ID,
                     property: DimensionProperty.color,
                 })
         }
@@ -329,7 +334,7 @@ export class EditorBasicTab extends React.Component<{ editor: ChartEditor }> {
             )
             if (!hasSize)
                 grapher.addDimension({
-                    variableId: 597929, // "Population (various sources, 2023.1)"
+                    variableId: POPULATION_INDICATOR_ID_USED_IN_ADMIN,
                     property: DimensionProperty.size,
                 })
         }

packages/@ourworldindata/grapher/src/core/Grapher.tsx

@@ -118,8 +118,6 @@ import {
     DEFAULT_GRAPHER_CONFIG_SCHEMA,
     DEFAULT_GRAPHER_WIDTH,
     DEFAULT_GRAPHER_HEIGHT,
-    getVariableDataRoute,
-    getVariableMetadataRoute,
     DEFAULT_GRAPHER_FRAME_PADDING,
     DEFAULT_GRAPHER_ENTITY_TYPE,
     DEFAULT_GRAPHER_ENTITY_TYPE_PLURAL,
@@ -128,7 +126,10 @@ import {
     GRAPHER_LIGHT_TEXT,
     GRAPHER_LOADED_EVENT_NAME,
     GRAPHER_DRAWER_ID,
+    isContinentsVariableId,
+    isPopulationVariableId,
 } from "../core/GrapherConstants"
+import { loadVariableDataAndMetadata } from "./loadVariable"
 import Cookies from "js-cookie"
 import {
     ChartDimension,
@@ -253,21 +254,9 @@ async function loadVariablesDataSite(
     variableIds: number[],
     dataApiUrl: string
 ): Promise<MultipleOwidVariableDataDimensionsMap> {
-    const loadVariableDataPromises = variableIds.map(async (variableId) => {
-        const dataPromise = fetch(getVariableDataRoute(dataApiUrl, variableId))
-        const metadataPromise = fetch(
-            getVariableMetadataRoute(dataApiUrl, variableId)
-        )
-        const [dataResponse, metadataResponse] = await Promise.all([
-            dataPromise,
-            metadataPromise,
-        ])
-        if (!dataResponse.ok) throw new Error(dataResponse.statusText)
-        if (!metadataResponse.ok) throw new Error(metadataResponse.statusText)
-        const data = await dataResponse.json()
-        const metadata = await metadataResponse.json()
-        return { data, metadata }
-    })
+    const loadVariableDataPromises = variableIds.map((variableId) =>
+        loadVariableDataAndMetadata(variableId, dataApiUrl)
+    )
     const variablesData: OwidVariableDataMetadataDimensions[] =
         await Promise.all(loadVariableDataPromises)
     const variablesDataMap = new Map(
@@ -1577,22 +1566,13 @@ export class Grapher
             this
         const columnSlugs: string[] = []
 
-        // "Countries Continent"
-        const isContinentsVariableId = (id: string): boolean => id === "123"
-
         // exclude "Countries Continent" if it's used as the color dimension in a scatter plot, slope chart etc.
         if (
             colorColumnSlug !== undefined &&
             !isContinentsVariableId(colorColumnSlug)
         )
             columnSlugs.push(colorColumnSlug)
 
-        const isPopulationVariableId = (id: string): boolean =>
-            id === "525709" || // "Population (historical + projections), Gapminder, HYDE & UN"
-            id === "525711" || // "Population (historical estimates), Gapminder, HYDE & UN"
-            id === "597929" || // "Population (various sources, 2023.1)"
-            id === "597930" // "Population (various sources, 2023.1)"
-
         if (xColumnSlug !== undefined) {
             // exclude population variable if it's used as the x dimension in a marimekko
             if (!isMarimekko || !isPopulationVariableId(xColumnSlug))

packages/@ourworldindata/grapher/src/core/GrapherConstants.ts

@@ -53,28 +53,22 @@ export const ThereWasAProblemLoadingThisChart = `There was a problem loading thi
 
 export const WorldEntityName = "World"
 
-export const getVariableDataRoute = (
-    dataApiUrl: string,
-    variableId: number
-): string => {
-    if (dataApiUrl.includes("v1/indicators/")) {
-        // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.data.json
-        return `${dataApiUrl}${variableId}.data.json`
-    } else {
-        throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`)
-    }
-}
-
-export const getVariableMetadataRoute = (
-    dataApiUrl: string,
-    variableId: number
-): string => {
-    if (dataApiUrl.includes("v1/indicators/")) {
-        // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.metadata.json
-        return `${dataApiUrl}${variableId}.metadata.json`
-    } else {
-        throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`)
-    }
+export const CONTINENTS_INDICATOR_ID = 123 // "Countries Continent"
+export const POPULATION_INDICATOR_ID_USED_IN_ADMIN = 597929 // "Population (various sources, 2023.1)"
+export const POPULATION_INDICATOR_ID_USED_IN_ENTITY_SELECTOR = 597930 // "Population (various sources, 2023.1)"
+export const GDP_PER_CAPITA_INDICATOR_ID_USED_IN_ENTITY_SELECTOR = 735665 // "World Development Indicators - World Bank (2023.05.11)"
+
+export const isContinentsVariableId = (id: string | number): boolean =>
+    id.toString() === CONTINENTS_INDICATOR_ID.toString()
+
+export const isPopulationVariableId = (id: string | number): boolean => {
+    const idString = id.toString()
+    return (
+        idString === "525709" || // "Population (historical + projections), Gapminder, HYDE & UN"
+        idString === "525711" || // "Population (historical estimates), Gapminder, HYDE & UN"
+        idString === "597929" || // "Population (various sources, 2023.1)"
+        idString === "597930" // "Population (various sources, 2023.1)"
+    )
 }
 
 export enum Patterns {

packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts

@@ -9,6 +9,7 @@ import {
     OwidColumnDef,
     LegacyGrapherInterface,
     OwidVariableDimensions,
+    OwidVariableDataMetadataDimensions,
 } from "@ourworldindata/types"
 import {
     OwidTable,
@@ -37,6 +38,7 @@ import {
     OwidChartDimensionInterface,
     OwidVariableType,
 } from "@ourworldindata/utils"
+import { isContinentsVariableId } from "./GrapherConstants"
 
 export const legacyToOwidTableAndDimensions = (
     json: MultipleOwidVariableDataDimensionsMap,
@@ -613,7 +615,7 @@ const columnDefFromOwidVariable = (
     } = variable
 
     // Without this the much used var 123 appears as "Countries Continent". We could rename in Grapher but not sure the effects of that.
-    const isContinent = variable.id === 123
+    const isContinent = isContinentsVariableId(variable.id)
     const name = isContinent ? "Continent" : variable.name
 
     // The column's type
@@ -738,3 +740,78 @@ const annotationsToMap = (annotations: string): Map<string, string> => {
     })
     return entityAnnotationsMap
 }
+
+/**
+ * Loads a single variable into an OwidTable.
+ */
+export function buildVariableTable(
+    variable: OwidVariableDataMetadataDimensions
+): OwidTable {
+    const entityMeta = variable.metadata.dimensions.entities.values
+    const entityMetaById: OwidEntityKey = Object.fromEntries(
+        entityMeta.map((entity) => [entity.id.toString(), entity])
+    )
+
+    // Base column defs, present in all OwidTables
+    const baseColumnDefs: Map<ColumnSlug, CoreColumnDef> = new Map()
+    StandardOwidColumnDefs.forEach((def) => {
+        baseColumnDefs.set(def.slug, def)
+    })
+
+    const columnDefs = new Map(baseColumnDefs)
+
+    // Time column
+    const timeColumnDef = timeColumnDefFromOwidVariable(variable.metadata)
+    columnDefs.set(timeColumnDef.slug, timeColumnDef)
+
+    // Value column
+    const valueColumnDef = columnDefFromOwidVariable(variable.metadata)
+    // Because database columns can contain mixed types, we want to avoid
+    // parsing for Grapher data until we fix that.
+    valueColumnDef.skipParsing = true
+    columnDefs.set(valueColumnDef.slug, valueColumnDef)
+
+    // Column values
+
+    const times = timeColumnValuesFromOwidVariable(
+        variable.metadata,
+        variable.data
+    )
+    const entityIds = variable.data.entities ?? []
+    const entityNames = entityIds.map(
+        // if entityMetaById[id] does not exist, then we don't have entity
+        // from variable metadata in MySQL. This can happen because we take
+        // data from S3 and metadata from MySQL. After we unify it, it should
+        // no longer be a problem
+        (id) => entityMetaById[id]?.name ?? id.toString()
+    )
+    // see comment above about entityMetaById[id]
+    const entityCodes = entityIds.map((id) => entityMetaById[id]?.code)
+
+    // If there is a conversionFactor, apply it.
+    let values = variable.data.values || []
+    const conversionFactor = valueColumnDef.display?.conversionFactor
+    if (conversionFactor !== undefined) {
+        values = values.map((value) =>
+            isNumber(value) ? value * conversionFactor : value
+        )
+
+        // If a non-int conversion factor is applied to an integer column,
+        // we end up with a numeric column.
+        if (
+            valueColumnDef.type === ColumnTypeNames.Integer &&
+            !isInteger(conversionFactor)
+        )
+            valueColumnDef.type = ColumnTypeNames.Numeric
+    }
+
+    const columnStore: { [key: string]: any[] } = {
+        [OwidTableSlugs.entityId]: entityIds,
+        [OwidTableSlugs.entityCode]: entityCodes,
+        [OwidTableSlugs.entityName]: entityNames,
+        [timeColumnDef.slug]: times,
+        [valueColumnDef.slug]: values,
+    }
+
+    return new OwidTable(columnStore, Array.from(columnDefs.values()))
+}

packages/@ourworldindata/grapher/src/core/loadVariable.ts

@@ -0,0 +1,44 @@
+import { OwidVariableDataMetadataDimensions } from "@ourworldindata/types"
+
+export const getVariableDataRoute = (
+    dataApiUrl: string,
+    variableId: number
+): string => {
+    if (dataApiUrl.includes("v1/indicators/")) {
+        // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.data.json
+        return `${dataApiUrl}${variableId}.data.json`
+    } else {
+        throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`)
+    }
+}
+
+export const getVariableMetadataRoute = (
+    dataApiUrl: string,
+    variableId: number
+): string => {
+    if (dataApiUrl.includes("v1/indicators/")) {
+        // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.metadata.json
+        return `${dataApiUrl}${variableId}.metadata.json`
+    } else {
+        throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`)
+    }
+}
+
+export async function loadVariableDataAndMetadata(
+    variableId: number,
+    dataApiUrl: string
+): Promise<OwidVariableDataMetadataDimensions> {
+    const dataPromise = fetch(getVariableDataRoute(dataApiUrl, variableId))
+    const metadataPromise = fetch(
+        getVariableMetadataRoute(dataApiUrl, variableId)
+    )
+    const [dataResponse, metadataResponse] = await Promise.all([
+        dataPromise,
+        metadataPromise,
+    ])
+    if (!dataResponse.ok) throw new Error(dataResponse.statusText)
+    if (!metadataResponse.ok) throw new Error(metadataResponse.statusText)
+    const data = await dataResponse.json()
+    const metadata = await metadataResponse.json()
+    return { data, metadata }
+}