diff --git a/adminSiteClient/EditorBasicTab.tsx b/adminSiteClient/EditorBasicTab.tsx index 72de8a6e6e7..c2e4994e689 100644 --- a/adminSiteClient/EditorBasicTab.tsx +++ b/adminSiteClient/EditorBasicTab.tsx @@ -13,7 +13,12 @@ import { EntitySelectionMode, StackMode, } from "@ourworldindata/types" -import { DimensionSlot, WorldEntityName } from "@ourworldindata/grapher" +import { + DimensionSlot, + WorldEntityName, + CONTINENTS_INDICATOR_ID, + POPULATION_INDICATOR_ID_USED_IN_ADMIN, +} from "@ourworldindata/grapher" import { DimensionProperty, moveArrayItemToIndex, @@ -317,7 +322,7 @@ export class EditorBasicTab extends React.Component<{ editor: ChartEditor }> { ) if (!hasColor) grapher.addDimension({ - variableId: 123, // "Countries Continents" + variableId: CONTINENTS_INDICATOR_ID, property: DimensionProperty.color, }) } @@ -329,7 +334,7 @@ export class EditorBasicTab extends React.Component<{ editor: ChartEditor }> { ) if (!hasSize) grapher.addDimension({ - variableId: 597929, // "Population (various sources, 2023.1)" + variableId: POPULATION_INDICATOR_ID_USED_IN_ADMIN, property: DimensionProperty.size, }) } diff --git a/packages/@ourworldindata/grapher/src/core/Grapher.tsx b/packages/@ourworldindata/grapher/src/core/Grapher.tsx index a562def88e5..f8ebc3da7af 100644 --- a/packages/@ourworldindata/grapher/src/core/Grapher.tsx +++ b/packages/@ourworldindata/grapher/src/core/Grapher.tsx @@ -118,8 +118,6 @@ import { DEFAULT_GRAPHER_CONFIG_SCHEMA, DEFAULT_GRAPHER_WIDTH, DEFAULT_GRAPHER_HEIGHT, - getVariableDataRoute, - getVariableMetadataRoute, DEFAULT_GRAPHER_FRAME_PADDING, DEFAULT_GRAPHER_ENTITY_TYPE, DEFAULT_GRAPHER_ENTITY_TYPE_PLURAL, @@ -128,7 +126,10 @@ import { GRAPHER_LIGHT_TEXT, GRAPHER_LOADED_EVENT_NAME, GRAPHER_DRAWER_ID, + isContinentsVariableId, + isPopulationVariableId, } from "../core/GrapherConstants" +import { loadVariableDataAndMetadata } from "./loadVariable" import Cookies from "js-cookie" import { ChartDimension, @@ -253,21 +254,9 @@ async function loadVariablesDataSite( variableIds: number[], dataApiUrl: string ): Promise { - const loadVariableDataPromises = variableIds.map(async (variableId) => { - const dataPromise = fetch(getVariableDataRoute(dataApiUrl, variableId)) - const metadataPromise = fetch( - getVariableMetadataRoute(dataApiUrl, variableId) - ) - const [dataResponse, metadataResponse] = await Promise.all([ - dataPromise, - metadataPromise, - ]) - if (!dataResponse.ok) throw new Error(dataResponse.statusText) - if (!metadataResponse.ok) throw new Error(metadataResponse.statusText) - const data = await dataResponse.json() - const metadata = await metadataResponse.json() - return { data, metadata } - }) + const loadVariableDataPromises = variableIds.map((variableId) => + loadVariableDataAndMetadata(variableId, dataApiUrl) + ) const variablesData: OwidVariableDataMetadataDimensions[] = await Promise.all(loadVariableDataPromises) const variablesDataMap = new Map( @@ -1577,9 +1566,6 @@ export class Grapher this const columnSlugs: string[] = [] - // "Countries Continent" - const isContinentsVariableId = (id: string): boolean => id === "123" - // exclude "Countries Continent" if it's used as the color dimension in a scatter plot, slope chart etc. if ( colorColumnSlug !== undefined && @@ -1587,12 +1573,6 @@ export class Grapher ) columnSlugs.push(colorColumnSlug) - const isPopulationVariableId = (id: string): boolean => - id === "525709" || // "Population (historical + projections), Gapminder, HYDE & UN" - id === "525711" || // "Population (historical estimates), Gapminder, HYDE & UN" - id === "597929" || // "Population (various sources, 2023.1)" - id === "597930" // "Population (various sources, 2023.1)" - if (xColumnSlug !== undefined) { // exclude population variable if it's used as the x dimension in a marimekko if (!isMarimekko || !isPopulationVariableId(xColumnSlug)) diff --git a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts index 1642f89d893..f747ca28119 100644 --- a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts +++ b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts @@ -53,28 +53,22 @@ export const ThereWasAProblemLoadingThisChart = `There was a problem loading thi export const WorldEntityName = "World" -export const getVariableDataRoute = ( - dataApiUrl: string, - variableId: number -): string => { - if (dataApiUrl.includes("v1/indicators/")) { - // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.data.json - return `${dataApiUrl}${variableId}.data.json` - } else { - throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`) - } -} - -export const getVariableMetadataRoute = ( - dataApiUrl: string, - variableId: number -): string => { - if (dataApiUrl.includes("v1/indicators/")) { - // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.metadata.json - return `${dataApiUrl}${variableId}.metadata.json` - } else { - throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`) - } +export const CONTINENTS_INDICATOR_ID = 123 // "Countries Continent" +export const POPULATION_INDICATOR_ID_USED_IN_ADMIN = 597929 // "Population (various sources, 2023.1)" +export const POPULATION_INDICATOR_ID_USED_IN_ENTITY_SELECTOR = 597930 // "Population (various sources, 2023.1)" +export const GDP_PER_CAPITA_INDICATOR_ID_USED_IN_ENTITY_SELECTOR = 735665 // "World Development Indicators - World Bank (2023.05.11)" + +export const isContinentsVariableId = (id: string | number): boolean => + id.toString() === CONTINENTS_INDICATOR_ID.toString() + +export const isPopulationVariableId = (id: string | number): boolean => { + const idString = id.toString() + return ( + idString === "525709" || // "Population (historical + projections), Gapminder, HYDE & UN" + idString === "525711" || // "Population (historical estimates), Gapminder, HYDE & UN" + idString === "597929" || // "Population (various sources, 2023.1)" + idString === "597930" // "Population (various sources, 2023.1)" + ) } export enum Patterns { diff --git a/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts b/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts index 55afbe6d257..364beec4cc3 100644 --- a/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts +++ b/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts @@ -9,6 +9,7 @@ import { OwidColumnDef, LegacyGrapherInterface, OwidVariableDimensions, + OwidVariableDataMetadataDimensions, } from "@ourworldindata/types" import { OwidTable, @@ -37,6 +38,7 @@ import { OwidChartDimensionInterface, OwidVariableType, } from "@ourworldindata/utils" +import { isContinentsVariableId } from "./GrapherConstants" export const legacyToOwidTableAndDimensions = ( json: MultipleOwidVariableDataDimensionsMap, @@ -613,7 +615,7 @@ const columnDefFromOwidVariable = ( } = variable // Without this the much used var 123 appears as "Countries Continent". We could rename in Grapher but not sure the effects of that. - const isContinent = variable.id === 123 + const isContinent = isContinentsVariableId(variable.id) const name = isContinent ? "Continent" : variable.name // The column's type @@ -738,3 +740,78 @@ const annotationsToMap = (annotations: string): Map => { }) return entityAnnotationsMap } + +/** + * Loads a single variable into an OwidTable. + */ +export function buildVariableTable( + variable: OwidVariableDataMetadataDimensions +): OwidTable { + const entityMeta = variable.metadata.dimensions.entities.values + const entityMetaById: OwidEntityKey = Object.fromEntries( + entityMeta.map((entity) => [entity.id.toString(), entity]) + ) + + // Base column defs, present in all OwidTables + const baseColumnDefs: Map = new Map() + StandardOwidColumnDefs.forEach((def) => { + baseColumnDefs.set(def.slug, def) + }) + + const columnDefs = new Map(baseColumnDefs) + + // Time column + const timeColumnDef = timeColumnDefFromOwidVariable(variable.metadata) + columnDefs.set(timeColumnDef.slug, timeColumnDef) + + // Value column + const valueColumnDef = columnDefFromOwidVariable(variable.metadata) + // Because database columns can contain mixed types, we want to avoid + // parsing for Grapher data until we fix that. + valueColumnDef.skipParsing = true + columnDefs.set(valueColumnDef.slug, valueColumnDef) + + // Column values + + const times = timeColumnValuesFromOwidVariable( + variable.metadata, + variable.data + ) + const entityIds = variable.data.entities ?? [] + const entityNames = entityIds.map( + // if entityMetaById[id] does not exist, then we don't have entity + // from variable metadata in MySQL. This can happen because we take + // data from S3 and metadata from MySQL. After we unify it, it should + // no longer be a problem + (id) => entityMetaById[id]?.name ?? id.toString() + ) + // see comment above about entityMetaById[id] + const entityCodes = entityIds.map((id) => entityMetaById[id]?.code) + + // If there is a conversionFactor, apply it. + let values = variable.data.values || [] + const conversionFactor = valueColumnDef.display?.conversionFactor + if (conversionFactor !== undefined) { + values = values.map((value) => + isNumber(value) ? value * conversionFactor : value + ) + + // If a non-int conversion factor is applied to an integer column, + // we end up with a numeric column. + if ( + valueColumnDef.type === ColumnTypeNames.Integer && + !isInteger(conversionFactor) + ) + valueColumnDef.type = ColumnTypeNames.Numeric + } + + const columnStore: { [key: string]: any[] } = { + [OwidTableSlugs.entityId]: entityIds, + [OwidTableSlugs.entityCode]: entityCodes, + [OwidTableSlugs.entityName]: entityNames, + [timeColumnDef.slug]: times, + [valueColumnDef.slug]: values, + } + + return new OwidTable(columnStore, Array.from(columnDefs.values())) +} diff --git a/packages/@ourworldindata/grapher/src/core/loadVariable.ts b/packages/@ourworldindata/grapher/src/core/loadVariable.ts new file mode 100644 index 00000000000..9ea7bfae79b --- /dev/null +++ b/packages/@ourworldindata/grapher/src/core/loadVariable.ts @@ -0,0 +1,44 @@ +import { OwidVariableDataMetadataDimensions } from "@ourworldindata/types" + +export const getVariableDataRoute = ( + dataApiUrl: string, + variableId: number +): string => { + if (dataApiUrl.includes("v1/indicators/")) { + // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.data.json + return `${dataApiUrl}${variableId}.data.json` + } else { + throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`) + } +} + +export const getVariableMetadataRoute = ( + dataApiUrl: string, + variableId: number +): string => { + if (dataApiUrl.includes("v1/indicators/")) { + // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.metadata.json + return `${dataApiUrl}${variableId}.metadata.json` + } else { + throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`) + } +} + +export async function loadVariableDataAndMetadata( + variableId: number, + dataApiUrl: string +): Promise { + const dataPromise = fetch(getVariableDataRoute(dataApiUrl, variableId)) + const metadataPromise = fetch( + getVariableMetadataRoute(dataApiUrl, variableId) + ) + const [dataResponse, metadataResponse] = await Promise.all([ + dataPromise, + metadataPromise, + ]) + if (!dataResponse.ok) throw new Error(dataResponse.statusText) + if (!metadataResponse.ok) throw new Error(metadataResponse.statusText) + const data = await dataResponse.json() + const metadata = await metadataResponse.json() + return { data, metadata } +} diff --git a/packages/@ourworldindata/grapher/src/entitySelector/EntitySelector.tsx b/packages/@ourworldindata/grapher/src/entitySelector/EntitySelector.tsx index 8f5c2374769..dea9c805a7a 100644 --- a/packages/@ourworldindata/grapher/src/entitySelector/EntitySelector.tsx +++ b/packages/@ourworldindata/grapher/src/entitySelector/EntitySelector.tsx @@ -18,6 +18,8 @@ import { regions, sortBy, Tippy, + excludeUndefined, + intersection, } from "@ourworldindata/utils" import { Checkbox } from "@ourworldindata/components" import { FuzzySearch } from "../controls/FuzzySearch" @@ -36,18 +38,26 @@ import { DEFAULT_GRAPHER_ENTITY_TYPE_PLURAL, GRAPHER_ENTITY_SELECTOR_CLASS, GRAPHER_SCROLLABLE_CONTAINER_CLASS, + POPULATION_INDICATOR_ID_USED_IN_ENTITY_SELECTOR, + GDP_PER_CAPITA_INDICATOR_ID_USED_IN_ENTITY_SELECTOR, + isPopulationVariableId, } from "../core/GrapherConstants" import { CoreColumn, OwidTable } from "@ourworldindata/core-table" import { SortIcon } from "../controls/SortIcon" import { Dropdown } from "../controls/Dropdown" import { scaleLinear, type ScaleLinear } from "d3-scale" import { ColumnSlug } from "@ourworldindata/types" +import { buildVariableTable } from "../core/LegacyToOwidTable" +import { loadVariableDataAndMetadata } from "../core/loadVariable" export interface EntitySelectorState { searchInput: string sortConfig: SortConfig localEntityNames?: string[] mostRecentlySelectedEntityName?: string + populationColumn?: CoreColumn + gdpPerCapitaColumn?: CoreColumn + isLoadingExternalSortColumn?: boolean } export interface EntitySelectorManager { @@ -58,6 +68,7 @@ export interface EntitySelectorManager { entityType?: string entityTypePlural?: string activeColumnSlugs?: string[] + dataApiUrl: string } interface SortConfig { @@ -80,6 +91,14 @@ interface DropdownOption { label: string } +const DEFAULT_POPULATION_INDICATOR_ID = + POPULATION_INDICATOR_ID_USED_IN_ENTITY_SELECTOR +const DEFAULT_POPULATION_LABEL = "Population" + +const DEFAULT_GDP_PER_CAPITA_INDICATOR_ID = + GDP_PER_CAPITA_INDICATOR_ID_USED_IN_ENTITY_SELECTOR +const DEFAULT_GDP_PER_CAPITA_LABEL = "GDP per capita" + @observer export class EntitySelector extends React.Component<{ manager: EntitySelectorManager @@ -216,20 +235,121 @@ export class EntitySelector extends React.Component<{ return this.manager.entitySelectorState.localEntityNames } + @computed private get populationColumn(): CoreColumn | undefined { + return this.manager.entitySelectorState.populationColumn + } + + @computed private get gdpPerCapitaColumn(): CoreColumn | undefined { + return this.manager.entitySelectorState.gdpPerCapitaColumn + } + + @computed private get isLoadingExternalSortColumn(): boolean { + return ( + this.manager.entitySelectorState.isLoadingExternalSortColumn ?? + false + ) + } + @computed private get table(): OwidTable { return this.manager.tableForSelection } - @computed private get sortColumns(): CoreColumn[] { - const activeSlugs = this.manager.activeColumnSlugs ?? [] + @computed private get hasCountryOrRegionEntities(): boolean { + return ( + intersection( + this.availableEntityNames, + regions.map((region) => region.name) + ).length > 0 + ) + } - const sortColumns = activeSlugs + @computed private get numericalChartColumns(): CoreColumn[] { + const activeSlugs = this.manager.activeColumnSlugs ?? [] + return activeSlugs .map((slug) => this.table.get(slug)) .filter( (column) => column.hasNumberFormatting && !column.isProjection ) + } + + @computed + private get numericalChartColumnsWithoutPopulationAndGdpPerCapita(): CoreColumn[] { + return this.numericalChartColumns.filter( + (column) => + column.slug !== this.populationSlug && + column.slug !== this.gdpPerCapitaSlug + ) + } + + @computed private get sortColumns(): CoreColumn[] { + return excludeUndefined([ + this.table.entityNameColumn, + this.populationColumn, + this.gdpPerCapitaColumn, + ...this.numericalChartColumnsWithoutPopulationAndGdpPerCapita, + ]) + } + + @computed private get populationColumnUsedInChart(): + | CoreColumn + | undefined { + const activePopulationColumn = this.numericalChartColumns.find( + (column) => + isPopulationVariableId(column.slug) || + !!makeColumnLabel(column).match(/\bpopulation\b/i) + ) + return activePopulationColumn + } + + /** + * Either the ID of a population indicator used in the chart, + * or the default population indicator ID (or undefined if we + * don't want to sort by population) + */ + @computed private get populationIndicatorId(): number | undefined { + if (this.populationColumnUsedInChart) + return +this.populationColumnUsedInChart.slug + + if (this.hasCountryOrRegionEntities) + return DEFAULT_POPULATION_INDICATOR_ID + + return undefined + } + + @computed private get populationSlug(): ColumnSlug | undefined { + return this.populationIndicatorId !== undefined + ? this.populationIndicatorId.toString() + : undefined + } + + @computed private get gdpPerCapitaColumnUsedInChart(): + | CoreColumn + | undefined { + const activeGdpPerCapitaColumn = this.numericalChartColumns.find( + (column) => !!makeColumnLabel(column).match(/\bgdp per capita\b/i) + ) + return activeGdpPerCapitaColumn + } + + /** + * Either the ID of a GDP per capita indicator used in the chart, + * or the default GDP per capita indicator ID (or undefined if we + * don't want to sort by GDP per capita) + */ + @computed private get gdpPerCapitaIndicatorId(): number | undefined { + if (this.gdpPerCapitaColumnUsedInChart) + return +this.gdpPerCapitaColumnUsedInChart.slug + + if (this.hasCountryOrRegionEntities) + return DEFAULT_GDP_PER_CAPITA_INDICATOR_ID + + return undefined + } - return [this.table.entityNameColumn, ...sortColumns] + @computed private get gdpPerCapitaSlug(): ColumnSlug | undefined { + return this.gdpPerCapitaIndicatorId !== undefined + ? this.gdpPerCapitaIndicatorId.toString() + : undefined } @computed private get sortColumnsBySlug(): Record { @@ -239,16 +359,40 @@ export class EntitySelector extends React.Component<{ @computed get sortOptions(): DropdownOption[] { const options: DropdownOption[] = [] - const getDropdownLabel = (column: CoreColumn): string => { - if (column.slug === this.table.entityNameSlug) return "Name" - return column.titlePublicOrDisplayName.title + // the first dropdown option is always the entity name + options.push({ + value: this.table.entityNameSlug, + label: "Name", + }) + + // add population to the dropdown if applicable + if (this.populationSlug) { + options.push({ + value: this.populationSlug, + label: this.populationColumnUsedInChart + ? makeColumnLabel(this.populationColumnUsedInChart) + : DEFAULT_POPULATION_LABEL, + }) + } + + // add GDP per capita to the dropdown if applicable + if (this.gdpPerCapitaSlug) { + options.push({ + value: this.gdpPerCapitaSlug, + label: this.gdpPerCapitaColumnUsedInChart + ? makeColumnLabel(this.gdpPerCapitaColumnUsedInChart) + : DEFAULT_GDP_PER_CAPITA_LABEL, + }) } + // add chart columns to the dropdown + const chartColumns = + this.numericalChartColumnsWithoutPopulationAndGdpPerCapita options.push( - ...this.sortColumns.map((column) => { + ...chartColumns.map((column) => { return { value: column.slug, - label: getDropdownLabel(column), + label: makeColumnLabel(column), } }) ) @@ -448,10 +592,86 @@ export class EntitySelector extends React.Component<{ } } - @action.bound onChangeSortSlug(selected: unknown): void { + @action.bound async loadPopulationColumn(): Promise { + if (this.populationColumn) return + + if (this.populationColumnUsedInChart) { + this.set({ populationColumn: this.populationColumnUsedInChart }) + return + } + + if (this.populationIndicatorId === undefined) return + + this.set({ isLoadingExternalSortColumn: true }) + + try { + const variable = await loadVariableDataAndMetadata( + this.populationIndicatorId, + this.manager.dataApiUrl + ) + + const variableTable = buildVariableTable(variable) + if (variableTable) { + this.set({ + populationColumn: variableTable.get(this.populationSlug), + }) + } + } catch { + console.error( + `Failed to load variable with id ${this.populationIndicatorId}` + ) + } + + this.set({ isLoadingExternalSortColumn: false }) + } + + @action.bound async loadGdpPerCapitaColumn(): Promise { + if (this.gdpPerCapitaColumn) return + + if (this.gdpPerCapitaColumnUsedInChart) { + this.set({ gdpPerCapitaColumn: this.gdpPerCapitaColumnUsedInChart }) + return + } + + if (this.gdpPerCapitaIndicatorId === undefined) return + + this.set({ isLoadingExternalSortColumn: true }) + + try { + const variable = await loadVariableDataAndMetadata( + this.gdpPerCapitaIndicatorId, + this.manager.dataApiUrl + ) + const variableTable = buildVariableTable(variable) + if (variableTable) { + this.set({ + gdpPerCapitaColumn: variableTable.get( + this.gdpPerCapitaSlug + ), + }) + } + } catch { + console.error( + `Failed to load variable with id ${this.gdpPerCapitaIndicatorId}` + ) + } + + this.set({ isLoadingExternalSortColumn: false }) + } + + @action.bound async onChangeSortSlug(selected: unknown): Promise { if (selected) { - const selectedOption = selected as DropdownOption - this.updateSortSlug(selectedOption.value) + const { value } = selected as DropdownOption + + // if population or GDP per capita is selected, load the column + // (this is a no-op if the column is already loaded) + if (value === this.populationSlug) { + await this.loadPopulationColumn() + } else if (value === this.gdpPerCapitaSlug) { + await this.loadGdpPerCapitaColumn() + } + + this.updateSortSlug(value) } } @@ -505,6 +725,7 @@ export class EntitySelector extends React.Component<{ options={this.sortOptions} onChange={this.onChangeSortSlug} value={this.sortValue} + isLoading={this.isLoadingExternalSortColumn} />