From 5977c2b9637cd49653ee31e1410cb540f92b4a8a Mon Sep 17 00:00:00 2001 From: sophiamersmann Date: Tue, 9 Apr 2024 11:47:53 +0000 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20(entity=20selector)=20sort=20by?= =?UTF-8?q?=20external=20indicators?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../grapher/src/core/Grapher.tsx | 28 +-- .../grapher/src/core/GrapherConstants.ts | 28 +-- .../grapher/src/core/LegacyToOwidTable.ts | 76 +++++++ .../grapher/src/core/loadVariable.ts | 44 ++++ .../src/entitySelector/EntitySelector.tsx | 206 +++++++++++++++++- packages/@ourworldindata/grapher/src/index.ts | 6 +- 6 files changed, 328 insertions(+), 60 deletions(-) create mode 100644 packages/@ourworldindata/grapher/src/core/loadVariable.ts diff --git a/packages/@ourworldindata/grapher/src/core/Grapher.tsx b/packages/@ourworldindata/grapher/src/core/Grapher.tsx index 672a1ec8b99..359669fa1e1 100644 --- a/packages/@ourworldindata/grapher/src/core/Grapher.tsx +++ b/packages/@ourworldindata/grapher/src/core/Grapher.tsx @@ -118,8 +118,7 @@ import { DEFAULT_GRAPHER_CONFIG_SCHEMA, DEFAULT_GRAPHER_WIDTH, DEFAULT_GRAPHER_HEIGHT, - getVariableDataRoute, - getVariableMetadataRoute, + isPopulationVariableId, DEFAULT_GRAPHER_FRAME_PADDING, DEFAULT_GRAPHER_ENTITY_TYPE, DEFAULT_GRAPHER_ENTITY_TYPE_PLURAL, @@ -129,6 +128,7 @@ import { GRAPHER_LOADED_EVENT_NAME, GRAPHER_DRAWER_ID, } from "../core/GrapherConstants" +import { loadVariableDataAndMetadata } from "./loadVariable" import Cookies from "js-cookie" import { ChartDimension, @@ -253,21 +253,9 @@ async function loadVariablesDataSite( variableIds: number[], dataApiUrl: string ): Promise { - const loadVariableDataPromises = variableIds.map(async (variableId) => { - const dataPromise = fetch(getVariableDataRoute(dataApiUrl, variableId)) - const metadataPromise = fetch( - getVariableMetadataRoute(dataApiUrl, variableId) - ) - const [dataResponse, metadataResponse] = await Promise.all([ - dataPromise, - metadataPromise, - ]) - if (!dataResponse.ok) throw new Error(dataResponse.statusText) - if (!metadataResponse.ok) throw new Error(metadataResponse.statusText) - const data = await dataResponse.json() - const metadata = await metadataResponse.json() - return { data, metadata } - }) + const loadVariableDataPromises = variableIds.map((variableId) => + loadVariableDataAndMetadata(variableId, dataApiUrl) + ) const variablesData: OwidVariableDataMetadataDimensions[] = await Promise.all(loadVariableDataPromises) const variablesDataMap = new Map( @@ -1591,12 +1579,6 @@ export class Grapher ) columnSlugs.push(colorColumnSlug) - const isPopulationVariableId = (id: string): boolean => - id === "525709" || // "Population (historical + projections), Gapminder, HYDE & UN" - id === "525711" || // "Population (historical estimates), Gapminder, HYDE & UN" - id === "597929" || // "Population (various sources, 2023.1)" - id === "597930" // "Population (various sources, 2023.1)" - if (xColumnSlug !== undefined) { // exclude population variable if it's used as the x dimension in a marimekko if (!isMarimekko || !isPopulationVariableId(xColumnSlug)) diff --git a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts index 70a8a713ab5..c06e4819d0d 100644 --- a/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts +++ b/packages/@ourworldindata/grapher/src/core/GrapherConstants.ts @@ -50,29 +50,11 @@ export const ThereWasAProblemLoadingThisChart = `There was a problem loading thi export const WorldEntityName = "World" -export const getVariableDataRoute = ( - dataApiUrl: string, - variableId: number -): string => { - if (dataApiUrl.includes("v1/indicators/")) { - // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.data.json - return `${dataApiUrl}${variableId}.data.json` - } else { - throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`) - } -} - -export const getVariableMetadataRoute = ( - dataApiUrl: string, - variableId: number -): string => { - if (dataApiUrl.includes("v1/indicators/")) { - // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.metadata.json - return `${dataApiUrl}${variableId}.metadata.json` - } else { - throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`) - } -} +export const isPopulationVariableId = (id: string): boolean => + id === "525709" || // "Population (historical + projections), Gapminder, HYDE & UN" + id === "525711" || // "Population (historical estimates), Gapminder, HYDE & UN" + id === "597929" || // "Population (various sources, 2023.1)" + id === "597930" // "Population (various sources, 2023.1)" export enum Patterns { noDataPattern = "noDataPattern", diff --git a/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts b/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts index 55afbe6d257..000e06bdff3 100644 --- a/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts +++ b/packages/@ourworldindata/grapher/src/core/LegacyToOwidTable.ts @@ -9,6 +9,7 @@ import { OwidColumnDef, LegacyGrapherInterface, OwidVariableDimensions, + OwidVariableDataMetadataDimensions, } from "@ourworldindata/types" import { OwidTable, @@ -738,3 +739,78 @@ const annotationsToMap = (annotations: string): Map => { }) return entityAnnotationsMap } + +/** + * Loads a single variable into an OwidTable. + */ +export function buildVariableTable( + variable: OwidVariableDataMetadataDimensions +): OwidTable { + const entityMeta = variable.metadata.dimensions.entities.values + const entityMetaById: OwidEntityKey = Object.fromEntries( + entityMeta.map((entity) => [entity.id.toString(), entity]) + ) + + // Base column defs, present in all OwidTables + const baseColumnDefs: Map = new Map() + StandardOwidColumnDefs.forEach((def) => { + baseColumnDefs.set(def.slug, def) + }) + + const columnDefs = new Map(baseColumnDefs) + + // Time column + const timeColumnDef = timeColumnDefFromOwidVariable(variable.metadata) + columnDefs.set(timeColumnDef.slug, timeColumnDef) + + // Value column + const valueColumnDef = columnDefFromOwidVariable(variable.metadata) + // Because database columns can contain mixed types, we want to avoid + // parsing for Grapher data until we fix that. + valueColumnDef.skipParsing = true + columnDefs.set(valueColumnDef.slug, valueColumnDef) + + // Column values + + const times = timeColumnValuesFromOwidVariable( + variable.metadata, + variable.data + ) + const entityIds = variable.data.entities ?? [] + const entityNames = entityIds.map( + // if entityMetaById[id] does not exist, then we don't have entity + // from variable metadata in MySQL. This can happen because we take + // data from S3 and metadata from MySQL. After we unify it, it should + // no longer be a problem + (id) => entityMetaById[id]?.name ?? id.toString() + ) + // see comment above about entityMetaById[id] + const entityCodes = entityIds.map((id) => entityMetaById[id]?.code) + + // If there is a conversionFactor, apply it. + let values = variable.data.values || [] + const conversionFactor = valueColumnDef.display?.conversionFactor + if (conversionFactor !== undefined) { + values = values.map((value) => + isNumber(value) ? value * conversionFactor : value + ) + + // If a non-int conversion factor is applied to an integer column, + // we end up with a numeric column. + if ( + valueColumnDef.type === ColumnTypeNames.Integer && + !isInteger(conversionFactor) + ) + valueColumnDef.type = ColumnTypeNames.Numeric + } + + const columnStore: { [key: string]: any[] } = { + [OwidTableSlugs.entityId]: entityIds, + [OwidTableSlugs.entityCode]: entityCodes, + [OwidTableSlugs.entityName]: entityNames, + [timeColumnDef.slug]: times, + [valueColumnDef.slug]: values, + } + + return new OwidTable(columnStore, Array.from(columnDefs.values())) +} diff --git a/packages/@ourworldindata/grapher/src/core/loadVariable.ts b/packages/@ourworldindata/grapher/src/core/loadVariable.ts new file mode 100644 index 00000000000..9ea7bfae79b --- /dev/null +++ b/packages/@ourworldindata/grapher/src/core/loadVariable.ts @@ -0,0 +1,44 @@ +import { OwidVariableDataMetadataDimensions } from "@ourworldindata/types" + +export const getVariableDataRoute = ( + dataApiUrl: string, + variableId: number +): string => { + if (dataApiUrl.includes("v1/indicators/")) { + // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.data.json + return `${dataApiUrl}${variableId}.data.json` + } else { + throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`) + } +} + +export const getVariableMetadataRoute = ( + dataApiUrl: string, + variableId: number +): string => { + if (dataApiUrl.includes("v1/indicators/")) { + // fetching from Data API, e.g. https://api.ourworldindata.org/v1/indicators/123.metadata.json + return `${dataApiUrl}${variableId}.metadata.json` + } else { + throw new Error(`dataApiUrl format not supported: ${dataApiUrl}`) + } +} + +export async function loadVariableDataAndMetadata( + variableId: number, + dataApiUrl: string +): Promise { + const dataPromise = fetch(getVariableDataRoute(dataApiUrl, variableId)) + const metadataPromise = fetch( + getVariableMetadataRoute(dataApiUrl, variableId) + ) + const [dataResponse, metadataResponse] = await Promise.all([ + dataPromise, + metadataPromise, + ]) + if (!dataResponse.ok) throw new Error(dataResponse.statusText) + if (!metadataResponse.ok) throw new Error(metadataResponse.statusText) + const data = await dataResponse.json() + const metadata = await metadataResponse.json() + return { data, metadata } +} diff --git a/packages/@ourworldindata/grapher/src/entitySelector/EntitySelector.tsx b/packages/@ourworldindata/grapher/src/entitySelector/EntitySelector.tsx index bf40453269f..4f6e0c21664 100644 --- a/packages/@ourworldindata/grapher/src/entitySelector/EntitySelector.tsx +++ b/packages/@ourworldindata/grapher/src/entitySelector/EntitySelector.tsx @@ -18,6 +18,7 @@ import { regions, sortBy, Tippy, + excludeUndefined, } from "@ourworldindata/utils" import { Checkbox } from "@ourworldindata/components" import { FuzzySearch } from "../controls/FuzzySearch" @@ -35,6 +36,7 @@ import { DEFAULT_GRAPHER_ENTITY_TYPE, DEFAULT_GRAPHER_ENTITY_TYPE_PLURAL, GRAPHER_SCROLLABLE_CONTAINER_CLASS, + isPopulationVariableId, } from "../core/GrapherConstants" import { CoreColumn, @@ -44,12 +46,19 @@ import { import { SortIcon } from "../controls/SortIcon" import { Dropdown } from "../controls/Dropdown" import { scaleLinear, type ScaleLinear } from "d3-scale" +import { buildVariableTable } from "../core/LegacyToOwidTable" +import { loadVariableDataAndMetadata } from "../core/loadVariable" export interface EntitySelectorState { searchInput: string sortConfig: SortConfig localEntityNames?: string[] mostRecentlySelectedEntityName?: string + isLoadingExternalSortColumn?: boolean + + // external columns to sort by + populationColumn?: CoreColumn + gdpPerCapitaColumn?: CoreColumn } export interface EntitySelectorManager { @@ -61,6 +70,7 @@ export interface EntitySelectorManager { entityType?: string entityTypePlural?: string activeColumnSlugs?: string[] + dataApiUrl: string } type Slug = string @@ -85,6 +95,12 @@ interface DropdownOption { label: string } +const DEFAULT_POPULATION_INDICATOR_ID = 597930 +const DEFAULT_POPULATION_LABEL = "Population" + +const DEFAULT_GDP_PER_CAPITA_INDICATOR_ID = 735665 +const DEFAULT_GDP_PER_CAPITA_LABEL = "GDP per capita" + @observer export class EntitySelector extends React.Component<{ manager: EntitySelectorManager @@ -220,18 +236,94 @@ export class EntitySelector extends React.Component<{ return this.manager.entitySelectorState.localEntityNames } + @computed private get populationColumn(): CoreColumn | undefined { + return this.manager.entitySelectorState.populationColumn + } + + @computed private get gdpPerCapitaColumn(): CoreColumn | undefined { + return this.manager.entitySelectorState.gdpPerCapitaColumn + } + + @computed private get isLoadingExternalSortColumn(): boolean { + return ( + this.manager.entitySelectorState.isLoadingExternalSortColumn ?? + false + ) + } + @computed private get table(): OwidTable { return this.manager.tableForSelection } - @computed private get sortColumns(): CoreColumn[] { + @computed private get numericalChartColumns(): CoreColumn[] { const activeSlugs = this.manager.activeColumnSlugs ?? [] - - const sortColumns = activeSlugs + const activeColumns = activeSlugs .map((slug) => this.table.get(slug)) .filter((column) => isColumnWithNumberFormatting(column)) + return activeColumns + } - return [this.table.entityNameColumn, ...sortColumns] + @computed private get sortColumns(): CoreColumn[] { + return excludeUndefined([ + this.table.entityNameColumn, + this.populationColumn, + this.gdpPerCapitaColumn, + ...this.numericalChartColumns, + ]) + } + + @computed private get populationColumnUsedInChart(): + | CoreColumn + | undefined { + const activePopulationColumn = this.numericalChartColumns.find( + (column) => + isPopulationVariableId(column.slug) || + !!makeColumnLabel(column).match(/\bpopulation\b/i) + ) + return activePopulationColumn + } + + @computed private get populationIndicatorId(): number | undefined { + const { entitiesAreCountryLike } = this.manager + + if (this.populationColumnUsedInChart) + return +this.populationColumnUsedInChart.slug + + if (entitiesAreCountryLike) return DEFAULT_POPULATION_INDICATOR_ID + + return undefined + } + + @computed private get populationSlug(): Slug | undefined { + return this.populationIndicatorId !== undefined + ? this.populationIndicatorId.toString() + : undefined + } + + @computed private get gdpPerCapitaColumnUsedInChart(): + | CoreColumn + | undefined { + const activePopulationColumn = this.numericalChartColumns.find( + (column) => !!makeColumnLabel(column).match(/\bgdp\b/i) + ) + return activePopulationColumn + } + + @computed private get gdpPerCapitaIndicatorId(): number | undefined { + const { entitiesAreCountryLike } = this.manager + + if (this.gdpPerCapitaColumnUsedInChart) + return +this.gdpPerCapitaColumnUsedInChart.slug + + if (entitiesAreCountryLike) return DEFAULT_GDP_PER_CAPITA_INDICATOR_ID + + return undefined + } + + @computed private get gdpPerCapitaSlug(): Slug | undefined { + return this.gdpPerCapitaIndicatorId !== undefined + ? this.gdpPerCapitaIndicatorId.toString() + : undefined } @computed private get sortColumnsBySlug(): Record { @@ -241,16 +333,42 @@ export class EntitySelector extends React.Component<{ @computed get sortOptions(): DropdownOption[] { const options: DropdownOption[] = [] - const getDropdownLabel = (column: CoreColumn): string => { - if (column.slug === this.table.entityNameSlug) return "Name" - return column.displayName || column.nonEmptyName + const [nameColumn, ...numericalColumns] = this.sortColumns + + const chartColumns = numericalColumns.filter( + (column) => + column.slug !== this.populationSlug && + column.slug !== this.gdpPerCapitaSlug + ) + + options.push({ + value: nameColumn.slug, + label: "Name", + }) + + if (this.populationSlug) { + options.push({ + value: this.populationSlug, + label: this.populationColumnUsedInChart + ? makeColumnLabel(this.populationColumnUsedInChart) + : DEFAULT_POPULATION_LABEL, + }) + } + + if (this.gdpPerCapitaSlug) { + options.push({ + value: this.gdpPerCapitaSlug, + label: this.gdpPerCapitaColumnUsedInChart + ? makeColumnLabel(this.gdpPerCapitaColumnUsedInChart) + : DEFAULT_GDP_PER_CAPITA_LABEL, + }) } options.push( - ...this.sortColumns.map((column) => { + ...chartColumns.map((column) => { return { value: column.slug, - label: getDropdownLabel(column), + label: makeColumnLabel(column), } }) ) @@ -447,10 +565,69 @@ export class EntitySelector extends React.Component<{ ) } - @action.bound onChangeSortSlug(selected: unknown): void { + @action.bound async loadPopulationColumn(): Promise { + if (this.populationColumn) return + + if (this.populationColumnUsedInChart) { + this.set({ populationColumn: this.populationColumnUsedInChart }) + return + } + + if (this.populationIndicatorId === undefined) return + + this.set({ isLoadingExternalSortColumn: true }) + + const variable = await loadVariableDataAndMetadata( + this.populationIndicatorId, + this.manager.dataApiUrl + ) + const variableTable = buildVariableTable(variable) + if (variableTable) { + this.set({ + populationColumn: variableTable.get(this.populationSlug), + }) + } + + this.set({ isLoadingExternalSortColumn: false }) + } + + @action.bound async loadGdpPerCapitaColumn(): Promise { + if (this.gdpPerCapitaColumn) return + + if (this.gdpPerCapitaColumnUsedInChart) { + this.set({ gdpPerCapitaColumn: this.gdpPerCapitaColumnUsedInChart }) + return + } + + if (this.gdpPerCapitaIndicatorId === undefined) return + + this.set({ isLoadingExternalSortColumn: true }) + + const variable = await loadVariableDataAndMetadata( + this.gdpPerCapitaIndicatorId, + this.manager.dataApiUrl + ) + const variableTable = buildVariableTable(variable) + if (variableTable) { + this.set({ + gdpPerCapitaColumn: variableTable.get(this.gdpPerCapitaSlug), + }) + } + + this.set({ isLoadingExternalSortColumn: false }) + } + + @action.bound async onChangeSortSlug(selected: unknown): Promise { if (selected) { - const selectedOption = selected as DropdownOption - this.updateSortSlug(selectedOption.value) + const { value } = selected as DropdownOption + + if (value === this.populationSlug) { + await this.loadPopulationColumn() + } else if (value === this.gdpPerCapitaSlug) { + await this.loadGdpPerCapitaColumn() + } + + this.updateSortSlug(value) } } @@ -506,6 +683,7 @@ export class EntitySelector extends React.Component<{ options={this.sortOptions} onChange={this.onChangeSortSlug} value={this.sortValue} + isLoading={this.isLoadingExternalSortColumn} />