Skip to content

Commit

Permalink
🚧 wip
Browse files Browse the repository at this point in the history
  • Loading branch information
@sophiamersmann
sophiamersmann committed Jul 17, 2024
1 parent da250c0 commit 27d2a32
Show file tree
Hide file tree
Showing 9 changed files with 528 additions and 205 deletions.
459 changes: 346 additions & 113 deletions adminSiteServer/apiRouter.ts
View file

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions baker/GrapherBaker.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -150,8 +150,8 @@ export async function renderDataPageV2(
knex: db.KnexReadWriteTransaction
) {
const grapherConfigForVariable = await getMergedGrapherConfigForVariable(
variableId,
knex
knex,
variableId
)
// Only merge the grapher config on the indicator if the caller tells us to do so -
// this is true for preview pages for datapages on the indicator level but false
Expand Down
37 changes: 22 additions & 15 deletions baker/siteRenderers.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import {
OwidGdoc,
OwidGdocDataInsightInterface,
DbRawPost,
mergeGrapherConfigs,

Check warning on line 51 in baker/siteRenderers.tsx

View workflow job for this annotation

GitHub Actions / eslint 

'mergeGrapherConfigs' is defined but never used. Allowed unused vars must match /^_/u

Check warning on line 51 in baker/siteRenderers.tsx

View workflow job for this annotation

GitHub Actions / eslint 

'mergeGrapherConfigs' is defined but never used. Allowed unused vars must match /^_/u
} from "@ourworldindata/utils"
import { extractFormattingOptions } from "../serverUtils/wordpressUtils.js"
import {
Expand Down Expand Up @@ -751,7 +752,16 @@ export const renderExplorerPage = async (
if (requiredVariableIds.length) {
partialGrapherConfigRows = await knexRaw(
knex,
`SELECT id, grapherConfigETL, grapherConfigAdmin FROM variables WHERE id IN (?)`,
`-- sql
SELECT
v.id,
cc_etl.full AS grapherConfigETL,
cc_admin.full AS grapherConfigAdmin
FROM variables v
JOIN chart_configs cc_admin ON cc.id=v.grapherConfigIdAdmin
JOIN chart_configs cc_etl ON cc.id=v.grapherConfigIdETL
WHERE v.id IN (?)
`,
[requiredVariableIds]
)

Expand Down Expand Up @@ -779,20 +789,17 @@ export const renderExplorerPage = async (
const partialGrapherConfigs = partialGrapherConfigRows
.filter((row) => row.grapherConfigAdmin || row.grapherConfigETL)
.map((row) => {
const adminConfig = row.grapherConfigAdmin
? parseGrapherConfigFromRow({
id: row.id,
config: row.grapherConfigAdmin as string,
})
: {}
const etlConfig = row.grapherConfigETL
? parseGrapherConfigFromRow({
id: row.id,
config: row.grapherConfigETL as string,
})
: {}
// TODO(inheritance): use mergeGrapherConfigs instead
return mergePartialGrapherConfigs(etlConfig, adminConfig)
if (row.grapherConfigAdmin) {
return parseGrapherConfigFromRow({
id: row.id,
config: row.grapherConfigAdmin as string,
})
} else {
return parseGrapherConfigFromRow({
id: row.id,
config: row.grapherConfigETL as string,
})
}
})

const wpContent = transformedProgram.wpBlockId
Expand Down
10 changes: 10 additions & 0 deletions db/db.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -668,3 +668,13 @@ export async function getLinkedIndicatorSlugs({
.then((gdocs) => gdocs.flatMap((gdoc) => gdoc.linkedKeyIndicatorSlugs))
.then((slugs) => new Set(slugs))
}

export const getBinaryUUID = async (
knex: KnexReadonlyTransaction
): Promise<Buffer> => {
const { id } = (await knexRawFirst<{ id: Buffer }>(
knex,
`SELECT UUID_TO_BIN(UUID(), 1) AS id`
))!
return id
}
17 changes: 7 additions & 10 deletions db/migration/1721134584504-MoveIndicatorChartsToTheChartsConfigTable.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -71,22 +71,14 @@ export class MoveIndicatorChartsToTheChartsConfigTable1721134584504
config.$schema = defaultGrapherConfig.$schema
}

// ETL configs inherit from the default config
const patchConfig = diffGrapherConfigs(config, defaultGrapherConfig)
const fullConfig = mergeGrapherConfigs(defaultGrapherConfig, config)

// insert config into the chart_configs table
const configId = await getBinaryUUID(queryRunner)
await queryRunner.query(
`-- sql
INSERT INTO chart_configs (id, patch, full)
VALUES (?, ?, ?)
`,
[
configId,
JSON.stringify(patchConfig),
JSON.stringify(fullConfig),
]
[configId, JSON.stringify(config), JSON.stringify(config)]
)

// update reference in the variables table
Expand All @@ -108,7 +100,7 @@ export class MoveIndicatorChartsToTheChartsConfigTable1721134584504
`)

await queryRunner.query(`-- sql
CREATE VIEW inheritance_indicators_x_charts AS (
CREATE VIEW inheriting_charts AS (
WITH y_dimensions AS (
SELECT
*
Expand Down Expand Up @@ -145,6 +137,11 @@ export class MoveIndicatorChartsToTheChartsConfigTable1721134584504
}

public async down(queryRunner: QueryRunner): Promise<void> {
// drop view
await queryRunner.query(`-- sql
DROP VIEW inheriting_charts
`)

// add back the `grapherConfigAdmin` and `grapherConfigETL` columns
await queryRunner.query(`-- sql
ALTER TABLE variables
Expand Down
159 changes: 138 additions & 21 deletions db/model/Variable.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,13 @@
import _ from "lodash"
import { Writable } from "stream"
import * as db from "../db.js"
import { retryPromise, isEmpty } from "@ourworldindata/utils"
import {
retryPromise,
isEmpty,
excludeUndefined,

Check warning on line 7 in db/model/Variable.ts

View workflow job for this annotation

GitHub Actions / eslint 

'excludeUndefined' is defined but never used. Allowed unused vars must match /^_/u

Check warning on line 7 in db/model/Variable.ts

View workflow job for this annotation

GitHub Actions / eslint 

'excludeUndefined' is defined but never used. Allowed unused vars must match /^_/u
omitUndefinedValues,
mergeGrapherConfigs,
} from "@ourworldindata/utils"
import {
getVariableDataRoute,
getVariableMetadataRoute,
Expand All @@ -20,33 +26,144 @@ import {
GrapherInterface,
DbRawVariable,
VariablesTableName,
DbRawChartConfig,
parseChartConfig,
DbEnrichedChartConfig,
} from "@ourworldindata/types"
import { knexRaw } from "../db.js"
import { knexRaw, knexRawFirst } from "../db.js"

export async function getGrapherConfigsForVariable(
knex: db.KnexReadonlyTransaction,
variableId: number
): Promise<
| (Pick<
DbRawVariable,
"id" | "grapherConfigIdAdmin" | "grapherConfigIdETL"
> & {
grapherConfigAdmin?: DbEnrichedChartConfig["patch"]
grapherConfigETL?: DbEnrichedChartConfig["patch"]
})
| undefined
> {
const variable = await knexRawFirst<
Pick<
DbRawVariable,
"id" | "grapherConfigIdAdmin" | "grapherConfigIdETL"
> & {
grapherConfigAdmin?: DbRawChartConfig["patch"]
grapherConfigETL?: DbRawChartConfig["patch"]
}
>(
knex,
`-- sql
SELECT
v.id,
v.grapherConfigIdAdmin,
v.grapherConfigIdETL,
cc_admin.patch AS grapherConfigAdmin,
cc_etl.patch AS grapherConfigETL
FROM variables v
LEFT JOIN chart_configs cc_admin ON v.grapherConfigIdAdmin = cc_admin.id
LEFT JOIN chart_configs cc_etl ON v.grapherConfigIdETL = cc_etl.id
WHERE v.id = ?
`,
[variableId]
)

if (!variable) return

//export type Field = keyof VariableRow
return omitUndefinedValues({
...variable,
grapherConfigAdmin: variable.grapherConfigAdmin
? parseChartConfig(variable.grapherConfigAdmin)
: undefined,
grapherConfigETL: variable.grapherConfigETL
? parseChartConfig(variable.grapherConfigETL)
: undefined,
})
}

export async function getMergedGrapherConfigForVariable(
variableId: number,
knex: db.KnexReadonlyTransaction
knex: db.KnexReadonlyTransaction,
variableId: number
): Promise<GrapherInterface | undefined> {
const rows: Pick<
DbRawVariable,
"grapherConfigAdmin" | "grapherConfigETL"
>[] = await knexRaw(
const variable = await getGrapherConfigsForVariable(knex, variableId)
if (!variable) return
if (!variable.grapherConfigETL && !variable.grapherConfigAdmin) return
return mergeGrapherConfigs(
variable.grapherConfigETL ?? {},
variable.grapherConfigAdmin ?? {}
)
}

export async function insertNewGrapherConfigForVariable(
knex: db.KnexReadonlyTransaction,
params: {
type: "admin" | "etl"
variableId: number
config: GrapherInterface
}
): Promise<void> {
const { type, variableId, config } = params

// insert chart config into the database
const configId = await db.getBinaryUUID(knex)
await db.knexRaw(
knex,
`SELECT grapherConfigAdmin, grapherConfigETL FROM variables WHERE id = ?`,
[variableId]
`-- sql
INSERT INTO chart_configs (id, patch, full)
VALUES (?, ?, ?)
`,
[configId, JSON.stringify(config), JSON.stringify(config)]
)

// make a reference to the config from the variables table
const column =
type === "admin" ? "grapherConfigIdAdmin" : "grapherConfigIdETL"
await db.knexRaw(
knex,
`-- sql
UPDATE variables
SET ?? = ?
WHERE id = ?
`,
[column, configId, variableId]
)
}

export async function updateExistingGrapherConfigForVariable(
knex: db.KnexReadonlyTransaction,
params: {
type: "admin" | "etl"
variableId: number
config: GrapherInterface
configId: Buffer
}
): Promise<void> {
const { variableId, config, configId, type } = params

Check warning on line 143 in db/model/Variable.ts

View workflow job for this annotation

GitHub Actions / eslint 

'variableId' is assigned a value but never used. Allowed unused vars must match /^_/u

Check warning on line 143 in db/model/Variable.ts

View workflow job for this annotation

GitHub Actions / eslint 

'type' is assigned a value but never used. Allowed unused vars must match /^_/u

Check warning on line 143 in db/model/Variable.ts

View workflow job for this annotation

GitHub Actions / eslint 

'variableId' is assigned a value but never used. Allowed unused vars must match /^_/u

Check warning on line 143 in db/model/Variable.ts

View workflow job for this annotation

GitHub Actions / eslint 

'type' is assigned a value but never used. Allowed unused vars must match /^_/u

// let configId: Buffer | undefined
// if (!params.configId) {
// const variable = await getGrapherConfigsForVariable(knex, variableId)
// if (!variable) return
// configId =
// type === "admin"
// ? variable.grapherConfigIdAdmin ?? undefined
// : variable.grapherConfigIdETL ?? undefined
// }
// if (!configId) return

await db.knexRaw(
knex,
`-- sql
UPDATE chart_configs
SET
patch = ?,
full = ?
WHERE id = ?
`,
[JSON.stringify(config), JSON.stringify(config), configId]
)
if (!rows.length) return
const row = rows[0]
const grapherConfigAdmin = row.grapherConfigAdmin
? JSON.parse(row.grapherConfigAdmin)
: undefined
const grapherConfigETL = row.grapherConfigETL
? JSON.parse(row.grapherConfigETL)
: undefined
// TODO(inheritance): use mergeGrapherConfigs instead
return _.merge({}, grapherConfigAdmin, grapherConfigETL)
}

// TODO: these are domain functions and should live somewhere else
Expand Down
1 change: 1 addition & 0 deletions packages/@ourworldindata/grapher/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ export {
grapherInterfaceWithHiddenTabsOnly,
CONTINENTS_INDICATOR_ID,
POPULATION_INDICATOR_ID_USED_IN_ADMIN,
DEFAULT_GRAPHER_CONFIG_SCHEMA,
} from "./core/GrapherConstants"
export {
getVariableDataRoute,
Expand Down
42 changes: 2 additions & 40 deletions packages/@ourworldindata/types/src/dbTypes/Variables.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,8 +20,8 @@ export interface DbInsertVariable {
descriptionShort?: string | null
dimensions?: JsonString | null
display: JsonString
grapherConfigAdmin?: JsonString | null
grapherConfigETL?: JsonString | null
grapherConfigIdAdmin?: Buffer | null
grapherConfigIdETL?: Buffer | null
id?: number
license?: JsonString | null
licenses?: JsonString | null
Expand Down Expand Up @@ -66,8 +66,6 @@ export type DbEnrichedVariable = Omit<
| "dimensions"
| "descriptionKey"
| "originalMetadata"
| "grapherConfigAdmin"
| "grapherConfigETL"
| "processingLog"
| "sort"
> & {
Expand All @@ -77,8 +75,6 @@ export type DbEnrichedVariable = Omit<
dimensions: VariableDisplayDimension | null
descriptionKey: string[] | null
originalMetadata: unknown | null
grapherConfigAdmin: GrapherInterface | null
grapherConfigETL: GrapherInterface | null
processingLog: unknown | null
sort: string[] | null
}
Expand Down Expand Up @@ -149,30 +145,6 @@ export function serializeVariableOriginalMetadata(
return originalMetadata ? JSON.stringify(originalMetadata) : null
}

export function parseVariableGrapherConfigAdmin(
grapherConfigAdmin: JsonString | null
): GrapherInterface {
return grapherConfigAdmin ? JSON.parse(grapherConfigAdmin) : null
}

export function serializeVariableGrapherConfigAdmin(
grapherConfigAdmin: GrapherInterface | null
): JsonString | null {
return grapherConfigAdmin ? JSON.stringify(grapherConfigAdmin) : null
}

export function parseVariableGrapherConfigETL(
grapherConfigETL: JsonString | null
): GrapherInterface {
return grapherConfigETL ? JSON.parse(grapherConfigETL) : null
}

export function serializeVariableGrapherConfigETL(
grapherConfigETL: GrapherInterface | null
): JsonString | null {
return grapherConfigETL ? JSON.stringify(grapherConfigETL) : null
}

export function parseVariableProcessingLog(
processingLog: JsonString | null
): any {
Expand Down Expand Up @@ -204,10 +176,6 @@ export function parseVariablesRow(row: DbRawVariable): DbEnrichedVariable {
dimensions: parseVariableDimensions(row.dimensions),
descriptionKey: parseVariableDescriptionKey(row.descriptionKey),
originalMetadata: parseVariableOriginalMetadata(row.originalMetadata),
grapherConfigAdmin: parseVariableGrapherConfigAdmin(
row.grapherConfigAdmin
),
grapherConfigETL: parseVariableGrapherConfigETL(row.grapherConfigETL),
processingLog: parseVariableProcessingLog(row.processingLog),
sort: parseVariableSort(row.sort),
}
Expand All @@ -224,12 +192,6 @@ export function serializeVariablesRow(row: DbEnrichedVariable): DbRawVariable {
originalMetadata: serializeVariableOriginalMetadata(
row.originalMetadata
),
grapherConfigAdmin: serializeVariableGrapherConfigAdmin(
row.grapherConfigAdmin
),
grapherConfigETL: serializeVariableGrapherConfigETL(
row.grapherConfigETL
),
processingLog: serializeVariableProcessingLog(row.processingLog),
sort: serializeVariableSort(row.sort),
}
Expand Down
Loading

0 comments on commit 27d2a32

Please sign in to comment.