From ce70b0316b35b8dec1481dee9d0996313c25b34c Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Tue, 26 Nov 2024 13:28:53 +0100 Subject: [PATCH 1/9] =?UTF-8?q?=E2=9C=A8=20Add=20filter=20information=20to?= =?UTF-8?q?=20readme=20and=20metadata?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functions/_common/downloadFunctions.ts | 9 ++++++--- functions/_common/metadataTools.ts | 2 ++ functions/_common/readmeTools.ts | 28 +++++++++++++++++++++++--- functions/_common/urlTools.ts | 12 +++++++++++ 4 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 functions/_common/urlTools.ts diff --git a/functions/_common/downloadFunctions.ts b/functions/_common/downloadFunctions.ts index aa9f260434a..9b193550c99 100644 --- a/functions/_common/downloadFunctions.ts +++ b/functions/_common/downloadFunctions.ts @@ -125,14 +125,17 @@ export async function fetchReadmeForGrapher( await grapher.downloadLegacyDataFromOwidVariableIds() - const readme = assembleReadme(grapher) + const readme = assembleReadme(grapher, searchParams) return new Response(readme, { headers: { "Content-Type": "text/markdown; charset=utf-8", }, }) } -function assembleReadme(grapher: Grapher): string { +function assembleReadme( + grapher: Grapher, + searchParams: URLSearchParams +): string { const metadataCols = getColumnsForMetadata(grapher) - return constructReadme(grapher, metadataCols) + return constructReadme(grapher, metadataCols, searchParams) } diff --git a/functions/_common/metadataTools.ts b/functions/_common/metadataTools.ts index d5af2b69714..339b03daece 100644 --- a/functions/_common/metadataTools.ts +++ b/functions/_common/metadataTools.ts @@ -11,6 +11,7 @@ import { getAttributionFragmentsFromVariable, getCitationLong, } from "@ourworldindata/utils" +import { getGrapherFilters } from "./urlTools.js" type MetadataColumn = { titleShort: string @@ -189,6 +190,7 @@ export function assembleMetadata( columns: Object.fromEntries(columns), // date downloaded should be YYYY-MM-DD dateDownloaded: dateDownloaded.toISOString().split("T")[0], + activeFilters: getGrapherFilters(searchParams), } return fullMetadata diff --git a/functions/_common/readmeTools.ts b/functions/_common/readmeTools.ts index 81ba60a047a..d57c87f24e1 100644 --- a/functions/_common/readmeTools.ts +++ b/functions/_common/readmeTools.ts @@ -17,6 +17,7 @@ import { } from "@ourworldindata/utils" import { CoreColumn } from "@ourworldindata/core-table" import { Grapher } from "@ourworldindata/grapher" +import { getGrapherFilters } from "./urlTools.js" const markdownNewlineEnding = " " @@ -229,9 +230,28 @@ function* columnReadmeText(col: CoreColumn) { yield "" } +function* activeFilterSettings(searchParams: URLSearchParams) { + const filterSettings = getGrapherFilters(searchParams) + if (filterSettings) { + yield "" + yield `### Active Filters` + yield "" + yield `A filtered subset of the full data was downloaded. The following filters were applied:` + for (const entry of Object.entries(filterSettings)) { + const key = entry[0] + const val = entry[1] as string + if (key === "country") + yield `${key}: ${val.replace("~", ", ")}` // country filter is separated with tilde + else yield `${key}: ${val}` + } + yield "" + } +} + export function constructReadme( grapher: Grapher, - columns: CoreColumn[] + columns: CoreColumn[], + searchParams: URLSearchParams ): string { const isSingleColumn = columns.length === 1 // Some computed columns have neither a source nor origins - filter these away @@ -241,12 +261,14 @@ export function constructReadme( ) .flatMap((col) => [...columnReadmeText(col)]) let readme: string + const queryString = searchParams.size ? "?" + searchParams.toString() : "" + const downloadDate = formatDate(new Date()) // formats the date as "October 10, 2024" if (isSingleColumn) readme = `# ${grapher.title} - Data package -This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}) on the Our World in Data website. It was downloaded on ${downloadDate}. - +This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}${queryString}) on the Our World in Data website. It was downloaded on ${downloadDate}. +${[...activeFilterSettings(searchParams)].join("\n")} ## CSV Structure The high level structure of the CSV file is that each row is an observation for an entity (usually a country or region) and a timepoint (usually a year). diff --git a/functions/_common/urlTools.ts b/functions/_common/urlTools.ts new file mode 100644 index 00000000000..ba01d7d2306 --- /dev/null +++ b/functions/_common/urlTools.ts @@ -0,0 +1,12 @@ +export function getGrapherFilters( + searchParams: URLSearchParams +): Record { + const params = searchParams.size + ? Object.fromEntries(searchParams) + : undefined + // delete url query params that the download api uses but that are not related to grapher + delete params.v1 + delete params.csvType + delete params.useColumnShortNames + return params +} From 07d19c44325c24199fdfdb6d80e35d7674d49c5c Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Thu, 28 Nov 2024 10:57:57 +0100 Subject: [PATCH 2/9] =?UTF-8?q?=F0=9F=94=A8=20unify=20readme=20url=20handl?= =?UTF-8?q?ing?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functions/_common/readmeTools.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/functions/_common/readmeTools.ts b/functions/_common/readmeTools.ts index d57c87f24e1..e3804728793 100644 --- a/functions/_common/readmeTools.ts +++ b/functions/_common/readmeTools.ts @@ -262,12 +262,13 @@ export function constructReadme( .flatMap((col) => [...columnReadmeText(col)]) let readme: string const queryString = searchParams.size ? "?" + searchParams.toString() : "" + const urlWithFilters = `${grapher.canonicalUrl}${queryString}` const downloadDate = formatDate(new Date()) // formats the date as "October 10, 2024" if (isSingleColumn) readme = `# ${grapher.title} - Data package -This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}${queryString}) on the Our World in Data website. It was downloaded on ${downloadDate}. +This data package contains the data that powers the chart ["${grapher.title}"](${urlWithFilters}) on the Our World in Data website. It was downloaded on ${downloadDate}. ${[...activeFilterSettings(searchParams)].join("\n")} ## CSV Structure @@ -295,7 +296,7 @@ ${sources.join("\n")} else readme = `# ${grapher.title} - Data package -This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}) on the Our World in Data website. +This data package contains the data that powers the chart ["${grapher.title}"](${urlWithFilters}) on the Our World in Data website. ## CSV Structure From 70b5b4246ec2e6925a4f592b5ca20dc980d6fe5a Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Fri, 6 Dec 2024 20:52:02 +0100 Subject: [PATCH 3/9] =?UTF-8?q?=F0=9F=94=A8=20incorporate=20PR=20feedback?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functions/_common/metadataTools.ts | 2 ++ functions/_common/readmeTools.ts | 9 ++++----- functions/_common/urlTools.ts | 8 ++++++-- functions/package.json | 27 ++++++++++++++------------- yarn.lock | 1 + 5 files changed, 27 insertions(+), 20 deletions(-) diff --git a/functions/_common/metadataTools.ts b/functions/_common/metadataTools.ts index 339b03daece..83cf0bb02ce 100644 --- a/functions/_common/metadataTools.ts +++ b/functions/_common/metadataTools.ts @@ -190,6 +190,8 @@ export function assembleMetadata( columns: Object.fromEntries(columns), // date downloaded should be YYYY-MM-DD dateDownloaded: dateDownloaded.toISOString().split("T")[0], + // NOTE: this is filtered by whitelisted grapher query params - if you want other params to be + // inlucded here (e.g. MDIM selection), add them to the whitelist inside getGrapherFilters activeFilters: getGrapherFilters(searchParams), } diff --git a/functions/_common/readmeTools.ts b/functions/_common/readmeTools.ts index e3804728793..f2b3cbf2528 100644 --- a/functions/_common/readmeTools.ts +++ b/functions/_common/readmeTools.ts @@ -231,15 +231,15 @@ function* columnReadmeText(col: CoreColumn) { } function* activeFilterSettings(searchParams: URLSearchParams) { + // NOTE: this is filtered by whitelisted grapher query params - if you want other params to be + // inlucded here (e.g. MDIM selection), add them to the whitelist inside getGrapherFilters const filterSettings = getGrapherFilters(searchParams) if (filterSettings) { yield "" yield `### Active Filters` yield "" yield `A filtered subset of the full data was downloaded. The following filters were applied:` - for (const entry of Object.entries(filterSettings)) { - const key = entry[0] - const val = entry[1] as string + for (const [key, val] of Object.entries(filterSettings)) { if (key === "country") yield `${key}: ${val.replace("~", ", ")}` // country filter is separated with tilde else yield `${key}: ${val}` @@ -261,8 +261,7 @@ export function constructReadme( ) .flatMap((col) => [...columnReadmeText(col)]) let readme: string - const queryString = searchParams.size ? "?" + searchParams.toString() : "" - const urlWithFilters = `${grapher.canonicalUrl}${queryString}` + const urlWithFilters = `${grapher.canonicalUrl}` const downloadDate = formatDate(new Date()) // formats the date as "October 10, 2024" if (isSingleColumn) diff --git a/functions/_common/urlTools.ts b/functions/_common/urlTools.ts index ba01d7d2306..0fc6419ea2b 100644 --- a/functions/_common/urlTools.ts +++ b/functions/_common/urlTools.ts @@ -1,12 +1,16 @@ +import { pick } from "@ourworldindata/utils" +import { GRAPHER_QUERY_PARAM_KEYS } from "@ourworldindata/types" + export function getGrapherFilters( searchParams: URLSearchParams -): Record { +): Record | undefined { const params = searchParams.size ? Object.fromEntries(searchParams) : undefined + if (!params) return undefined // delete url query params that the download api uses but that are not related to grapher delete params.v1 delete params.csvType delete params.useColumnShortNames - return params + return pick(params, GRAPHER_QUERY_PARAM_KEYS) } diff --git a/functions/package.json b/functions/package.json index a2748e3ab34..3f5a5881914 100644 --- a/functions/package.json +++ b/functions/package.json @@ -1,15 +1,16 @@ { - "name": "owid-functions", - "type": "module", - "dependencies": { - "@ourworldindata/grapher": "workspace:^", - "@ourworldindata/utils": "workspace:^", - "itty-router": "^5.0.17", - "littlezipper": "^0.1.4", - "stripe": "^14.20.0", - "svg2png-wasm": "^1.4.1" - }, - "devDependencies": { - "@cloudflare/workers-types": "^4.20240919.0" - } + "name": "owid-functions", + "type": "module", + "dependencies": { + "@ourworldindata/grapher": "workspace:^", + "@ourworldindata/types": "workspace:^", + "@ourworldindata/utils": "workspace:^", + "itty-router": "^5.0.17", + "littlezipper": "^0.1.4", + "stripe": "^14.20.0", + "svg2png-wasm": "^1.4.1" + }, + "devDependencies": { + "@cloudflare/workers-types": "^4.20240919.0" + } } diff --git a/yarn.lock b/yarn.lock index f1c336ae199..ee58d9662fd 100644 --- a/yarn.lock +++ b/yarn.lock @@ -15419,6 +15419,7 @@ __metadata: dependencies: "@cloudflare/workers-types": "npm:^4.20240919.0" "@ourworldindata/grapher": "workspace:^" + "@ourworldindata/types": "workspace:^" "@ourworldindata/utils": "workspace:^" itty-router: "npm:^5.0.17" littlezipper: "npm:^0.1.4" From 93f7ae78d038c90a2bd1de9119221f3d6b4efe40 Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Thu, 12 Dec 2024 19:27:49 +0100 Subject: [PATCH 4/9] =?UTF-8?q?=F0=9F=90=9B=20fix=20minor=20bug?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functions/_common/readmeTools.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/functions/_common/readmeTools.ts b/functions/_common/readmeTools.ts index f2b3cbf2528..e2620b21d1a 100644 --- a/functions/_common/readmeTools.ts +++ b/functions/_common/readmeTools.ts @@ -241,7 +241,7 @@ function* activeFilterSettings(searchParams: URLSearchParams) { yield `A filtered subset of the full data was downloaded. The following filters were applied:` for (const [key, val] of Object.entries(filterSettings)) { if (key === "country") - yield `${key}: ${val.replace("~", ", ")}` // country filter is separated with tilde + yield `${key}: ${val.replaceAll("~", ", ")}` // country filter is separated with tilde else yield `${key}: ${val}` } yield "" From 214b9ad566bb926377c120152789659247098813 Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Thu, 12 Dec 2024 21:01:20 +0100 Subject: [PATCH 5/9] =?UTF-8?q?=F0=9F=94=A8=20change=20filtered=20csv=20fi?= =?UTF-8?q?le=20download=20name?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx b/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx index b5a45c73b77..fbcf42ead3d 100644 --- a/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx +++ b/packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx @@ -783,8 +783,10 @@ export const DownloadModalDataTab = (props: DownloadModalProps) => { shortColNames: false, } if (serverSideDownloadAvailable) { + const fullOrFiltered = + csvDownloadType === CsvDownloadType.Full ? "" : ".filtered" triggerDownloadFromUrl( - ctx.slug + ".zip", + ctx.slug + fullOrFiltered + ".zip", getDownloadUrl("zip", ctx) ) } else { From cd3f891eb5f01ef087080ebe57f3f15f4fea5b3f Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Fri, 13 Dec 2024 15:28:32 +0100 Subject: [PATCH 6/9] =?UTF-8?q?=F0=9F=90=9D=20dummy=20change=20to=20trigge?= =?UTF-8?q?r=20new=20staging=20server.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 21fa8b604cf..0c5ef29c04c 100644 --- a/README.md +++ b/README.md @@ -38,13 +38,13 @@ The Grapher project is built with [Lerna](https://github.com/lerna/lerna/) and [ A Grapher-based tool that creates more complex [data visualization user interfaces](https://ourworldindata.org/explorers/migration). -Each explorer can be configured via a [panel](explorerAdminServer/) in the admin client. Their config files are stored in [a separate repository](https://github.com/owid/owid-content/tree/master/explorers). +Each explorer can be configured via a [panel](explorerAdminServer/) in the admin client. Their config files are stored in [a separate repository](https://github.com/owid/owid-content/tree/master/explorers) but will probably be moved to the DB soon. ### Grapher Admin -- A [client-side](adminSiteClient/) project that provides a user interface for configuring graphers, explorers, and managing and uploading data. +- A [client-side](adminSiteClient/) project that provides a user interface for configuring graphers, explorers, and managing and uploading data. -- A [server-side](adminSiteServer/) project that manages the MySQL database used by graphers. +- A [server-side](adminSiteServer/) project that manages the MySQL database used by graphers. ### [Baker](baker/) @@ -76,10 +76,10 @@ The following is an excerpt explaining the origin of this repo and what the alte > > Using our own system has very important advantages: > -> - **Integration with our global development database**: Our database of global development metrics is integrated into our visualization tool so that when we add and update empirical data the visualizations are all updated. (In contrast to this, a pre-existing tool would make the exploration of a database impossible and would require the preparation of each dataset separately for each visualisation.) -> - **Flexibility**: We can use automation to change our entire system all at once. For example, if we decide we want to use a different source referencing style, we could easily update this across hundreds of charts. This makes it possible to scale our publication and to sustainably improve our work without starting from scratch at each round. -> - **Risk mitigation**: We hope(!) that Our World in Data is a long-term project and we want the visualizations we produce to continue to be useful and available years from now. An external web service may be shut down or change for reasons we cannot control. We have had this experience in the past and learned our lesson from it. -> - **Keeping everything up-to-date**: Because we want to be a useful resource for some time we make sure that we have a technology in place that allows us to keep all of our work up-to-date without starting from scratch each time. We have our global development database directly integrated in the Grapher and as soon as new data becomes available (for example from a UN agency) we can run a script that pulls in that data and updates all the visualizations that present that data. +> - **Integration with our global development database**: Our database of global development metrics is integrated into our visualization tool so that when we add and update empirical data the visualizations are all updated. (In contrast to this, a pre-existing tool would make the exploration of a database impossible and would require the preparation of each dataset separately for each visualisation.) +> - **Flexibility**: We can use automation to change our entire system all at once. For example, if we decide we want to use a different source referencing style, we could easily update this across hundreds of charts. This makes it possible to scale our publication and to sustainably improve our work without starting from scratch at each round. +> - **Risk mitigation**: We hope(!) that Our World in Data is a long-term project and we want the visualizations we produce to continue to be useful and available years from now. An external web service may be shut down or change for reasons we cannot control. We have had this experience in the past and learned our lesson from it. +> - **Keeping everything up-to-date**: Because we want to be a useful resource for some time we make sure that we have a technology in place that allows us to keep all of our work up-to-date without starting from scratch each time. We have our global development database directly integrated in the Grapher and as soon as new data becomes available (for example from a UN agency) we can run a script that pulls in that data and updates all the visualizations that present that data. --- From 71660d50fdb2ddf2016118f3454cfd4cdd2685cf Mon Sep 17 00:00:00 2001 From: danyx23 Date: Fri, 13 Dec 2024 14:30:47 +0000 Subject: [PATCH 7/9] =?UTF-8?q?=F0=9F=A4=96=20style:=20prettify=20code?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- README.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0c5ef29c04c..7069a570867 100644 --- a/README.md +++ b/README.md @@ -42,9 +42,9 @@ Each explorer can be configured via a [panel](explorerAdminServer/) in the admin ### Grapher Admin -- A [client-side](adminSiteClient/) project that provides a user interface for configuring graphers, explorers, and managing and uploading data. +- A [client-side](adminSiteClient/) project that provides a user interface for configuring graphers, explorers, and managing and uploading data. -- A [server-side](adminSiteServer/) project that manages the MySQL database used by graphers. +- A [server-side](adminSiteServer/) project that manages the MySQL database used by graphers. ### [Baker](baker/) @@ -76,10 +76,10 @@ The following is an excerpt explaining the origin of this repo and what the alte > > Using our own system has very important advantages: > -> - **Integration with our global development database**: Our database of global development metrics is integrated into our visualization tool so that when we add and update empirical data the visualizations are all updated. (In contrast to this, a pre-existing tool would make the exploration of a database impossible and would require the preparation of each dataset separately for each visualisation.) -> - **Flexibility**: We can use automation to change our entire system all at once. For example, if we decide we want to use a different source referencing style, we could easily update this across hundreds of charts. This makes it possible to scale our publication and to sustainably improve our work without starting from scratch at each round. -> - **Risk mitigation**: We hope(!) that Our World in Data is a long-term project and we want the visualizations we produce to continue to be useful and available years from now. An external web service may be shut down or change for reasons we cannot control. We have had this experience in the past and learned our lesson from it. -> - **Keeping everything up-to-date**: Because we want to be a useful resource for some time we make sure that we have a technology in place that allows us to keep all of our work up-to-date without starting from scratch each time. We have our global development database directly integrated in the Grapher and as soon as new data becomes available (for example from a UN agency) we can run a script that pulls in that data and updates all the visualizations that present that data. +> - **Integration with our global development database**: Our database of global development metrics is integrated into our visualization tool so that when we add and update empirical data the visualizations are all updated. (In contrast to this, a pre-existing tool would make the exploration of a database impossible and would require the preparation of each dataset separately for each visualisation.) +> - **Flexibility**: We can use automation to change our entire system all at once. For example, if we decide we want to use a different source referencing style, we could easily update this across hundreds of charts. This makes it possible to scale our publication and to sustainably improve our work without starting from scratch at each round. +> - **Risk mitigation**: We hope(!) that Our World in Data is a long-term project and we want the visualizations we produce to continue to be useful and available years from now. An external web service may be shut down or change for reasons we cannot control. We have had this experience in the past and learned our lesson from it. +> - **Keeping everything up-to-date**: Because we want to be a useful resource for some time we make sure that we have a technology in place that allows us to keep all of our work up-to-date without starting from scratch each time. We have our global development database directly integrated in the Grapher and as soon as new data becomes available (for example from a UN agency) we can run a script that pulls in that data and updates all the visualizations that present that data. --- From 020e861a1404e622bdb5000eda9e63f2a251308c Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Mon, 16 Dec 2024 14:29:37 +0100 Subject: [PATCH 8/9] =?UTF-8?q?=F0=9F=90=9D=20dummy=20commit=20for=20new?= =?UTF-8?q?=20staging=20server?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functions/_common/urlTools.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/functions/_common/urlTools.ts b/functions/_common/urlTools.ts index 0fc6419ea2b..113c5c83dc9 100644 --- a/functions/_common/urlTools.ts +++ b/functions/_common/urlTools.ts @@ -8,7 +8,8 @@ export function getGrapherFilters( ? Object.fromEntries(searchParams) : undefined if (!params) return undefined - // delete url query params that the download api uses but that are not related to grapher + // delete url query params that the download api uses but that are not related to grapher. + // Might want to store these in a separate object in the future delete params.v1 delete params.csvType delete params.useColumnShortNames From 45e79fb4c67a10f37e18e68a0f507cf51361613f Mon Sep 17 00:00:00 2001 From: Daniel Bachler Date: Mon, 16 Dec 2024 14:41:25 +0100 Subject: [PATCH 9/9] =?UTF-8?q?=F0=9F=90=9D=20dummy=20commit?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- functions/_common/urlTools.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/functions/_common/urlTools.ts b/functions/_common/urlTools.ts index 113c5c83dc9..50aed41572f 100644 --- a/functions/_common/urlTools.ts +++ b/functions/_common/urlTools.ts @@ -9,7 +9,6 @@ export function getGrapherFilters( : undefined if (!params) return undefined // delete url query params that the download api uses but that are not related to grapher. - // Might want to store these in a separate object in the future delete params.v1 delete params.csvType delete params.useColumnShortNames