✨ Add filter information to readme and metadata (#4229)

When using the download API, add information about active filters to the metadata and readme.

README.md

@@ -38,7 +38,7 @@ The Grapher project is built with [Lerna](https://github.com/lerna/lerna/) and [
 
 A Grapher-based tool that creates more complex [data visualization user interfaces](https://ourworldindata.org/explorers/migration).
 
-Each explorer can be configured via a [panel](explorerAdminServer/) in the admin client. Their config files are stored in [a separate repository](https://github.com/owid/owid-content/tree/master/explorers).
+Each explorer can be configured via a [panel](explorerAdminServer/) in the admin client. Their config files are stored in [a separate repository](https://github.com/owid/owid-content/tree/master/explorers) but will probably be moved to the DB soon.
 
 ### Grapher Admin
 

functions/_common/downloadFunctions.ts

@@ -125,14 +125,17 @@ export async function fetchReadmeForGrapher(
 
     await grapher.downloadLegacyDataFromOwidVariableIds()
 
-    const readme = assembleReadme(grapher)
+    const readme = assembleReadme(grapher, searchParams)
     return new Response(readme, {
         headers: {
             "Content-Type": "text/markdown; charset=utf-8",
         },
     })
 }
-function assembleReadme(grapher: Grapher): string {
+function assembleReadme(
+    grapher: Grapher,
+    searchParams: URLSearchParams
+): string {
     const metadataCols = getColumnsForMetadata(grapher)
-    return constructReadme(grapher, metadataCols)
+    return constructReadme(grapher, metadataCols, searchParams)
 }

functions/_common/metadataTools.ts

@@ -11,6 +11,7 @@ import {
     getAttributionFragmentsFromVariable,
     getCitationLong,
 } from "@ourworldindata/utils"
+import { getGrapherFilters } from "./urlTools.js"
 
 type MetadataColumn = {
     titleShort: string
@@ -189,6 +190,9 @@ export function assembleMetadata(
         columns: Object.fromEntries(columns),
         // date downloaded should be YYYY-MM-DD
         dateDownloaded: dateDownloaded.toISOString().split("T")[0],
+        // NOTE: this is filtered by whitelisted grapher query params - if you want other params to be
+        //       inlucded here (e.g. MDIM selection), add them to the whitelist inside getGrapherFilters
+        activeFilters: getGrapherFilters(searchParams),
     }
 
     return fullMetadata

functions/_common/readmeTools.ts

@@ -17,6 +17,7 @@ import {
 } from "@ourworldindata/utils"
 import { CoreColumn } from "@ourworldindata/core-table"
 import { Grapher } from "@ourworldindata/grapher"
+import { getGrapherFilters } from "./urlTools.js"
 
 const markdownNewlineEnding = "  "
 
@@ -229,9 +230,28 @@ function* columnReadmeText(col: CoreColumn) {
     yield ""
 }
 
+function* activeFilterSettings(searchParams: URLSearchParams) {
+    // NOTE: this is filtered by whitelisted grapher query params - if you want other params to be
+    //       inlucded here (e.g. MDIM selection), add them to the whitelist inside getGrapherFilters
+    const filterSettings = getGrapherFilters(searchParams)
+    if (filterSettings) {
+        yield ""
+        yield `### Active Filters`
+        yield ""
+        yield `A filtered subset of the full data was downloaded. The following filters were applied:`
+        for (const [key, val] of Object.entries(filterSettings)) {
+            if (key === "country")
+                yield `${key}: ${val.replaceAll("~", ", ")}` // country filter is separated with tilde
+            else yield `${key}: ${val}`
+        }
+        yield ""
+    }
+}
+
 export function constructReadme(
     grapher: Grapher,
-    columns: CoreColumn[]
+    columns: CoreColumn[],
+    searchParams: URLSearchParams
 ): string {
     const isSingleColumn = columns.length === 1
     // Some computed columns have neither a source nor origins - filter these away
@@ -241,12 +261,14 @@ export function constructReadme(
         )
         .flatMap((col) => [...columnReadmeText(col)])
     let readme: string
+    const urlWithFilters = `${grapher.canonicalUrl}`
+
     const downloadDate = formatDate(new Date()) // formats the date as "October 10, 2024"
     if (isSingleColumn)
         readme = `# ${grapher.title} - Data package
 
-This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}) on the Our World in Data website. It was downloaded on ${downloadDate}.
-
+This data package contains the data that powers the chart ["${grapher.title}"](${urlWithFilters}) on the Our World in Data website. It was downloaded on ${downloadDate}.
+${[...activeFilterSettings(searchParams)].join("\n")}
 ## CSV Structure
 
 The high level structure of the CSV file is that each row is an observation for an entity (usually a country or region) and a timepoint (usually a year).
@@ -273,7 +295,7 @@ ${sources.join("\n")}
     else
         readme = `# ${grapher.title} - Data package
 
-This data package contains the data that powers the chart ["${grapher.title}"](${grapher.canonicalUrl}) on the Our World in Data website.
+This data package contains the data that powers the chart ["${grapher.title}"](${urlWithFilters}) on the Our World in Data website.
 
 ## CSV Structure
 

functions/_common/urlTools.ts

@@ -0,0 +1,16 @@
+import { pick } from "@ourworldindata/utils"
+import { GRAPHER_QUERY_PARAM_KEYS } from "@ourworldindata/types"
+
+export function getGrapherFilters(
+    searchParams: URLSearchParams
+): Record<string, string> | undefined {
+    const params = searchParams.size
+        ? Object.fromEntries(searchParams)
+        : undefined
+    if (!params) return undefined
+    // delete url query params that the download api uses but that are not related to grapher
+    delete params.v1
+    delete params.csvType
+    delete params.useColumnShortNames
+    return pick(params, GRAPHER_QUERY_PARAM_KEYS)
+}

functions/package.json

@@ -1,15 +1,16 @@
 {
-  "name": "owid-functions",
-  "type": "module",
-  "dependencies": {
-    "@ourworldindata/grapher": "workspace:^",
-    "@ourworldindata/utils": "workspace:^",
-    "itty-router": "^5.0.17",
-    "littlezipper": "^0.1.4",
-    "stripe": "^14.20.0",
-    "svg2png-wasm": "^1.4.1"
-  },
-  "devDependencies": {
-    "@cloudflare/workers-types": "^4.20240919.0"
-  }
+    "name": "owid-functions",
+    "type": "module",
+    "dependencies": {
+        "@ourworldindata/grapher": "workspace:^",
+        "@ourworldindata/types": "workspace:^",
+        "@ourworldindata/utils": "workspace:^",
+        "itty-router": "^5.0.17",
+        "littlezipper": "^0.1.4",
+        "stripe": "^14.20.0",
+        "svg2png-wasm": "^1.4.1"
+    },
+    "devDependencies": {
+        "@cloudflare/workers-types": "^4.20240919.0"
+    }
 }

packages/@ourworldindata/grapher/src/modal/DownloadModal.tsx

@@ -790,8 +790,10 @@ export const DownloadModalDataTab = (props: DownloadModalProps) => {
                 shortColNames: false,
             }
             if (serverSideDownloadAvailable) {
+                const fullOrFiltered =
+                    csvDownloadType === CsvDownloadType.Full ? "" : ".filtered"
                 triggerDownloadFromUrl(
-                    ctx.slug + ".zip",
+                    ctx.slug + fullOrFiltered + ".zip",
                     getDownloadUrl("zip", ctx)
                 )
             } else {

yarn.lock

@@ -15437,6 +15437,7 @@ __metadata:
   dependencies:
     "@cloudflare/workers-types": "npm:^4.20240919.0"
     "@ourworldindata/grapher": "workspace:^"
+    "@ourworldindata/types": "workspace:^"
     "@ourworldindata/utils": "workspace:^"
     itty-router: "npm:^5.0.17"
     littlezipper: "npm:^0.1.4"