Preprocess data for faster page load

vacs-map-app/scripts/format-yields-data.js

@@ -5,9 +5,17 @@ const source_filename = 'public/data-raw/crop-yields-mean-models.csv';
 const yields_target_filename = 'public/data/crop-yields-mean-models.csv';
 const grid_target_filename = 'public/data/grid.csv';
 
-const file = fs.readFileSync(source_filename, 'utf-8')
+const crop_info_filename = 'public/data/crop-info-general.csv';
 
-const yields = d3.csvParse(file, (d, i) => {
+const yields_file = fs.readFileSync(source_filename, 'utf-8')
+const crop_info_file = fs.readFileSync(crop_info_filename, 'utf-8')
+
+const cropInfo = d3.csvParse(crop_info_file, d3.autoType);
+
+// ---------------------------------------
+// SPLIT SOURCE YIELDS FILE INTO TWO, ONE THAT STORES THE DATA AND ASSOCIATES
+// EACH GRID CELL WITH AND ID, AND ONE THAT STORES THE GRID IDS AND COORDS
+const yields = d3.csvParse(yields_file, (d, i) => {
   let obj = Object.fromEntries(
     Object.entries(d).slice(1).map(([k, v]) => {
       return [k, v && v !== '' ? +v : null]
@@ -17,7 +25,7 @@ const yields = d3.csvParse(file, (d, i) => {
   return obj;
 })
 
-const grid = d3.csvParse(file, (d, i) => {
+const grid = d3.csvParse(yields_file, (d, i) => {
   const regExp = /\(([^)]+)\)/g;
   const coords = [...d.geometry.matchAll(regExp)].flat()[1]
   return {
@@ -26,7 +34,97 @@ const grid = d3.csvParse(file, (d, i) => {
     Y: coords.split(' ')[1]
   }
 })
+// ---------------------------------------
+// PRECALCULATE YIELD RATIOS FOR EACH GRID CELL AND CONSOLIDATE THE INFORMATION STORED
+const yieldKeys = Object.keys(yields[0]).filter((k) => k.startsWith('yield'))
+
+// add yield ratio columns for all crops and scenarios
+const yieldsWithRatios = yields.map((d, i) => {
+  const rowWithYields = Object.fromEntries(Object.entries(d).filter(([k, v]) => v !== null))
+
+  yieldKeys.forEach((k) => {
+    const [_, crop, timeframe, scenario] = k.split('_')
+    if (timeframe === 'historical') return
+
+    // NB: abbreviating these could get more performance improvements
+    const historicalKey = ['yield', crop, 'historical'].join('_')
+
+    if (!yieldKeys.includes(historicalKey)) return
+
+    const yieldRatioKey = ['yieldratio', crop, timeframe, scenario].join('_')
+
+    let yieldRatioValue = null
+    if (
+      rowWithYields[k] !== null &&
+      rowWithYields[historicalKey] !== null &&
+      rowWithYields[historicalKey]
+    ) {
+      yieldRatioValue =
+        (rowWithYields[k] - rowWithYields[historicalKey]) / rowWithYields[historicalKey]
+    }
+
+    if (yieldRatioValue === null) return
+    rowWithYields[yieldRatioKey] = yieldRatioValue
+  })
+
+  return rowWithYields
+})
+
+// ---------------------------------------
+// PRECACLULATE THE INFO FOR CROP GROUP MAPS
+const cropGroups = Array.from(new Set(cropInfo.map((d) => d.crop_group)))
+const futureScenarios = ['future_ssp126', 'future_ssp370']
+
+const groupYieldRatioKeysHash = Object.fromEntries(
+  d3.cross(futureScenarios, cropGroups).map(([scenario, cropGroup]) => {
+    return [`${cropGroup}_${scenario}`,
+      cropInfo
+        .filter((c) => c.crop_group === cropGroup)
+        .map((c) => {
+          return ['yieldratio', c.id, scenario].join('_')
+        })
+    ];
+  })
+);
+
+const yieldsWithCropGroups = yieldsWithRatios.map((row, i) => {
+  d3.cross(futureScenarios, cropGroups)
+    .forEach(([scenario, cropGroup]) => {
+      const groupKey = [cropGroup, scenario].join('_')
+      const groupYieldRatioKeys = groupYieldRatioKeysHash[groupKey];
+      const rowHasYieldRatios = Object.keys(row)
+        .some((k) => groupYieldRatioKeys.includes(k))
+      if (!rowHasYieldRatios) {
+        return
+      }
+      const obj = {
+        maxCrop: 'none',
+        minCrop: 'none',
+        maxVal: null,
+        minVal: null
+      }
+
+      // TODO might be faster just to pull out the relevant entries and sort
+      // them?
+      groupYieldRatioKeys.forEach((k) => {
+        if (row[k] === null) return;
+        if (row[k] > obj.maxVal) {
+          obj.maxVal = row[k]
+          obj.maxCrop = k.split('_')[1]
+        }
+        if (row[k] < obj.minVal) {
+          obj.minVal = row[k]
+          obj.minCrop = k.split('_')[1]
+        }
+      })
+
+      Object.entries(obj).forEach(([k, v]) => {
+        row[groupKey+"_"+k] = v;
+      })
+    })
+  return { ...row };
+})
 
-fs.writeFileSync(yields_target_filename, d3.csvFormat(yields));
+fs.writeFileSync(yields_target_filename, d3.csvFormat(yieldsWithCropGroups));
 fs.writeFileSync(grid_target_filename, d3.csvFormat(grid));
 

vacs-map-app/src/components/CropGroupStackedBarChart.vue

@@ -43,21 +43,24 @@ const cropGroupCrops = computed(() => {
 })
 
 const cropGroupColumn = computed(() => {
-  if (!selectedCropInfo.value || !selectedModel.value) {
+  if (!selectedCropInfo.value || !selectedModel.value || !cropGroupMetric.value) {
     return null
   }
 
-  return [selectedCropInfo.value.crop_group, selectedModel.value].join('_')
+  return [
+    selectedCropInfo.value.crop_group,
+    selectedModel.value,
+    cropGroupMetric.value + 'Crop'
+  ].join('_')
 })
 
 const bars = computed(() => {
   if (!cropGroupColumn.value) return null
   const cropGroupCells = cropYieldsData.value.filter((d) => !!d[cropGroupColumn.value])
 
   const crops = cropGroupCrops.value.map((crop, i) => {
-    const cropCells = cropGroupCells.filter(
-      (d) => d[cropGroupColumn.value][cropGroupMetric.value + 'Crop'] === crop.id
-    )
+    const cropCells = cropGroupCells.filter((d) => d[cropGroupColumn.value] === crop.id)
+
     const gridShare = (cropCells.length / cropGroupCells.length) * 100
     return {
       id: crop.id,
@@ -67,9 +70,7 @@ const bars = computed(() => {
     }
   })
 
-  const noDataCells = cropGroupCells.filter(
-    (d) => d[cropGroupColumn.value][cropGroupMetric.value + 'Crop'] == 'none'
-  )
+  const noDataCells = cropGroupCells.filter((d) => d[cropGroupColumn.value] == 'none')
 
   const noDataGridShare = (noDataCells.length / cropGroupCells.length) * 100
 

vacs-map-app/src/components/GridOverlay.vue

@@ -147,7 +147,7 @@ const addHoverListeners = () => {
   map.value.on('mousemove', id.value, (event) => {
     if (!event?.features?.length) return
     const feature = event?.features[0]
-    if (feature?.id) {
+    if (feature.properties.id) {
       hoveredId.value = feature.properties.id
     }
   })
@@ -312,18 +312,13 @@ const getCircleColorByCrop = () => {
     .concat(
       cropGroupCrops.value
         .map((crop, i) => {
-          return [
-            ['==', ['get', cropGroupMetric.value + 'Crop', ['get', cropGroupColumn.value]], crop],
-            ordinalScheme.value[i]
-          ]
+          return [['==', ['get', cropGroupColumn.value], crop], ordinalScheme.value[i]]
         })
         .flat()
     )
     .concat([noDataFill.value])
 
   return ['case', ['!=', ['get', cropGroupColumn.value], null], cases, 'transparent']
-
-  return cases
 }
 
 const getCircleFillColor = () => {

vacs-map-app/src/components/MapContainerColorAcrossScenarios.vue

@@ -77,11 +77,11 @@ const selectedCropGroup = computed(() => {
 })
 
 const cropGroupColumn = computed(() => {
-  if (!selectedCropGroup.value || !selectedModel.value) {
+  if (!selectedCropGroup.value || !selectedModel.value || !cropGroupMetric.value) {
     return null
   }
 
-  return [selectedCropGroup.value, selectedModel.value].join('_')
+  return [selectedCropGroup.value, selectedModel.value, cropGroupMetric.value + 'Crop'].join('_')
 })
 
 const cropGroupCrops = computed(() => {

vacs-map-app/src/components/MapLegendCropGroups.vue

@@ -87,7 +87,7 @@ const openCropGroupModal = () => {
   display: flex;
   flex-direction: column;
   gap: 0.25rem;
-  width: 20rem;
+  width: 22rem;
   margin-top: auto;
 }
 

vacs-map-app/src/components/MapTooltip.vue

@@ -1,5 +1,5 @@
 <template>
-  <TooltipWrapper v-if="hoveredValue">
+  <TooltipWrapper v-if="hoveredValue || hoveredCropId">
     {{ sentence }}
   </TooltipWrapper>
 </template>
@@ -40,24 +40,18 @@ const sentence = computed(() => {
       hoveredValue.value
     )} at this location`
   } else {
-    const cropField = cropGroupMetric.value + 'Crop'
-    const valueField = cropGroupMetric.value + 'Val'
-
-    const hoveredCropId = hoveredValue.value[cropField]
-    const hoveredYieldRatio = hoveredValue.value[valueField]
-
     const descriptor = cropGroupMetric.value === 'max' ? 'increase' : 'decrease'
 
-    if (hoveredCropId === 'none') {
+    if (hoveredCropId.value === 'none') {
       return `At this location, no ${selectedCropInfo.value.crop_group} are projected to ${descriptor} in yield`
     }
 
-    const hoveredCropName = cropInfo.value.find((d) => d.id === hoveredCropId).label
+    const hoveredCropName = cropInfo.value.find((d) => d.id === hoveredCropId.value).label
 
     return `Of the ${
       selectedCropInfo.value.crop_group
     }, ${hoveredCropName} is projected to have the greatest yield ${descriptor} (${pFormat(
-      hoveredYieldRatio
+      hoveredValue.value
     )}) at this location, in ${modelDescriptor}`
   }
 })
@@ -66,13 +60,41 @@ const selectedCropInfo = computed(() => {
   return cropInfo.value?.find((d) => d.id === selectedCrop.value)
 })
 
+const hoveredCropId = computed(() => {
+  if (
+    !yieldData.value ||
+    !selectedCrop.value ||
+    !selectedModel.value ||
+    !hoveredId.value ||
+    !cropGroupMetric.value
+  )
+    return null
+
+  const cellObject = yieldData.value.find((d) => d.id === hoveredId.value)
+
+  if (!cellObject) return null
+  return cellObject[
+    [selectedCropInfo.value?.crop_group, selectedModel.value, cropGroupMetric.value + 'Crop'].join(
+      '_'
+    )
+  ]
+})
+
 const hoveredValue = computed(() => {
-  if (!yieldData.value || !selectedCrop.value || !selectedModel.value || !hoveredId.value)
+  if (
+    !yieldData.value ||
+    !selectedCrop.value ||
+    !selectedModel.value ||
+    !hoveredId.value ||
+    !cropGroupMetric.value
+  )
     return null
 
   const columnName = !showCropGroupMap.value
     ? ['yieldratio', selectedCrop.value, selectedModel.value].join('_')
-    : [selectedCropInfo.value?.crop_group, selectedModel.value].join('_')
+    : [selectedCropInfo.value?.crop_group, selectedModel.value, cropGroupMetric.value + 'Val'].join(
+        '_'
+      )
 
   const cellObject = yieldData.value.find((d) => d.id === hoveredId.value)