From 37d0131e9a40b736ceb3b41371c189ef829581ce Mon Sep 17 00:00:00 2001 From: Gabriel Fosse Date: Fri, 29 Mar 2024 12:08:43 -0700 Subject: [PATCH 1/3] update datetime for 'time beginning' measurements --- src/adapters/ust-ist.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/adapters/ust-ist.js b/src/adapters/ust-ist.js index 938f5b24..82fb1f73 100644 --- a/src/adapters/ust-ist.js +++ b/src/adapters/ust-ist.js @@ -106,7 +106,7 @@ function parseParams(params) { .filter(key => !isNaN(parseInt(key))) // Filter out keys that are not indices .map(index => { const measurement = params[p][index]; - const date = DateTime.fromFormat(measurement.endtime.trimEnd(), 'yyyy-LL-dd HH:mm:ss', { zone: 'Atlantic/Reykjavik' }); + const date = DateTime.fromFormat(measurement.endtime.trimEnd(), 'yyyy-LL-dd HH:mm:ss', { zone: 'Atlantic/Reykjavik' }).plus({ hours: 1 }); const resolution = params[p].resolution === '1h' ? { value: 1, unit: 'hours' } From 5787ecdb7a61366ec894a03f864b6260fe5d55d3 Mon Sep 17 00:00:00 2001 From: Gabriel Fosse Date: Wed, 3 Apr 2024 12:22:13 -0700 Subject: [PATCH 2/3] Update ust-ist adapter to use /latest data endpoint --- src/adapters/ust-ist.js | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/src/adapters/ust-ist.js b/src/adapters/ust-ist.js index 82fb1f73..a3f3be96 100644 --- a/src/adapters/ust-ist.js +++ b/src/adapters/ust-ist.js @@ -12,12 +12,9 @@ import log from '../lib/logger.js'; export const name = 'ust-ist'; -// the dataUrl only works for the current date - 1 day. // a list of endpoints can be found at https://api.ust.is/aq/a -// the old endpoint https://api.ust.is/aq/a/getLatest is not returning data anymore -const yesterday = DateTime.utc().minus({ days: 1 }).toISODate(); // format "YYYY-MM-DD" const stationsUrl = 'https://api.ust.is/aq/a/getStations'; -const dataUrl = `https://api.ust.is/aq/a/getDate/date/${yesterday}`; +const dataUrl = `https://api.ust.is/aq/a/getLatest` /** * Fetches air quality data for Iceland from a specific date and compiles it into a structured format. @@ -44,7 +41,7 @@ export async function fetchData(source, cb) { // Skip processing this station if metadata is missing if (!stationMeta) { - console.warn(`Metadata missing for station ID: ${stationId}. Skipping...`); + log.warn(`Metadata missing for station ID: ${stationId}. Skipping...`); return acc; } @@ -66,6 +63,7 @@ export async function fetchData(source, cb) { return acc.concat(latestMeasurements.map(m => ({ ...baseMeta, ...m }))); }, []); log.debug(measurements[0]); + console.log(measurements, {depth: null}); cb(null, { name: 'unused', measurements }); } catch (e) { cb(e); @@ -124,6 +122,6 @@ function parseParams(params) { }; }); - return measurements; + return measurements; }); } From 69276284697af32dec6e4305b8004df0b958c51c Mon Sep 17 00:00:00 2001 From: Christian Parker Date: Wed, 3 Apr 2024 14:17:19 -0700 Subject: [PATCH 3/3] Added a note about the time ending/beginning issue --- src/adapters/ust-ist.js | 57 +++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 25 deletions(-) diff --git a/src/adapters/ust-ist.js b/src/adapters/ust-ist.js index a3f3be96..7b8f891b 100644 --- a/src/adapters/ust-ist.js +++ b/src/adapters/ust-ist.js @@ -14,7 +14,7 @@ export const name = 'ust-ist'; // a list of endpoints can be found at https://api.ust.is/aq/a const stationsUrl = 'https://api.ust.is/aq/a/getStations'; -const dataUrl = `https://api.ust.is/aq/a/getLatest` +const dataUrl = `https://api.ust.is/aq/a/getLatest`; /** * Fetches air quality data for Iceland from a specific date and compiles it into a structured format. @@ -62,8 +62,7 @@ export async function fetchData(source, cb) { return acc.concat(latestMeasurements.map(m => ({ ...baseMeta, ...m }))); }, []); - log.debug(measurements[0]); - console.log(measurements, {depth: null}); + console.debug("Example measurements", measurements.slice(-5)); cb(null, { name: 'unused', measurements }); } catch (e) { cb(e); @@ -100,28 +99,36 @@ function parseParams(params) { const validParams = Object.keys(params).filter(p => acceptableParameters.includes(p.toLowerCase().replace('.', ''))); return validParams.flatMap(p => { - const measurements = Object.keys(params[p]) - .filter(key => !isNaN(parseInt(key))) // Filter out keys that are not indices - .map(index => { - const measurement = params[p][index]; - const date = DateTime.fromFormat(measurement.endtime.trimEnd(), 'yyyy-LL-dd HH:mm:ss', { zone: 'Atlantic/Reykjavik' }).plus({ hours: 1 }); - - const resolution = params[p].resolution === '1h' - ? { value: 1, unit: 'hours' } - : {}; - - return { - date: { - utc: date.toUTC().toISO({ suppressMilliseconds: true }), - local: date.toISO({ suppressMilliseconds: true }) - }, - parameter: p.toLowerCase().replace('.', ''), - value: parseFloat(measurement.value), - unit: params[p].unit, - averagingPeriod: resolution - }; - }); + const measurements = Object.keys(params[p]) + .filter(key => !isNaN(parseInt(key))) // Filter out keys that are not indices + .map(index => { + const measurement = params[p][index]; + // arguement for assuming time beginning even though the variable name is 'endtime' + // ---------------------------------------------------------------------------------- + // When we look at their endpoint for a full days worth of data e.g. https://api.ust.is/aq/a/getDate/date/2024-04-02 + // We see 24 hours with the first hour having an endtime of 00:00:00 and the last hour being 23:00:00 + // and so we assume that the data here is time beginning (i.e. the data for 23:00:00 is the average from 11pm to midnight) + // And then when we look at the latest data we see essentially the same thing + // where the two endpoints overlap the data for the overlapping hours match + // and we dont see data for the previous hour, always for 2 hours back, suggesting the previous hour is marked time beginning + const date = DateTime.fromFormat(measurement.endtime.trimEnd(), 'yyyy-LL-dd HH:mm:ss', { zone: 'Atlantic/Reykjavik' }).plus({ hours: 1 }); + + const resolution = params[p].resolution === '1h' + ? { value: 1, unit: 'hours' } + : {}; + + return { + date: { + utc: date.toUTC().toISO({ suppressMilliseconds: true }), + local: date.toISO({ suppressMilliseconds: true }) + }, + parameter: p.toLowerCase().replace('.', ''), + value: parseFloat(measurement.value), + unit: params[p].unit, + averagingPeriod: resolution + }; + }); return measurements; }); - } +}