v6

.github/workflows/test.yml

@@ -13,7 +13,7 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        node-version: ['16', '18']
+        node-version: ['20']
         redis-version: ['6']
 
     steps:

index.js

@@ -94,19 +94,14 @@ const createCachedHafasClient = (hafas, storage, opt = {}) => {
 
 		// rather static data
 		reachableFrom: (_, opt = {}) => dynamicCachePeriod(5, 12, 60, opt.when),
-		locations: HOUR,
-		stop: HOUR,
-		nearby: HOUR,
+		locations: () => HOUR,
+		stop: () => HOUR,
+		nearby: () => HOUR,
 
 		...(opt.cachePeriods || {}),
 	}
 	for (const [key, val] of Object.entries(cachePeriods)) {
-		// todo [breaking]: always expect a function
 		if ('function' === typeof val) continue
-		if ('number' === typeof val) {
-			cachePeriods[key] = () => val
-			continue
-		}
 		throw new TypeError(`opt.cachePeriods.${key} must be a number or a function returning a number`)
 	}
 

package.json

@@ -23,7 +23,7 @@
 	"bugs": "https://github.com/public-transport/cached-hafas-client/issues",
 	"license": "ISC",
 	"engines": {
-		"node": ">=16"
+		"node": ">=20"
 	},
 	"dependencies": {
 		"common-prefix": "^1.1.0",
@@ -46,7 +46,7 @@
 		"vbb-hafas": "^8.0.0"
 	},
 	"cached-hafas-client": {
-		"dataVersion": 4
+		"dataVersion": 5
 	},
 	"peerDependencies": {
 		"hafas-client": ">=6.0 <=6.0",

readme.md

@@ -78,7 +78,7 @@ const MINUTE = 60 * SECOND
 
 const cachePeriods = {
 	// cache all cachedHafas.stop(…) calls for 10m
-	stop: 10 * MINUTE,
+	stop: () => 10 * MINUTE,
 	// cache cachedHafas.trip(tripId, opt) based on sqrt(opt.when - now)
 	trip: (_, opt = {}) => {
 		const diffSecs = (new Date(opt.when) - Date.now()) / SECOND
@@ -133,15 +133,15 @@ createCachedHafas(hafas, storage, opt = {})
 ```js
 {
 	cachePeriods: {
-		departures: 30_1000, arrivals: 30_1000, // 30s
-		journeys: 30_1000, // 30s
-		refreshJourney: 60_1000, // 1m
-		trip: 30_1000, // 30s
-		radar: 10_1000, // 10s
-		locations: 3_600_1000, // 1h
-		stop: 3_600_1000, // 1h
-		nearby: 3_600_1000, // 1h
-		reachableFrom: 30_1000,
+		departures: () => 30_1000, arrivals: () => 30_1000, // 30s
+		journeys: () => 30_1000, // 30s
+		refreshJourney: () => 60_1000, // 1m
+		trip: () => 30_1000, // 30s
+		radar: () => 10_1000, // 10s
+		locations: () => 3_600_1000, // 1h
+		stop: () => 3_600_1000, // 1h
+		nearby: () => 3_600_1000, // 1h
+		reachableFrom: () => 30_1000,
 	},
 }
 ```

stores/redis.js

@@ -38,7 +38,8 @@ local function read_collection (id)
 		cursor = res[1];
 
 		for _, key in ipairs(res[2]) do
-			local __, ___, when, i = string.find(key, "[^:]+:[^:]+:[^:]+:([^:]+):([^:]+)");
+			-- version:collections_rows:{method:inputHash}:collection_id:when:i
+			local __, ___, when, i = string.find(key, "[^:]+:[^:]+:[^:]+:[^:]+:[^:]+:([^:]+):([^:]+)");
 			when = tonumber(when);
 			i = tonumber(i);
 
@@ -155,6 +156,20 @@ const createRedisStore = (db) => {
 		debug('init')
 	}
 
+	// collections
+	// version:collections:{method:inputHash}:created -> collectionId:when:duration
+	// version:collections_rows:{method:inputHash}:collection_id:when:i
+
+	// We're using a Redis Cluster hash tag `{method:inputHash}` here. Both the
+	// collection entry and the row entries must be read in atomicly, so we need
+	// to make sure they have the same hash tag!
+	const _collectionPrefix = (method, inputHash) => {
+		return `${VERSION}:${COLLECTIONS}:{${method}:${inputHash}}`
+	}
+	const _collectionRowsPrefix = (method, inputHash) => {
+		return `${VERSION}:${COLLECTIONS_ROWS}:{${method}:${inputHash}}`
+	}
+
 	const readCollection = async (args) => {
 		debug('readCollection', args)
 		const {
@@ -164,11 +179,14 @@ const createRedisStore = (db) => {
 		const createdMin = Math.floor(args.createdMin / 1000)
 		const createdMax = Math.ceil(args.createdMax / 1000)
 
+		const colPrefix = _collectionPrefix(method, inputHash) + ':'
+		// The common prefix is usually longer than just colPrefix because createdMin
+		// & createdMax usually share some digits.
 		const prefix = commonPrefix([
-			[VERSION, COLLECTIONS, method, inputHash, createdMin].join(':'),
-			[VERSION, COLLECTIONS, method, inputHash, createdMax].join(':')
+			`${colPrefix}${createdMin}`,
+			`${colPrefix}${createdMax}`,
 		])
-		const rowsPrefix = `${VERSION}:${COLLECTIONS_ROWS}:`
+		const rowsPrefix = _collectionRowsPrefix(method, inputHash) + ':'
 		const rows = await db[_readMatchingCollection](
 			prefix, rowsPrefix,
 			createdMin, createdMax,
@@ -194,29 +212,35 @@ const createRedisStore = (db) => {
 
 		const collectionId = randomBytes(10).toString('hex')
 
-
+		const rowsPrefix = _collectionRowsPrefix(method, inputHash)
 		const cmds = [
 			[
 				'set',
-				[VERSION, COLLECTIONS, method, inputHash, created].join(':'),
+				[_collectionPrefix(method, inputHash), created].join(':'),
 				[collectionId, when, duration].join(':'),
 				'PX', cachePeriod,
 			],
 			...rows.map((row, i) => {
 				// todo: fall back to plannedWhen?
 				const t = +new Date(row.when)
 				if (Number.isNaN(t)) throw new Error(`rows[${i}].when must be a number or an ISO 8601 string`)
-				const key = [VERSION, COLLECTIONS_ROWS, collectionId, t, i].join(':')
+				// todo: add method & inputHash to match hash tag of `COLLECTIONS:`?
+				const key = [rowsPrefix, collectionId, t, i].join(':')
 				return ['set', key, row.data, 'PX', cachePeriod]
 			}),
 		]
 		await db.multi(cmds).exec()
 	}
 
 	// atomics
-	// method:inputHash:created:id
+	// version:atoms:{method:inputHash}:created:id
 	// todo: this fails with `created` timestamps of different lengths (2033)
 
+	// We're using a Redis Cluster hash tag `{method:inputHash}` here.
+	const _atomPrefix = (method, inputHash) => {
+		return `${VERSION}:${ATOMS}:{${method}:${inputHash}}`
+	}
+
 	const readAtom = async (args) => {
 		debug('readAtom', args)
 		const {
@@ -226,9 +250,12 @@ const createRedisStore = (db) => {
 		const createdMax = Math.ceil(args.createdMax / 1000)
 		const deserialize = args.deserialize || JSON.parse
 
+		const pref = _atomPrefix(method, inputHash) + ':'
+		// The common prefix is usually longer than just colPrefix because createdMin
+		// & createdMax usually share some digits.
 		const keysPrefix = commonPrefix([
-			[VERSION, ATOMS, method, inputHash, createdMin].join(':'),
-			[VERSION, ATOMS, method, inputHash, createdMax].join(':')
+			`${pref}${createdMin}`,
+			`${pref}${createdMax}`
 		])
 		const val = await db[_readMatchingAtom]([
 			keysPrefix,
@@ -247,7 +274,7 @@ const createRedisStore = (db) => {
 		const created = Math.round(args.created / 1000)
 		const serialize = args.serialize || JSON.stringify
 
-		const key = [VERSION, ATOMS, method, inputHash, created].join(':')
+		const key = [_atomPrefix(method, inputHash), created].join(':')
 		await db.set(key, serialize(val), 'PX', cachePeriod)
 	}
 

test/index.js

@@ -62,11 +62,11 @@ const runTests = (storeName, createDb, createStore) => {
 		const store = createStore(db)
 		const cachedMocked = createCachedHafas(mocked, store, {
 			cachePeriods: {
-				departures: ttl, arrivals: ttl,
-				journeys: ttl, refreshJourney: ttl, trip: ttl,
-				radar: ttl,
-				locations: ttl, stop: ttl, nearby: ttl,
-				reachableFrom: ttl,
+				departures: () => ttl, arrivals: () => ttl,
+				journeys: () => ttl, refreshJourney: () => ttl, trip: () => ttl,
+				radar: () => ttl,
+				locations: () => ttl, stop: () => ttl, nearby: () => ttl,
+				reachableFrom: () => ttl,
 				...cachePeriods,
 			}
 		})

test/with-metrics.js

@@ -32,11 +32,11 @@ test('trackCachingMetrics works', async (t) => {
 	const store = createInMemoryStore()
 	const cachedMocked = createCachedHafas(mocked, store, {
 		cachePeriods: {
-			departures: ttl, arrivals: ttl,
-			journeys: ttl, refreshJourney: ttl, trip: ttl,
-			radar: ttl,
-			locations: ttl, stop: ttl, nearby: ttl,
-			reachableFrom: ttl,
+			departures: () => ttl, arrivals: () => ttl,
+			journeys: () => ttl, refreshJourney: () => ttl, trip: () => ttl,
+			radar: () => ttl,
+			locations: () => ttl, stop: () => ttl, nearby: () => ttl,
+			reachableFrom: () => ttl,
 		},
 	})