Skip to content

Commit

Permalink
Refactor cache
Browse files Browse the repository at this point in the history
  • Loading branch information
@steinbro
steinbro committed Dec 31, 2023
1 parent da72121 commit 8a9143d
Show file tree
Hide file tree
Showing 3 changed files with 157 additions and 156 deletions.
273 changes: 151 additions & 122 deletions app/js/data/cache.js
View file
Original file line number Diff line number Diff line change
@@ -1,14 +1,29 @@
// Copyright (c) Daniel W. Steinbrook.
// with many thanks to ChatGPT

// Bump this when changing schema (e.g. adding an index)
const dbVersion = 1;
const dbName = 'TielCache';
const maxAgeMilliseconds = 604800000; // 1 week

// Function to open the IndexedDB database
function openURLCache() {
async function openDatabase() {
return new Promise((resolve, reject) => {
const request = indexedDB.open('URLCache', 1);
const request = indexedDB.open(dbName, dbVersion);

request.onupgradeneeded = (event) => {
const db = event.target.result;
db.createObjectStore('urls', { keyPath: 'url' });
// Create object stores if not present
if (!db.objectStoreNames.contains("urls")) {
db.createObjectStore('urls', { keyPath: 'url' });
}
if (!db.objectStoreNames.contains("features")) {
db.createObjectStore('features', { keyPath: 'id', autoIncrement: true });
}
// Create feature indexes (should be harmless if they already exist)
var objectStore = event.currentTarget.transaction.objectStore("features");
objectStore.createIndex('tile', 'tile', { unique: false });
objectStore.createIndex("osm_ids", "osm_ids", { multiEntry: true });
};

request.onsuccess = (event) => {
Expand All @@ -22,145 +37,159 @@ function openURLCache() {
});
}

// Function to fetch a URL only if it hasn't been fetched for a certain duration
export async function fetchUrlIfNotCached(url, maxAgeInMilliseconds) {
const db = await openURLCache();

return new Promise(async (resolve, reject) => {
const transaction = db.transaction(['urls'], 'readwrite');
const objectStore = transaction.objectStore('urls');
function clearObjectStore(objectStoreName) {
var request = indexedDB.open(dbName);

// Check if the URL is in the cache
const getRequest = objectStore.get(url);
request.onsuccess = function (event) {
var db = event.target.result;
var transaction = db.transaction([objectStoreName], 'readwrite');
var objectStore = transaction.objectStore(objectStoreName);

getRequest.onsuccess = async (event) => {
const result = event.target.result;
// Clear all data in the object store
var clearRequest = objectStore.clear();

if (result) {
// Check if the cached entry is still valid based on maxAgeInMilliseconds
const currentTime = new Date().getTime();
const lastFetchTime = result.lastFetchTime || 0;

if (currentTime - lastFetchTime < maxAgeInMilliseconds) {
// URL is still fresh, resolve with the cached data
console.log("HIT: ", url);
// Assume data has already been handled when it was first cached
//resolve(result.cachedData);
return;
}
}

// Fetch the URL since it's not in the cache or has expired
console.log("MISS: ", url);
try {
const response = await fetch(url);
const data = await response.json();

const newTransaction = db.transaction(['urls'], 'readwrite');
const newObjectStore = newTransaction.objectStore('urls');

// Update or add the URL in the cache with the current timestamp
const putRequest = newObjectStore.put({
url: url,
lastFetchTime: new Date().getTime(),
// No need to keep the data (the individual features should be
// loaded into their own object store on first fetch)
//cachedData: data,
});

putRequest.onsuccess = () => {
console.log("Fetched: ", url)
resolve(data);
};

putRequest.onerror = (event) => {
reject(`Error updating cache: ${event.target.error}`);
};
} catch (error) {
reject(`Error fetching URL: ${error}`);
}
clearRequest.onsuccess = function () {
console.log('Data cleared successfully.');
};

getRequest.onerror = (event) => {
reject(`Error checking cache: ${event.target.error}`);
clearRequest.onerror = function (error) {
console.error('Error clearing data:', error);
};
});
}
};

export function clearFeatureCache() {
return cleanDatabase('GeoJSONCache', 'features');
request.onerror = function (error) {
console.error('Error opening database:', error);
};
}

export function clearURLCache() {
return cleanDatabase('URLCache', 'urls');
}

export function cleanDatabase(dbName, objectStoreName) {
var request = indexedDB.open(dbName);

request.onsuccess = function (event) {
var db = event.target.result;
var transaction = db.transaction([objectStoreName], 'readwrite');
var objectStore = transaction.objectStore(objectStoreName);

// Clear all data in the object store
var clearRequest = objectStore.clear();
const cache = {
db: null, // to be populated on first request

clearRequest.onsuccess = function () {
console.log('Data cleared successfully.');
};
clear: function() {
return clearObjectStore('features');
return clearObjectStore('urls');
},

clearRequest.onerror = function (error) {
console.error('Error clearing data:', error);
};
};
// Function to fetch a URL only if it hasn't been fetched for a certain duration
fetch: function(url) {
return new Promise(async (resolve, reject) => {
if (!cache.db) {
cache.db = await openDatabase();
}
const transaction = cache.db.transaction(['urls'], 'readwrite');
const objectStore = transaction.objectStore('urls');

// Check if the URL is in the cache
const getRequest = objectStore.get(url);

getRequest.onsuccess = async (event) => {
const result = event.target.result;

if (result) {
// Check if the cached entry is still valid based on maxAgeMilliseconds
const currentTime = new Date().getTime();
const lastFetchTime = result.lastFetchTime || 0;

if (currentTime - lastFetchTime < maxAgeMilliseconds) {
// URL is still fresh, resolve with the cached data
console.log("HIT: ", url);
// Assume data has already been handled when it was first cached
//resolve(result.cachedData);
return;
}
}

request.onerror = function (error) {
console.error('Error opening database:', error);
};
}
// Fetch the URL since it's not in the cache or has expired
console.log("MISS: ", url);
try {
const response = await fetch(url);
const data = await response.json();

const newTransaction = cache.db.transaction(['urls'], 'readwrite');
const newObjectStore = newTransaction.objectStore('urls');

// Update or add the URL in the cache with the current timestamp
const putRequest = newObjectStore.put({
url: url,
lastFetchTime: new Date().getTime(),
// No need to keep the data (the individual features should be
// loaded into their own object store on first fetch)
//cachedData: data,
});

putRequest.onsuccess = () => {
console.log("Fetched: ", url)
resolve(data);
};

putRequest.onerror = (event) => {
reject(`Error updating cache: ${event.target.error}`);
};
} catch (error) {
reject(`Error fetching URL: ${error}`);
}
};

getRequest.onerror = (event) => {
reject(`Error checking cache: ${event.target.error}`);
};
});
},

// Function to add GeoJSON feature to the cache
addFeature: function(feature, tile) {
return new Promise(async (resolve, reject) => {
if (!cache.db) {
cache.db = await openDatabase();
}
const transaction = cache.db.transaction(['features'], 'readwrite');
const objectStore = transaction.objectStore('features');

// Function to open the IndexedDB database
export function openFeatureCache() {
return new Promise((resolve, reject) => {
const request = indexedDB.open('GeoJSONCache', 1);
// Add the tile information to the feature before storing it
feature.tile = tile;

request.onupgradeneeded = function (event) {
const db = event.target.result;
const objectStore = db.createObjectStore('features', { keyPath: 'id', autoIncrement: true });
objectStore.createIndex('tile', 'tile', { unique: false });
};
const request = objectStore.add(feature);

request.onsuccess = function (event) {
const db = event.target.result;
resolve(db);
};
request.onsuccess = function () {
resolve();
};

request.onerror = function (event) {
reject(event.target.error);
};
});
}
request.onerror = function (event) {
reject(event.target.error);
};
});
},

// Function to add GeoJSON feature to the cache
export async function addToFeatureCache(feature, tile) {
const db = await openFeatureCache();
getFeatures: async function(tile) {
return new Promise(async (resolve, reject) => {
if (!cache.db) {
cache.db = await openDatabase();
}
const transaction = cache.db.transaction(['features'], 'readonly');
const objectStore = transaction.objectStore('features');
const tileIndex = objectStore.index('tile');

return new Promise((resolve, reject) => {
const transaction = db.transaction(['features'], 'readwrite');
const objectStore = transaction.objectStore('features');
const range = IDBKeyRange.only(tile.key);
const request = tileIndex.openCursor(range);

// Add the tile information to the feature before storing it
feature.tile = tile;
const features = [];

const request = objectStore.add(feature);
request.onsuccess = function (event) {
const cursor = event.target.result;

request.onsuccess = function () {
resolve();
};
if (cursor) {
features.push(cursor.value);
cursor.continue();
} else {
resolve(features);
}
};

request.onerror = function (event) {
reject(event.target.error);
};
});
request.onerror = function (event) {
reject(event.target.error);
};
});
}
}

export default cache;
35 changes: 4 additions & 31 deletions app/js/data/tile.js
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,10 @@
// Copyright (c) Daniel W. Steinbrook.
// with many thanks to ChatGPT

import { addToFeatureCache, fetchUrlIfNotCached, openFeatureCache } from './cache.js'
import cache from './cache.js'
import config from '../config.js'
import { createBoundingBox, enumerateTilesInBoundingBox } from '../spatial/geo.js'

const maxAge = 604800000; // 1 week, in ms
export const zoomLevel = 16;

// Track tiles that don't need to be re-requested at the moment
Expand All @@ -26,10 +25,10 @@ function createTile(x, y, z) {
tilesInProgressOrDone.add(tile.key);

const urlToFetch = `${config.tileServer}/${tile.key}.json`;
fetchUrlIfNotCached(urlToFetch, maxAge)
cache.fetch(urlToFetch)
.then((data) => {
for (const feature of data.features) {
addToFeatureCache(feature, tile.key);
cache.addFeature(feature, tile.key);
};
console.log(`Loaded ${data.features.length} new features.`)
})
Expand All @@ -41,33 +40,7 @@ function createTile(x, y, z) {
},

getFeatures: async function() {
const db = await openFeatureCache();

return new Promise((resolve, reject) => {
const transaction = db.transaction(['features'], 'readonly');
const objectStore = transaction.objectStore('features');
const tileIndex = objectStore.index('tile');

const range = IDBKeyRange.only(tile.key);
const request = tileIndex.openCursor(range);

const features = [];

request.onsuccess = function (event) {
const cursor = event.target.result;

if (cursor) {
features.push(cursor.value);
cursor.continue();
} else {
resolve(features);
}
};

request.onerror = function (event) {
reject(event.target.error);
};
});
return cache.getFeatures(tile);
},
}

Expand Down
5 changes: 2 additions & 3 deletions app/js/main.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@

import { createSpatialPlayer } from './audio/sound.js'
import { createCalloutAnnouncer } from './audio/callout.js'
import { clearFeatureCache, clearURLCache } from './data/cache.js'
import cache from './data/cache.js'
import { getLocation, watchLocation } from './spatial/geo.js';
import { createLocationProvider } from './spatial/location.js'
import { createMap } from './spatial/map.js';
Expand Down Expand Up @@ -87,7 +87,6 @@ document.addEventListener('DOMContentLoaded', function () {

var btnClear = document.getElementById('btn_clear');
btnClear.addEventListener('click', function() {
clearURLCache();
clearFeatureCache();
cache.clear();
});
});

0 comments on commit 8a9143d

Please sign in to comment.