Cache STAC API data, various fixes

.gitignore

@@ -1,16 +1,18 @@
 /node_modules/
-/storage/collections/*.json
+/storage/collections/
 /storage/database/*.db
-/storage/user_files/*
-/storage/temp_files/*
-/storage/job_files/*
-/storage/service_files/*
+/storage/user_files/
+/storage/temp_files/
+/storage/job_files/
+/storage/service_files/
+/storage/item_thumb_cache/
 /nbproject/
 /package-lock.json
 /.node-xmlhttprequest-sync-*
 /privatekey.json
 coverage/
 /test-report.html
 /logs
-.idea
 /config_server.json
+.idea
+.vscode

src/api/collections.js

@@ -1,6 +1,7 @@
 import Utils from '../utils/utils.js';
 import Errors from '../utils/errors.js';
 import GeeProcessing from '../processes/utils/processing.js';
+import HttpUtils from '../utils/http.js';
 
 const sortPropertyMap = {
 	'properties.datetime': 'system:time_start',
@@ -203,7 +204,6 @@ export default class Data {
 			if (!prop) {
 				throw new Errors.ParameterValueUnsupported({parameter: "sortby", message: "Selected field can't be sorted by."});
 			}
-			console.log(prop, order);
 			ic = ic.sort(prop, order);
 		}
 
@@ -225,6 +225,10 @@ export default class Data {
 			items.pop();
 		}
 
+		Promise.all(items.map(item => this.catalog.itemCache.addItem(item)))
+			.then(() => this.catalog.itemCache.removeOutdated())
+			.catch(console.error);
+
 		// Convert to STAC
 		const features = items.map(item => this.catalog.convertImageToStac(item, id));
 		// Add links
@@ -279,51 +283,59 @@ export default class Data {
 			throw new Errors.CollectionNotFound();
 		}
 
-		// Load the collection and read a "page" of items
-		const img = this.ee.Image(cid + '/' + id);
-		// Retrieve the item
-		let metadata;
-		try {
-			metadata = await GeeProcessing.evaluate(img);
-		} catch (e) {
-			throw new Errors.Internal({message: e.message});
+		const fullId = `${cid}/${id}`;
+		let metadata = await this.catalog.itemCache.getItem(fullId);
+		if (!metadata) {
+			const img = this.ee.Image(fullId);
+			try {
+				metadata = await GeeProcessing.evaluate(img);
+				this.catalog.itemCache.addItem(metadata).catch(console.error);
+			} catch (e) {
+				throw new Errors.Internal({message: e.message});
+			}
 		}
-		// Convert to STAC and deliver
+
 		res.json(this.catalog.convertImageToStac(metadata, cid));
 	}
 
 	async getThumbnailById(req, res) {
 		const id = req.params['*'];
+		const filepath = this.catalog.itemCache.getThumbPath(id);
 
-		const idParts = id.split('/');
-		idParts.pop();
-		const cid = idParts.join('/');
-
-		const vis = this.catalog.getImageVisualization(cid);
-		if (vis === null) {
-			throw new Errors.Internal({message: 'No visualization parameters found.'});
+		if (await this.catalog.itemCache.hasThumb(id)) {
+			await HttpUtils.sendFile(filepath, res);
 		}
+		else {
+			const idParts = id.split('/');
+			idParts.pop();
+			const cid = idParts.join('/');
+
+			const vis = this.catalog.getImageVisualization(cid);
+			if (vis === null) {
+				throw new Errors.Internal({message: 'No visualization parameters found.'});
+			}
 
-		const img = this.ee.Image(id);
-		const geeURL = await new Promise((resolve, reject) => {
-			img.visualize(vis.band_vis).getThumbURL({
-				dimensions: 1000,
-				crs: 'EPSG:3857',
-				format: 'jpg'
-			}, (url, err) => {
-				if (typeof err === 'string') {
-					reject(new Errors.Internal({message: err}));
-				}
-				else if (typeof url !== 'string' || url.length === 0) {
-					reject(new Errors.Internal({message: 'Download URL provided by Google Earth Engine is empty.'}));
-				}
-				else {
-					resolve(url);
-				}
+			const img = this.ee.Image(id);
+			const geeURL = await new Promise((resolve, reject) => {
+				img.visualize(vis.band_vis).getThumbURL({
+					dimensions: 1000,
+					crs: 'EPSG:3857',
+					format: 'png'
+				}, (geeUrl, err) => {
+					if (typeof err === 'string') {
+						reject(new Errors.Internal({message: err}));
+					}
+					else if (typeof geeUrl !== 'string' || geeUrl.length === 0) {
+						reject(new Errors.Internal({message: 'Download URL provided by Google Earth Engine is empty.'}));
+					}
+					else {
+						resolve(geeUrl);
+					}
+				});
 			});
-		});
 
-		return res.redirect(geeURL, Utils.noop);
+			await HttpUtils.streamToFile(geeURL, filepath, res);
+		}
 	}
 
 	async getAssetById(req, res) {

src/api/jobs.js

@@ -45,17 +45,7 @@ export default class JobsAPI {
 
 	async deliverFile(res, filepath) {
 		await HttpUtils.isFile(filepath);
-
-		res.header('Content-Type', Utils.extensionToMediaType(filepath));
-		return await new Promise((resolve, reject) => {
-			const stream = fse.createReadStream(filepath);
-			stream.pipe(res);
-			stream.on('error', reject);
-			stream.on('close', () => {
-				res.end();
-				resolve();
-			});
-		});
+		await HttpUtils.sendFile(filepath, res);
 	}
 
 	init(req) {
@@ -121,7 +111,7 @@ export default class JobsAPI {
 			user_id: req.user._id
 		};
 		const db = this.storage.database();
-		const numRemoved = db.removeAsync(query);
+		const numRemoved = await db.removeAsync(query);
 		if (numRemoved === 0) {
 			throw new Errors.JobNotFound();
 		}
@@ -334,7 +324,7 @@ export default class JobsAPI {
 		await Promise.all(promises);
 
 		const db = this.storage.database();
-		const { numAffected } = db.updateAsync(query, { $set: data });
+		const { numAffected } = await db.updateAsync(query, { $set: data });
 		if (numAffected === 0) {
 			throw new Errors.Internal({message: 'Number of changed elements was 0.'});
 		}

src/api/storedprocessgraphs.js

@@ -92,7 +92,7 @@ export default class StoredProcessGraphs {
 			user_id: req.user._id
 		};
 		const db = this.storage.database();
-		const numRemoved = db.removeAsync(query);
+		const numRemoved = await db.removeAsync(query);
 		if (numRemoved === 0) {
 			throw new Errors.ProcessGraphNotFound();
 		}

src/models/catalog.js

@@ -1,6 +1,7 @@
 import Utils from '../utils/utils.js';
 import fse from 'fs-extra';
 import path from 'path';
+import ItemStore from './itemstore.js';
 import { Storage } from '@google-cloud/storage';
 
 const STAC_DATACUBE_EXTENSION = "https://stac-extensions.github.io/datacube/v2.2.0/schema.json";
@@ -57,9 +58,11 @@ export default class DataCatalog {
 		this.collections = {};
 		this.supportedGeeTypes = ['image', 'image_collection'];
 		this.serverContext = context;
+		this.itemCache = new ItemStore();
 	}
 
 	async readLocalCatalog() {
+		await fse.ensureDir(this.dataFolder);
 		this.collections = {};
 		const files = await fse.readdir(this.dataFolder, { withFileTypes: true });
 		const promises = files.map(async (file) => {
@@ -84,6 +87,8 @@ export default class DataCatalog {
 	}
 
 	async updateCatalog(force = false) {
+		await fse.ensureDir(this.dataFolder);
+		await this.itemCache.clear();
 		// To refresh the catalog manually, delete the catalog.json or set force to true
 		const catalogFile = this.dataFolder + 'catalog.json';
 		if (!force && await fse.exists(catalogFile)) {
@@ -226,10 +231,15 @@ export default class DataCatalog {
 		];
 
 		const id = img.properties["system:index"];
-		const geometry = img.properties["system:footprint"];
-		geometry.type = "Polygon";
-		geometry.coordinates = [geometry.coordinates];
-		const bbox = Utils.geoJsonBbox(geometry, true);
+		let geometry = null;
+		let bbox;
+		if (img.properties["system:footprint"]) {
+			geometry = {
+				type: "Polygon",
+				coordinates: [img.properties["system:footprint"].coordinates]
+			};
+			bbox = Utils.geoJsonBbox(geometry, true);
+		}
 
 		const bands = img.bands.map(b => ({
 			name: b.id,

src/models/itemstore.js

@@ -0,0 +1,51 @@
+import DB from '../utils/db.js';
+import fse from 'fs-extra';
+import path from 'path';
+
+export default class ItemStore {
+
+	constructor() {
+		this.thumbCacheFolder = './storage/item_thumb_cache';
+		this.db = DB.load('item_cache');
+	}
+
+	database() {
+		return this.db;
+	}
+
+	async clear() {
+		await fse.ensureDir(this.thumbCacheFolder);
+		await this.removeOutdated();
+	}
+
+	async getItem(id) {
+		return await this.db.findOneAsync({_id: id});
+	}
+
+	async removeOutdated() {
+		// 7 day cache
+		const daysAgo = Date.now() - 1000 * 60 * 60 * 24 * 7;
+		return await this.db.removeAsync({
+			_time: {$lt: daysAgo}
+		}, {
+			multi: true
+		});
+	}
+
+	async addItem(data) {
+		const dbData = Object.assign({
+			_id: data.id,
+			_time: Date.now()
+		}, data);
+		return await this.db.updateAsync(dbData, { upsert: true });
+	}
+
+	getThumbPath(id) {
+		return path.join(this.thumbCacheFolder, `${id}.png`);
+	}
+
+	async hasThumb(id) {
+		return await fse.pathExists(this.getThumbPath(id));
+	}
+
+}

src/models/jobstore.js

@@ -39,7 +39,7 @@ export default class JobStore {
 	}
 
 	async findJob(query) {
-		const job = this.db.findOneAsync(query);
+		const job = await this.db.findOneAsync(query);
 		if (job === null) {
 			throw new Errors.JobNotFound();
 		}

src/utils/http.js

@@ -1,6 +1,8 @@
 import axios from 'axios';
 import Errors from './errors.js';
 import fse from 'fs-extra';
+import Utils from './utils.js';
+import path from 'path';
 
 const HttpUtils = {
 
@@ -22,9 +24,9 @@ const HttpUtils = {
 		return response.data;
 	},
 
-	async isFile(path) {
+	async isFile(filepath) {
 		try {
-			const stat = await fse.stat(path);
+			const stat = await fse.stat(filepath);
 			if (stat.isFile()) {
 				return true;
 			}
@@ -60,8 +62,42 @@ const HttpUtils = {
 			}
 			throw error;
 		});
+	},
 
+	async streamToFile(url, filepath, res = null) {
+		await fse.ensureDir(path.dirname(filepath));
+		return await new Promise((resolve, reject) => {
+			const fileStream = fse.createWriteStream(filepath);
+			axios.get(url, {responseType: 'stream'})
+				.then(response => {
+					response.data.pipe(fileStream);
+					if (res) {
+						res.header('Content-Type', Utils.extensionToMediaType(filepath));
+						response.data.pipe(res);
+					}
+					fileStream.on('close', () => resolve());
+					fileStream.on('error', (e) => reject(e));
+				})
+				.catch(e => {
+					fileStream.end();
+					reject(e);
+				});
+		});
 	},
+
+	sendFile(filepath, res) {
+		res.header('Content-Type', Utils.extensionToMediaType(filepath));
+		return new Promise((resolve, reject) => {
+			const stream = fse.createReadStream(filepath);
+			stream.pipe(res);
+			stream.on('error', reject);
+			stream.on('close', () => {
+				res.end();
+				resolve();
+			});
+		});
+	}
+
 };
 
 export default HttpUtils;