Skip to content

Commit

Permalink
Implement draft for Google Drive Access
Browse files Browse the repository at this point in the history
  • Loading branch information
m-mohr committed Jul 30, 2024
1 parent e62ed8f commit 62dc422
Show file tree
Hide file tree
Showing 6 changed files with 154 additions and 52 deletions.
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@
"dependencies": {
"@google-cloud/storage": "^7.7.0",
"@google/earthengine": "^0.1.385",
"@googleapis/drive": "^8.11.0",
"@openeo/js-commons": "^1.4.1",
"@openeo/js-processgraphs": "^1.3.0",
"@seald-io/nedb": "^4.0.4",
Expand All @@ -47,6 +48,7 @@
"check-disk-space": "^3.4.0",
"epsg-index": "^2.0.0",
"fs-extra": "^11.2.0",
"googleapis-common": "^7.2.0",
"luxon": "^3.4.4",
"proj4": "^2.10.0",
"restify": "^11.1.0"
Expand Down
96 changes: 50 additions & 46 deletions src/api/worker/batchjob.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import path from 'path';
import ProcessGraph from '../../processgraph/processgraph.js';
import GeeResults from '../../processes/utils/results.js';
import Utils from '../../utils/utils.js';
import GDrive from '../../utils/gdrive.js';
const packageInfo = Utils.require('../../package.json');

export default async function run(config, storage, user, query) {
Expand Down Expand Up @@ -34,22 +35,30 @@ export default async function run(config, storage, user, query) {
const datacube = format.preprocess(GeeResults.BATCH, context, dc, logger);

if (format.canExport()) {
// Ensure early that we have access to the Google Drive API
const drive = new GDrive(context.server(), user);
await drive.connect();
// Start processing
const tasks = await format.export(context.ee, dc, context.getResource());
storage.addTasks(job, tasks);
context.startTaskMonitor();
const filepath = await new Promise((resolve, reject) => {
const driveUrls = await new Promise((resolve, reject) => {
setInterval(async () => {
const updatedJob = await storage.getById(job._id, job.user_id);
if (!updatedJob) {
reject(new Error("Job was deleted"));
}
if (['canceled', 'error', 'finished'].includes(updatedJob.status)) {
// todo: resolve google drive URLs
resolve(job.googleDriveResults);
}
}, 10000);
});
return { filepath, datacube };
// Handle Google Drive specifics (permissions and public URLs)
const folderName = GDrive.getFolderName(job);
await drive.publishFoldersByName(folderName);
const files = await drive.getAssetsForFolder(folderName);

return { files, datacube, links: driveUrls };
}
else {
const response = await format.retrieve(context.ee, dc);
Expand All @@ -62,21 +71,15 @@ export default async function run(config, storage, user, query) {
writer.on('error', reject);
writer.on('close', resolve);
});
return { filepath, datacube };
return { files: [filepath], datacube };
}
});

await Promise.all(computeTasks);

const results = [];
for (const task of computeTasks) {
const { filepath, datacube } = await task;
if (Array.isArray(filepath)) {
filepath.forEach(fp => results.push({ filepath: fp, datacube }));
}
else {
results.push({ filepath, datacube });
}
results.push(await task);
}

const item = await createSTAC(storage, job, results);
Expand Down Expand Up @@ -109,37 +112,12 @@ async function createSTAC(storage, job, results) {
let startTime = null;
let endTime = null;
const extents = [];
for(const { filepath, datacube } of results) {
if (!filepath) {
continue;
}

let asset;
let filename;
if (Utils.isUrl(filepath)) {
let url = new URL(filepath);
console.log(url);
filename = path.basename(url.pathname || url.hash.substring(1));
asset = {
href: filepath,
roles: ["data"],
// type: Utils.extensionToMediaType(filepath),
title: filename
};
}
else {
filename = path.basename(filepath);
const stat = await fse.stat(filepath);
asset = {
href: path.relative(folder, filepath),
roles: ["data"],
type: Utils.extensionToMediaType(filepath),
title: filename,
"file:size": stat.size,
created: stat.birthtime,
updated: stat.mtime
};
}
for(const result of results) {
const files = result.files || [];
const datacube = result.datacube;
const baseAsset = {
roles: ["data"],
};

if (datacube.hasT()) {
const t = datacube.dimT();
Expand All @@ -160,8 +138,8 @@ async function createSTAC(storage, job, results) {
const extent = datacube.getSpatialExtent();
let wgs84Extent = extent;
if (crs !== 4326) {
asset["proj:epsg"] = crs;
asset["proj:geometry"] = extent;
baseAsset["proj:epsg"] = crs;
baseAsset["proj:geometry"] = extent;
wgs84Extent = Utils.projExtent(extent, 4326);
}
// Check the coordinates with a delta of 0.0001 or so
Expand All @@ -171,8 +149,34 @@ async function createSTAC(storage, job, results) {
}
}

const params = datacube.getOutputFormatParameters();
assets[filename] = Object.assign(asset, params.metadata);
for (const file of files) {
let asset;
let filename;
if (Utils.isUrl(file)) {
let url = new URL(file);
filename = path.basename(url.pathname || url.hash.substring(1));
asset = {
href: file,
// type: Utils.extensionToMediaType(file),
title: filename
};
}
else {
filename = path.basename(file);
const stat = await fse.stat(file);
asset = {
href: path.relative(folder, file),
type: Utils.extensionToMediaType(file),
title: filename,
"file:size": stat.size,
created: stat.birthtime,
updated: stat.mtime
};
}

const params = datacube.getOutputFormatParameters();
assets[filename] = Object.assign(asset, baseAsset, params.metadata);
}
}
const item = {
stac_version: packageInfo.stac_version,
Expand Down
7 changes: 2 additions & 5 deletions src/models/userstore.js
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import Errors from '../utils/errors.js';
import crypto from "crypto";
import HttpUtils from '../utils/http.js';
import fse from 'fs-extra';
import GDrive from '../utils/gdrive.js';

export default class UserStore {

Expand All @@ -21,11 +22,7 @@ export default class UserStore {
"openid",
"email",
"https://www.googleapis.com/auth/earthengine",
"https://www.googleapis.com/auth/drive.file",
// "https://www.googleapis.com/auth/drive",
// "https://www.googleapis.com/auth/cloud-platform",
// "https://www.googleapis.com/auth/devstorage.full_control"
];
].concat(GDrive.SCOPES);
}

database() {
Expand Down
3 changes: 2 additions & 1 deletion src/processes/utils/results.js
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import GDrive from '../../utils/gdrive.js';
import Utils from '../../utils/utils.js';
import GeeProcessing from './processing.js';
import GeeTypes from './types.js';
Expand Down Expand Up @@ -76,7 +77,7 @@ const GeeResults = {
const task = ee.batch.Export.image.toDrive({
image,
description: job.title,
folder: 'gee-' + job._id,
folder: GDrive.getFolderName(job),
fileNamePrefix: imageId,
skipEmptyTiles: true,
crs,
Expand Down
2 changes: 2 additions & 0 deletions src/utils/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,8 @@ export default class Config {
certificate: null
};

// We need access to GEE + Drive
this.apiKey = null;
this.serviceAccountCredentialsFile = null;
this.googleAuthClients = [];

Expand Down
96 changes: 96 additions & 0 deletions src/utils/gdrive.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
import drive from '@googleapis/drive';
import { JWT } from 'googleapis-common';
import Utils from './utils.js';

export default class GDrive {

static SCOPES = [
//"https://www.googleapis.com/auth/drive.file",
"https://www.googleapis.com/auth/drive"
];

static getFolderName(job) {
return `gee-${job._id}`;
}

constructor(context, user) {
this.drive = null;
this.context = context;
this.user = user;
}

async connect() {
if (this.drive) {
return;
}

let authType;
const options = {
version: 'v3'
};
if (Utils.isGoogleUser(this.user._id)) {
authType = "user token";
options.access_token = this.user.token;
options.auth = this.context.apiKey;
}
else if (this.context.eePrivateKey) {
authType = "private key";
const client = this.context.eePrivateKey;
options.auth = new JWT(client.client_email, null, client.private_key, GDrive.SCOPES);
}
else {
throw new Error("No authentication method available, must have at least a private key configured.");
}

this.drive = drive.drive(options);
// await this.drive.files.list({pageSize: 1});
console.log(`Authenticated at Google Drive via ${authType}`);
}

// Get the ID from URL
// https://drive.google.com/#folders/1rqL0rZqBCvNS9ZhgiJmPGN72y9ZfS3Ly
// => 1rqL0rZqBCvNS9ZhgiJmPGN72y9ZfS3Ly is the ID
getIdFromUrl(url) {
const parsed = new URL(url);
return parsed.hash.split('/').pop();
}

async publishFoldersByName(name) {
const res = await this.drive.files.list({
q: `mimeType = 'application/vnd.google-apps.folder' and name = '${name}'`,
});
const folders = res.data.files;
if (folders.length === 0) {
throw new Error(`Folder not found: ${name}`);
}
else {
console.log(folders);
const promises = folders.map(folder => this.publishFolder(folder.id));
return Promise.all(promises);
}
}

async publishFolder(id) {
if (Utils.isUrl(id)) {
id = this.getIdFromUrl(id);
}
return await this.drive.permissions.create({
resource: {
'type': 'anyone',
'role': 'reader'
},
fileId: id,
// fields: 'id',
});
}

async getAssetsForFolder(name) {
const res = await this.drive.files.list({
pageSize: 1000,
q: `'${name}' in parents`,
});
const files = res.data.files;
console.log(files);
return files;
}
}

0 comments on commit 62dc422

Please sign in to comment.