Skip to content

Commit

Permalink
update loaded s3 files' filename and path, enable other load options
Browse files Browse the repository at this point in the history
  • Loading branch information
@helen-m-lin
helen-m-lin committed Mar 27, 2024
1 parent 85a97c3 commit 2ff9377
Show file tree
Hide file tree
Showing 3 changed files with 92 additions and 35 deletions.
102 changes: 76 additions & 26 deletions src/aws-cognito-s3.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ var s3 = new AWS.S3(); // Default S3 client
// ==================== COGNITO FUNCTIONS ====================
/**
* Wrapper for AWS.CognitoServiceProvider.globalSignOut()
* @param {*} callback - custom callback function (default to cognitoSignOutCallback)
* @param {function} callback - custom callback function (default to cognitoSignOutCallback)
*/
function cognitoSignOut(callback = cognitoSignOutCallback) {
var params = {
Expand All @@ -60,7 +60,7 @@ function cognitoSignOut(callback = cognitoSignOutCallback) {

/**
* Wrapper for AWS.CognitoServiceProvider.initiateAuth()
* @param {*} callback - custom callback function
* @param {function} callback - custom callback function
*/
function cognitoInitiateAuth(callback) {
currentUser.username = $('#username').val();
Expand Down Expand Up @@ -112,7 +112,7 @@ function resetTempCredentials() {
// ==================== COGNITO UI ====================
/**
* Prompt user for login credentials.
* @param {*} successCallback - Optional callback function to execute after successful login.
* @param {function | null} successCallback - Optional callback function to execute after successful login.
*/
function promptLogin(successCallback = null) {
bootbox.confirm({
Expand Down Expand Up @@ -161,7 +161,7 @@ function promptLogout() {
* Otherwise, re-prompt for login.
* @param {*} err - Error object
* @param {*} result - AuthenticationResult object
* @param {*} successCallback - Optional callback function to execute after successful login.
* @param {function | null} successCallback - Optional callback function to execute after successful login.
*/
function cognitoInititateAuthCallback(err, result, successCallback = null) {
if (err) {
Expand Down Expand Up @@ -207,7 +207,7 @@ function cognitoSignOutCallback(error) {
/**
* Wrapper for AWS.S3.listObjectsV2.
* Uses the global S3 client to make an authenticated listObjectsV2 request.
* @param {*} callback - Callback function (defaults to use s3draw() to update the UI)
* @param {function} callback - Callback function (defaults to use s3draw() to update the UI)
*/
function s3list(callback = s3draw) {
var scope = {
Expand Down Expand Up @@ -261,52 +261,56 @@ function s3list(callback = s3draw) {
* For each file, call AWS.S3.getObject to get the file.
* Calls project_file_add_local() function from via.js to add the file(s) to the app.
* @param {string} path - S3 path to file or folder
* @param {string} loadType - project load option (options are 'deeplabcut', 'lightning_pose', 'slp', 'local')
*/
function load_s3_into_app(path) {
const filepath = path.split('/').slice(1).join('/')
var params = {
function load_s3_into_app(path, loadType) {
path = path.replace(S3_BUCKET_NAME + '/', ''); // remove bucket name prefix
var s3Params = {
Bucket: S3_BUCKET_NAME,
Key: filepath
Key: path
};
var fakeEvent = { target: { files: [] } };
if (pathIsFolder(filepath)) {
var project_file_callback = getViaProjectFileCallback(loadType);
if (pathIsFolder(path)) {
// get list of objects in this folder
s3.makeRequest('listObjectsV2', { Bucket: S3_BUCKET_NAME, Prefix: filepath }, async (err, data) => {
s3.makeRequest('listObjectsV2', { Bucket: S3_BUCKET_NAME, Prefix: path }, async (err, data) => {
if (err) {
logErrorAndAlertUser("S3 Error", err)
} else {
const parentPrefix = getParentFolderFromPath(path);
// get each file, then add all files to VIA
await Promise.all(data.Contents.map((obj) => {
let filepath = obj.Key;
return new Promise((resolve, reject) => {
params.Key = filepath;
s3.makeRequest('getObject', params, (err, data) => {
s3Params.Key = filepath;
s3.makeRequest('getObject', s3Params, (err, data) => {
if (err) {
reject(err);
} else {
var file = new File([data.Body], filepath, { type: data.ContentType });
var file = new File([data.Body], getFileOrFolderNameFromPath(filepath), { type: data.ContentType });
file.s3RelativePath = filepath.replace(parentPrefix, ''); // preserve relative path
resolve(file);
}
});
});
})).then((files) => {
console.log(`Retrieved ${files.length} files from ${path}`)
fakeEvent.target.files = files;
project_file_add_local(fakeEvent);
project_file_callback(fakeEvent, true);
}).catch((err) => {
logErrorAndAlertUser("S3 Download Error", err)
});
}
});
} else {
s3.makeRequest('getObject', params, (err, data) => {
s3.makeRequest('getObject', s3Params, (err, data) => {
if (err) {
logErrorAndAlertUser("S3 Error", err)
} else {
console.log(`Retrieved file from ${path}`)
var file = new File([data.Body], filepath, { type: data.ContentType });
var file = new File([data.Body], getFileOrFolderNameFromPath(path), { type: data.ContentType });
fakeEvent.target.files = [file];
project_file_add_local(fakeEvent);
project_file_callback(fakeEvent, true);
}
});
}
Expand All @@ -318,10 +322,11 @@ function load_s3_into_app(path) {
* First prompts user to log in if not already logged in.
* Initializes a DataTable to display S3 objects.
* Includes delegated event handlers for folder entry, selection, reset, and back buttons.
* @param {string} loadType - project load option (default to 'local', other options are 'deeplabcut', 'lightning_pose', 'slp')
*/
function sel_s3_images() {
function sel_s3_images(loadType = 'local') {
if (currentUser.username === '') {
promptLogin(sel_s3_images);
promptLogin(sel_s3_images.bind(null, loadType));
return;
}
bootbox.prompt({
Expand Down Expand Up @@ -358,11 +363,9 @@ function sel_s3_images() {
if (type === 'display') {
// Display clickable folders or text span for file. Both have the key as data-path
if (pathIsFolder(data)) {
let folderName = data.split('/').slice(-2)[0] + '/';
return `<button type="button" class="btn btn-link" data-s3="folder" data-path="${data}">${folderName}</button>`;
return `<button type="button" class="btn btn-link" data-s3="folder" data-path="${data}">${getFileOrFolderNameFromPath(data)}</button>`;
} else {
let fileName = data.split('/').slice(-1)[0];
return `<span data-s3="object" data-path="${data}">${fileName}</button>`;
return `<span data-s3="object" data-path="${data}">${getFileOrFolderNameFromPath(data)}</button>`;
}
}
return data;
Expand All @@ -379,7 +382,7 @@ function sel_s3_images() {
},
callback: (result) => {
if (result) {
load_s3_into_app(result);
load_s3_into_app(result, loadType);
}
// Reset s3Prefix for next selection
s3Prefix = '';
Expand Down Expand Up @@ -434,7 +437,7 @@ function sel_s3_images() {
e.preventDefault();
console.log("Back button clicked");
if (s3Prefix) {
const parentPrefix = s3Prefix.split('/').slice(0, -2).join('/') + '/';
const parentPrefix = getParentFolderFromPath(s3Prefix);
s3Prefix = (parentPrefix === '/') ? '' : parentPrefix
s3list().go();
}
Expand Down Expand Up @@ -490,6 +493,34 @@ function pathIsFolder(path) {
return path.endsWith('/')
}

/**
* Extracts the filename or folder name from a full path.
* E.g. getFilenameFromPath('path/to/file.txt') => 'file.txt'
* E.g. getFilenameFromPath('path/to/folder/') => 'folder/'
* @param {string} path - The full path
* @returns - The filename or folder name
*/
function getFileOrFolderNameFromPath(path) {
if (pathIsFolder(path))
return path.split('/').slice(-2)[0] + '/'
else
return path.split('/').slice(-1)[0];
}

/**
* Extracts the parent folder from a full path.
* E.g. getParentFolderFromPath('path/to/file.txt') => 'path/to/'
* E.g. getParentFolderFromPath('path/to/folder/') => 'path/to/'
* @param {string} path - The full path
* @returns - The parent folder path
*/
function getParentFolderFromPath(path) {
if (pathIsFolder(path))
return path.split('/').slice(0, -2).join('/') + '/';
else
return path.split('/').slice(0, -1).join('/') + '/';
}

/**
* Logs error and alerts the user with an error message.
*
Expand All @@ -502,4 +533,23 @@ function logErrorAndAlertUser(title, error) {
title: title,
error: `Error: ${error.message}`
})
}

/**
* Returns the appropriate VIA project_file_add function based on the loadType.
*
* @param {string} loadType - The type of project load. Options are 'deeplabcut', 'lightning_pose', 'slp', or 'local'.
* @returns - The appropriate VIA project_file_add function.
*/
function getViaProjectFileCallback(loadType) {
switch (loadType) {
case 'deeplabcut':
return project_file_add_deeplabcut;
case 'lightning_pose':
return project_file_add_lightning_pose;
case 'slp':
return project_file_add_slp;
default:
return project_file_add_local;
}
}
5 changes: 4 additions & 1 deletion src/index.html
View file
Original file line number Diff line number Diff line change
Expand Up @@ -276,7 +276,10 @@

<li>Import
<ul>
<li onclick="sel_s3_images()" title="Load files from S3">Load files from S3</li>
<li onclick="sel_s3_images('deeplabcut')" title="Load DeepLabCut dataset from S3">S3: Load DeepLabCut dataset</li>
<li onclick="sel_s3_images('lightning_pose')" title="Load Lightning Pose dataset from S3">S3: Load Lightning Pose dataset</li>
<li onclick="sel_s3_images('slp')" title="Load SLP file from S3">S3: Load SLEAP dataset (SLP file)</li>
<li onclick="sel_s3_images()" title="Load image files/folder from S3">S3: Load image file/folder</li>
<li class="submenu_divider"></li>
<li onclick="sel_local_deeplabcut()" title="Load DeepLabCut dataset">Load DeepLabCut dataset</li>
<li onclick="sel_local_lightning_pose()" title="Load Lightning Pose dataset">Load Lightning Pose dataset</li>
Expand Down
20 changes: 12 additions & 8 deletions src/via.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8572,10 +8572,11 @@ async function project_file_add_slp(event) {
}


function project_file_add_deeplabcut(event) {
function project_file_add_deeplabcut(event, is_s3 = false) {
var files = Array.from(event.target.files);
files.sort(function(a, b) {
return a.webkitRelativePath.localeCompare(b.webkitRelativePath);
return is_s3 ? a.s3RelativePath.localeCompare(b.s3RelativePath)
: a.webkitRelativePath.localeCompare(b.webkitRelativePath);
});

// add the images
Expand All @@ -8589,7 +8590,7 @@ function project_file_add_deeplabcut(event) {
var img_index = _via_image_filename_list.indexOf(files[i].name);
if( img_index === -1) {
// a new file was added to project
var path = files[i].webkitRelativePath;
var path = is_s3 ? files[i].s3RelativePath : files[i].webkitRelativePath;
var pathList = path.split("/");
var folder = pathList[pathList.length - 2];
var name = pathList[pathList.length - 1];
Expand Down Expand Up @@ -8619,7 +8620,7 @@ function project_file_add_deeplabcut(event) {
// find the h5 file
var h5_file;
for ( var i = 0; i < files.length; ++i ) {
var path = files[i].webkitRelativePath;
var path = is_s3 ? files[i].s3RelativePath : files[i].webkitRelativePath;
if(path.slice(path.length-3) == ".h5") {
h5_file = files[i];
break;
Expand Down Expand Up @@ -8665,10 +8666,11 @@ function project_file_add_deeplabcut(event) {
}


function project_file_add_lightning_pose(event) {
function project_file_add_lightning_pose(event, is_s3 = false) {
var files = Array.from(event.target.files);
files.sort(function(a, b) {
return a.webkitRelativePath.localeCompare(b.webkitRelativePath);
return (is_s3) ? a.s3RelativePath.localeCompare(b.s3RelativePath)
: a.webkitRelativePath.localeCompare(b.webkitRelativePath);
});

// add the images
Expand All @@ -8682,7 +8684,7 @@ function project_file_add_lightning_pose(event) {
var img_index = _via_image_filename_list.indexOf(files[i].name);
if( img_index === -1) {
// a new file was added to project
var path = files[i].webkitRelativePath;
var path = is_s3 ? files[i].s3RelativePath : files[i].webkitRelativePath;
var pathList = path.split("/");
var folder = pathList[pathList.length - 2];
var name = pathList[pathList.length - 1];
Expand Down Expand Up @@ -8711,9 +8713,11 @@ function project_file_add_lightning_pose(event) {

// find the csv file
var csv_file;
var csv_file_path;
for ( var i = 0; i < files.length; ++i ) {
csv_file_path = is_s3 ? files[i].s3RelativePath : files[i].webkitRelativePath;
if( (files[i].type == 'text/csv') &&
(files[i].webkitRelativePath.split("/").length == 2) ) {
(csv_file_path.split("/").length == 2) ) {
// top level csv
csv_file = files[i];
break;
Expand Down

0 comments on commit 2ff9377

Please sign in to comment.