PTV-1886 staging area limit file size

.flake8

@@ -1,17 +1,26 @@
 [flake8]
 max-line-length = 100
+exclude =
+    scripts/*,
+    modules/*,
+    test/*,
+    kbase-extension,
+    nbextensions,
+    node_modules
+putty-ignore =
+    src/*/__init__.py : F401,E126
 # E203: whitespace before ‘,’, ‘;’, or ‘:’
 # E501: line length
 # W503: line break after binary operator
 ignore = E203, E501, W503, D, DAR
 extend-select = B902, B903, B904
 extend-exclude =
-*.pyc,
-.github,
-.husky,
-deployment,
-docs,
-node_modules,
-js-coverage,
-python-coverage,
-src/build
+    *.pyc,
+    .github,
+    .husky,
+    deployment,
+    docs,
+    node_modules,
+    js-coverage,
+    python-coverage,
+    src/build

docs/developer/data_import.md

@@ -10,6 +10,55 @@ The general flow of data goes from an external data source, to files uploaded to
 
 The Staging Service acts as a first stop for importing data. This service provides an API to a KBase-hosted file system, where each user has a separate file storage directory under their username. The Narrative front end uses a [REST client](../../kbase-extension/static/kbase/js/api/StagingServiceClient.js) to access that service and manipulate files.
 
+## Staging Service Proxy
+
+We limit staging file uploads to 5GB.
+
+This limit is enforced in the `fileUploadWidget`, but we should also enforce it at the server level as well. To do so, we need to configure the `nginx` proxy server which runs in front of the `staging_service`.
+
+Unfortunately, due to the design of the file upload service, this is accomplished by a limit on the request body, not the file size. In the current implementation of the the fileUploadWidget and Staging Service, the file is transmitted as `multipart/form-data`. This implies that the http request body is larger than the file itself. Therefore, the technique illustrated below adds a small, 1k padding to accommodate the destPath form part as well as the headers for each part.
+
+This section provides an example of how nginx may be configured to support both the request body limit and a custom 413 response.
+
+
+### set the client max body size
+
+Within the server block which handles the reverse proxy or the specific location for the `staging_server` set the `client_max_body_size` to the desired limit.
+
+```nginx
+    client_max_body_size 5000001000;
+```
+
+Note that the body size is 5GB with a 1K bytes padding.
+
+### create a custom response for 413
+
+A custom 413 response can provide more detailed information to the front end to facilitate debugging and better error reports.
+
+For a custom 413 response, first define a location within the server block handling the `staging_server`.
+
+```nginx
+    location @request_entity_too_large {
+        default_type application/json;
+        return 413
+         "{\"message\": \"Request entity is too large\", \"responseCode\": 413, \"maxBodySize\": \"5GB\", \"contentLength\": ${content_length}}";
+    }
+```
+
+This causes a JSON response to be sent. The fileUploadWidget will correctly handle this response (as well as the standard nginx plain text response).
+
+Note that this allows the front end to accurately report the content length that violated the max body size. In addition, I could not find a variable
+holding the value of `client_max_body_size`, nor determine if it was possible to use a variable, so the `maxBodySize` property must be set manually 
+to match the value of `client_max_body_size`.
+
+### Use the custom error page in the correct place
+
+The custom error page is defined, now it must be indicated as the 413 handler within the location. This may be set at the server level, or the location level.
+
+```nginx
+    error_page 413 @request_entity_too_large;   
+```
+
 ## Staging Service Viewer
 
 Users can interact with the Staging Service through the Import tab of the data slideout. The top portion of this tab is controlled by the [FileUploadWidget](../../kbase-extension/static/kbase/js/widgets/narrative_core/upload/fileUploadWidget.js). This uses [Dropzone](https://www.dropzone.dev/js/) to provide a drag-and-drop interface for uploading files. If the user's account is linked to Globus, a link is also provided. Another option is to upload via URL. This requires running an app that imports data to the Staging Service from an external, public URL.

docs/developer/local-docker.md

@@ -152,6 +152,9 @@ it would be possible to run any node-based tool via a simple node container, but
   - `NARRATIVE_SERVER_URL=http://localhost:8888 npm run test_local` if running w/o kbase-ui
   - `NARRATIVE_SERVER_URL=https://ci.kbase.us/narrative npm run test_local`, optionally if running with kbase-ui proxying ci.
 
+> NOTE: this no longer works, due to changes in the test scripts. So just use `make test-frontend-unit`, making sure that ci.base.us is no longer
+> routed to `127.0.0.1` in `/etc/hosts`.
+
 Please note the volume mounts in `scripts/local-dev-run.sh`. Not all directories in kbase-extension/static local narrative repo are mounted, due to
 the fact that ext_components is installed.
 

kbase-extension/jupyter_notebook_config.py

@@ -1,3 +1,5 @@
+import os
+
 # Configuration file for ipython-notebook.
 
 c = get_config()  # noqa: F821
@@ -193,8 +195,6 @@
 # c.NotebookApp.extra_config_file = u''
 
 
-import os
-
 try:
     myfile = __file__
 except NameError:

kbase-extension/scss/all_concat.scss

@@ -38,6 +38,7 @@
 @import "partials/cellToolbar";
 @import "partials/dataStaging";
 @import "partials/dropzoneDz";
+@import "partials/fileUploadWidget";
 @import "partials/errorDisplay";
 @import "partials/filePath";
 @import "partials/filetypePanel";

kbase-extension/scss/partials/_dropzoneDz.scss

@@ -2,6 +2,7 @@
   @extend %kbase-button;
 
   @include button-variant(use_color('primary'), use_color('primary-lightest'), transparent);
+
 }
 
 .dz-file {
@@ -78,8 +79,8 @@
 
 .kb-dropzone {
   border: 2px dashed use_color('mid-blue') !important;
-  margin-bottom: 5px;
-  max-height: 150px;
+  // margin-bottom: 5px;
+  max-height: 300px;
   overflow-y: auto;
 
   &-progress__header {
@@ -91,14 +92,23 @@
   .dz-message {
     color: rgb(0 0 0);
     font: normal 400 24px/28px $typeface-page-text;
-    margin: 2em 4.5em;
+    // margin: 2em 4.5em;
     mix-blend-mode: normal;
     text-align: center;
+    margin: 0;
   }
 
   // .kb-dropzone__message--upload
   &__message--upload {
     font-family: $typeface-page-text;
     font-weight: 700;
   }
-}
+
+  &.dropzone.dz-clickable button {
+    cursor: pointer;
+  }
+
+  &.dropzone.dz-clickable a {
+    cursor: pointer;
+  }
+}

kbase-extension/scss/partials/_fileUploadWidget.scss

@@ -0,0 +1,59 @@
+.kb-file-upload-widget {
+    display: flex;
+    flex-direction: column;
+    justify-items: center;
+    margin-bottom: 0.5rem;
+
+    &__error-message {
+        display: flex;
+        flex-direction: row;
+
+        .-title {
+            flex: 0 0 4rem;
+            align-items: center;
+        }
+
+        .-body {
+            flex: 1 1 0;
+            display: flex;
+            flex-direction: column;
+        }
+    }
+
+    &__clear-all-button-container {
+        margin-top: 0.5rem;
+    }
+
+    &__globus-upload-link-container {
+        display: inline-block;
+        vertical-align: baseline;
+    }
+
+    // Bootstrap overrides
+
+    // necessary?
+    .progress-bar {
+        width: 0;
+    }
+
+    &__globus_error_link.btn.btn-link {
+        padding: 0;
+        margin: 0;
+        font-size: inherit;
+    }
+
+    // Not really sure what is going on here.
+    // "row" is bootstrap
+    // "file-row" is kbase, bespoke, not defined elsewhere
+    .row.file-row {
+      padding: 0.25rem;
+    }
+
+    // Dropzone override
+
+    .row.file-row.dz-error {
+        border: 1px solid use_color('error-dark');
+        color: use_color('error-dark');
+    }
+
+}

kbase-extension/scss/partials/_stagingTable.scss

@@ -6,6 +6,7 @@
   }
 
   .kb-staging-table-header {
+
     // Overriding default datatables sort icons so they come
     // before the header title rather than after
     .sorting::before,
@@ -35,6 +36,7 @@
     }
 
     .kb-staging-table-header {
+
       // .kb-staging-table-header__checkbox
       &__checkbox {
         padding-right: 0;
@@ -70,9 +72,15 @@
 
   // .kb-staging-table-body
   &-body {
+
     // .kb-staging-table-body__cell--expander
     &__cell--expander {
       text-align: center;
+
+      &:hover {
+        background-color: use_color('base-lightest');
+        cursor: pointer;
+      }
     }
 
     // .kb-staging-table-body__cell--import
@@ -82,6 +90,7 @@
     }
 
     &__button {
+
       // .kb-staging-table-body__decompress
       &--decompress {
         @extend %kbase-button-sm;
@@ -158,6 +167,7 @@
 
       // style for Import As dropdown container text
       .select2-selection {
+
         // .kb-staging-table-body .kb-staging-table-body__import-dropdown .select2-selection__rendered
         &__rendered {
           color: use_color('ink');
@@ -189,6 +199,7 @@
           border-color: use_color('grey') transparent transparent;
         }
       }
+
       // Import as dropdown placeholder text is red
       // Change placeholder color when user clicks in
       .select2-container--focus .select2-selection__placeholder,
@@ -201,6 +212,7 @@
         border: 1px solid use-color('silver');
       }
     }
+
     // When user clicks into the dropdown or chooses a selection box, change border color
     .select2-container--focus .kb-staging-table-body__import-dropdown,
     .select2-container--open .kb-staging-table-body__import-dropdown {

kbase-extension/static/kbase/js/util/bootstrapDialog.js

@@ -119,6 +119,10 @@ define(['jquery', 'bootstrap'], ($) => {
         this.$modal.modal('show');
     };
 
+    BootstrapDialog.prototype.onShown = function (handler) {
+        this.$modal.on('shown.bs.modal', handler);
+    };
+
     BootstrapDialog.prototype.hide = function () {
         this.$modal.modal('hide');
     };

kbase-extension/static/kbase/js/widgets/common/AlertMessage.js

@@ -2,17 +2,16 @@ define([
     'jquery',
 
     // for effect
-    'bootstrap'
+    'bootstrap',
 ], ($) => {
     'use strict';
 
-    function $AlertMessage(message, { type } = {}) {
-        if (typeof type === 'string' && !['info', 'warning', 'danger', 'success'].includes(type)) {
+    function $AlertMessage(message, options = {}) {
+        const type = options.type || 'info';
+        if (!['info', 'warning', 'danger', 'success'].includes(type)) {
             throw new Error(`Alert type "${type}" not recognized`);
         }
-        return $('<div>')
-            .addClass(`alert alert-${type}`)
-            .text(message);
+        return $('<div>').addClass(`alert alert-${type}`).text(message);
     }
 
     return $AlertMessage;

kbase-extension/static/kbase/js/widgets/narrative_core/kbaseNarrativeStagingDataTab.js

@@ -85,6 +85,7 @@ define([
                     path: this.path,
                     userInfo: userInfo,
                     userId: Jupyter.narrative.userId,
+                    maxFileSize: Config.get('upload').max_file_size * 1000 * 1000, // max file size config is in MB, we want bytes
                 });
 
                 this.uploadWidget.dropzone.on('complete', () => {