From 1676cb22c6a1a8a6155182083c2bc2aa67e76467 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Feb 2019 12:57:29 -0500 Subject: [PATCH 01/12] Merged devenv.sh with webodm.sh --- devenv.sh | 56 +--------------------------------------------------- package.json | 2 +- webodm.sh | 39 +++++++++++++++++++++++++++++------- 3 files changed, 34 insertions(+), 63 deletions(-) diff --git a/devenv.sh b/devenv.sh index fe81be10d..6524b9ca4 100755 --- a/devenv.sh +++ b/devenv.sh @@ -1,57 +1,3 @@ #!/bin/bash -set -eo pipefail -__dirname=$(cd $(dirname "$0"); pwd -P) -${__dirname}/webodm.sh checkenv - -export WO_DEBUG=YES - -usage(){ - echo "Usage: $0 [options]" - echo - echo "This program helps to setup a development environment for WebODM using docker." - echo - echo "Command list:" - echo " start Start the development environment" - echo " stop Stop the development environment" - echo " down Tear down the development environment" - echo " runtests Run unit tests" - exit -} - -run(){ - echo $1 - eval $1 -} - -start(){ - run "docker-compose -f docker-compose.yml -f docker-compose.nodeodm.yml -f docker-compose.dev.yml up" -} - -stop(){ - run "${__dirname}/webodm.sh stop" -} - -down(){ - run "${__dirname}/webodm.sh down" -} - -runtests(){ - run "docker-compose exec webapp /bin/bash -c \"/webodm/webodm.sh test\"" -} - -if [[ $1 = "start" ]]; then - echo "Starting development environment..." - start -elif [[ $1 = "stop" ]]; then - echo "Stopping development environment..." - stop -elif [[ $1 = "down" ]]; then - echo "Tearing down development environment..." - down -elif [[ $1 = "runtests" ]]; then - echo "Starting tests..." - runtests "$2" -else - usage -fi +echo "devenv.sh is deprecated! Use \"./webodm.sh start --dev\" instead." diff --git a/package.json b/package.json index 7fd87c19c..66547619e 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "WebODM", - "version": "0.8.0", + "version": "0.8.1", "description": "Open Source Drone Image Processing", "main": "index.js", "scripts": { diff --git a/webodm.sh b/webodm.sh index 943e2d114..b4bcb1114 100755 --- a/webodm.sh +++ b/webodm.sh @@ -29,6 +29,7 @@ elif [[ $platform = "MacOS / OSX" ]] && [[ $(pwd) == /Users* ]]; then fi load_default_node=true +dev_mode=false # Load default values source .env @@ -83,6 +84,11 @@ case $key in --debug) export WO_DEBUG=YES shift # past argument + ;; + --dev) + export WO_DEBUG=YES + dev_mode=true + shift # past argument ;; --broker) export WO_BROKER="$2" @@ -137,6 +143,7 @@ usage(){ echo " --ssl-cert Manually specify a path to the certificate file (.pem) to use with nginx to enable SSL (default: None)" echo " --ssl-insecure-port-redirect Insecure port number to redirect from when SSL is enabled (default: $DEFAULT_SSL_INSECURE_PORT_REDIRECT)" echo " --debug Enable debug for development environments (default: disabled)" + echo " --dev Enable development mode. In development mode you can make modifications to WebODM source files and changes will be reflected live. (default: disabled)" echo " --broker Set the URL used to connect to the celery broker (default: $DEFAULT_BROKER)" if [[ $plugins_volume = false ]]; then echo " --mount-plugins-volume Always mount the ./plugins volume, even on unsupported platforms (developers only) (default: disabled)" @@ -184,7 +191,12 @@ run(){ } start(){ - echo "Starting WebODM..." + if [[ $dev_mode = true ]]; then + echo "Starting WebODM in development mode..." + down + else + echo "Starting WebODM..." + fi echo "" echo "Using the following environment:" echo "================================" @@ -205,6 +217,10 @@ start(){ if [[ $load_default_node = true ]]; then command+=" -f docker-compose.nodeodm.yml" fi + + if [[ $dev_mode = true ]]; then + command+=" -f docker-compose.dev.yml" + fi if [ "$WO_SSL" = "YES" ]; then if [ ! -z "$WO_SSL_KEY" ] && [ ! -e "$WO_SSL_KEY" ]; then @@ -328,14 +344,23 @@ plugin_disable(){ } run_tests(){ - echo -e "\033[1mRunning frontend tests\033[0m" - run "npm run test" + # If in a container, we run the actual test commands + # otherwise we launch this command from the container - echo "\033[1mRunning backend tests\033[0m" - run "python manage.py test" + in_container=$(grep 'docker\|lxc' /proc/1/cgroup) + if [[ "$in_container" != "" ]]; then + echo -e "\033[1mRunning frontend tests\033[0m" + run "npm run test" - echo "" - echo -e "\033[1mDone!\033[0m Everything looks in order." + echo "\033[1mRunning backend tests\033[0m" + run "python manage.py test" + + echo "" + echo -e "\033[1mDone!\033[0m Everything looks in order." + else + echo "Running tests in webapp container" + run "docker-compose exec webapp /bin/bash -c \"/webodm/webodm.sh test\"" + fi } resetpassword(){ From 2b9c346ac9b41a6ae67c255b7932e9d1466e46c3 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Feb 2019 13:04:26 -0500 Subject: [PATCH 02/12] grep exit code fix, docs update --- README.md | 2 +- slate/source/includes/_fordevelopers.md | 4 ++-- webodm.sh | 3 +-- 3 files changed, 4 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index a226d52dd..c03c8f256 100644 --- a/README.md +++ b/README.md @@ -314,7 +314,7 @@ You don't necessarily need to be a developer to become a contributor. We can use If you know how to code, we primarily use Python (Django), Javascript (React), HTML and SCSS. See the [Development Quickstart](http://docs.webodm.org/#development-quickstart) and [Contributing](/CONTRIBUTING.md) documents for more information. -To make a contribution, you will need to open a pull request ([here's how](https://github.com/Roshanjossey/first-contributions#fork-this-repository)). To make changes to WebODM, make a clone of the repository and run `./devenv.sh start`. +To make a contribution, you will need to open a pull request ([here's how](https://github.com/Roshanjossey/first-contributions#fork-this-repository)). To make changes to WebODM, make a clone of the repository and run `./webodm.sh start --dev`. If you have questions visit us on the [forum](http://community.opendronemap.org/c/webodm) and we'll be happy to help you out with your first contribution. diff --git a/slate/source/includes/_fordevelopers.md b/slate/source/includes/_fordevelopers.md index bf7650ef6..5937392fb 100644 --- a/slate/source/includes/_fordevelopers.md +++ b/slate/source/includes/_fordevelopers.md @@ -22,7 +22,7 @@ Once you have a development environment, read about the [project overview](#proj Follow the [Getting Started](https://github.com/OpenDroneMap/WebODM#getting-started) instructions, then run: -`./devenv.sh start` +`./webodm.sh start --dev` That's it! You can modify any of the files, including SASS and React.js files. Changes will be reflected in the running WebODM instance automatically. @@ -36,7 +36,7 @@ We think testing is a necessary part of delivering robust software. We try to ac To run the unit tests, simply type: -`./devenv.sh runtests` +`./webodm.sh test` ## Project Overview diff --git a/webodm.sh b/webodm.sh index b4bcb1114..f4983f8d8 100755 --- a/webodm.sh +++ b/webodm.sh @@ -346,8 +346,7 @@ plugin_disable(){ run_tests(){ # If in a container, we run the actual test commands # otherwise we launch this command from the container - - in_container=$(grep 'docker\|lxc' /proc/1/cgroup) + in_container=$(grep 'docker\|lxc' /proc/1/cgroup || true) if [[ "$in_container" != "" ]]; then echo -e "\033[1mRunning frontend tests\033[0m" run "npm run test" From 2d440cc0e217362075cf4f3c9701765300b0e37d Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Feb 2019 14:12:01 -0500 Subject: [PATCH 03/12] Changed default theme color --- app/models/theme.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/app/models/theme.py b/app/models/theme.py index 49463ef58..10abbf734 100644 --- a/app/models/theme.py +++ b/app/models/theme.py @@ -17,13 +17,13 @@ class Theme(models.Model): # Similar to how discourse.org does it primary = ColorField(default='#2c3e50', help_text="Most text, icons, and borders.") secondary = ColorField(default='#ffffff', help_text="The main background color, and text color of some buttons.") - tertiary = ColorField(default='#18bc9c', help_text="Navigation links.") + tertiary = ColorField(default='#3498db', help_text="Navigation links.") button_primary = ColorField(default='#2c3e50', help_text="Primary button color.") button_default = ColorField(default='#95a5a6', help_text="Default button color.") button_danger = ColorField(default='#e74c3c', help_text="Delete button color.") - header_background = ColorField(default='#18bc9c', help_text="Background color of the site's header.") + header_background = ColorField(default='#3498db', help_text="Background color of the site's header.") header_primary = ColorField(default='#ffffff', help_text="Text and icons in the site's header.") border = ColorField(default='#e7e7e7', help_text="The color of most borders.") From 1da95d7d56070f09f21def729b07e69e0ede92c7 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Feb 2019 16:30:43 -0500 Subject: [PATCH 04/12] Updated README --- README.md | 30 ++++++++---------------------- 1 file changed, 8 insertions(+), 22 deletions(-) diff --git a/README.md b/README.md index c03c8f256..9c46dcae9 100644 --- a/README.md +++ b/README.md @@ -30,10 +30,9 @@ A free, user-friendly, extendable application and [API](http://docs.webodm.org) * [Run it natively](#run-it-natively) * [Run it on the cloud (Google Compute, Amazon AWS)](#run-it-on-the-cloud-google-compute-amazon-aws) +![ezgif-1-c81c8daab2e0](https://user-images.githubusercontent.com/1951843/52976882-3db81d80-3399-11e9-8915-ffb00b062aaf.gif) -![Alt text](https://user-images.githubusercontent.com/1951843/28586405-af18e8cc-7141-11e7-9853-a7feca7c9c6b.gif) - -![Alt text](/screenshots/pointcloud.png?raw=true "3D Display") +![ezgif-1-4d8402e295f9](https://user-images.githubusercontent.com/1951843/52976888-43adfe80-3399-11e9-8bc6-1690806131d1.gif) ## Getting Started @@ -265,25 +264,12 @@ Developer, I'm looking to build an app that displays map results and takes care Developer, I'm looking to build an app that will stay behind a firewall and just needs raw results | [NodeODM](https://github.com/OpenDroneMap/NodeODM) ## Roadmap -- [X] User Registration / Authentication -- [X] UI mockup -- [X] Task Processing -- [X] 2D Map Display -- [X] 3D Model Display -- [ ] NDVI display -- [X] Volumetric Measurements -- [X] Cluster management and setup. -- [ ] Mission Planner -- [X] Plugins/Webhooks System -- [X] API -- [X] Documentation -- [ ] Android Mobile App -- [ ] iOS Mobile App -- [ ] Processing Nodes Volunteer Network -- [X] Unit Testing -- [X] SSL Support - -Don't see a feature that you want? [Help us make it happen](/CONTRIBUTING.md). + +We follow a bottom-up approach to decide what new features are added to WebODM. User feedback guides us in the decision making process and we collect such feedback via [improvement requests](https://github.com/OpenDroneMap/WebODM/issues?q=is%3Aopen+is%3Aissue+label%3Aimprovements). + +Don't see a feature that you want? [Open a feature request](https://github.com/OpenDroneMap/WebODM/issues) or [help us build it](/CONTRIBUTING.md). + +Sometimes we also prioritize work that has received financial backing. If your organization is in the position to financially support the development of a particular feature, [get in touch](https://community.opendronemap.org) and we'll make it happen. ## Getting Help From 321bb57cf23140968dd46634a7960c6d37986173 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Mon, 18 Feb 2019 16:33:37 -0500 Subject: [PATCH 05/12] Updated README --- README.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/README.md b/README.md index 9c46dcae9..70c88c233 100644 --- a/README.md +++ b/README.md @@ -290,7 +290,7 @@ There are many ways to contribute back to the project: - ⭐️ us on GitHub. - Spread the word about WebODM and OpenDroneMap on social media. - While we don't accept donations, you can purchase an [installer](https://webodm.org/download#installer) or a [premium support package](https://webodm.org/services#premium-support). - - Become a contributor (see below to get free swag 🤘) + - Become a contributor 🤘 ## Become a Contributor @@ -304,10 +304,6 @@ To make a contribution, you will need to open a pull request ([here's how](https If you have questions visit us on the [forum](http://community.opendronemap.org/c/webodm) and we'll be happy to help you out with your first contribution. -When your first pull request is accepted, don't forget to fill [this form](https://goo.gl/forms/PZkiPPeNKUHNz0qe2) to get your **free** WebODM T-Shirt 🤘 - -T-Shirt - ## Architecture Overview WebODM is built with scalability and performance in mind. While the default setup places all databases and applications on the same machine, users can separate its components for increased performance (ex. place a Celery worker on a separate machine for running background tasks). From 395b66503eb526859d59d712e7a4910a26337e5f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 19 Feb 2019 13:44:40 -0500 Subject: [PATCH 06/12] Volume calculation fix for anonymous users, added export measurements to GeoJSON --- app/api/tasks.py | 4 +-- app/static/app/js/Console.jsx | 22 +------------- app/static/app/js/classes/Utils.js | 23 ++++++++++++++ plugins/measure/public/MeasurePopup.jsx | 38 ++++++++++++++++++++++++ plugins/measure/public/MeasurePopup.scss | 5 ++++ plugins/measure/public/app.jsx | 22 +++++++++++++- 6 files changed, 90 insertions(+), 24 deletions(-) diff --git a/app/api/tasks.py b/app/api/tasks.py index 944fe7c06..4afd6c965 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -8,7 +8,7 @@ from django.http import HttpResponse from rest_framework import status, serializers, viewsets, filters, exceptions, permissions, parsers from rest_framework.decorators import detail_route -from rest_framework.permissions import IsAuthenticatedOrReadOnly +from rest_framework.permissions import AllowAny from rest_framework.response import Response from rest_framework.views import APIView @@ -198,7 +198,7 @@ def partial_update(self, request, *args, **kwargs): class TaskNestedView(APIView): queryset = models.Task.objects.all().defer('orthophoto_extent', 'dtm_extent', 'dsm_extent', 'console_output', ) - permission_classes = (IsAuthenticatedOrReadOnly, ) + permission_classes = (AllowAny, ) def get_and_check_task(self, request, pk, annotate={}): try: diff --git a/app/static/app/js/Console.jsx b/app/static/app/js/Console.jsx index 4ce7f59b4..99aadde39 100644 --- a/app/static/app/js/Console.jsx +++ b/app/static/app/js/Console.jsx @@ -66,27 +66,7 @@ class Console extends React.Component { } downloadTxt(filename="console.txt"){ - console.log(filename); - function saveAs(uri, filename) { - let link = document.createElement('a'); - if (typeof link.download === 'string') { - link.href = uri; - link.download = filename; - - //Firefox requires the link to be in the body - document.body.appendChild(link); - - //simulate click - link.click(); - - //remove the link when done - document.body.removeChild(link); - } else { - window.open(uri); - } - } - - saveAs("data:application/octet-stream," + encodeURIComponent(this.state.lines.join("\r\n")), filename); + Utils.saveAs(this.state.lines.join("\r\n"), filename); } copyTxt(){ diff --git a/app/static/app/js/classes/Utils.js b/app/static/app/js/classes/Utils.js index 5f429602c..7d1f6940c 100644 --- a/app/static/app/js/classes/Utils.js +++ b/app/static/app/js/classes/Utils.js @@ -80,6 +80,29 @@ export default { let path= scripts[scripts.length-1].src.split('?')[0]; // remove any ?query let mydir= path.split('/').slice(0, -1).join('/')+'/'; // remove last filename part of path return mydir; + }, + + saveAs: function(text, filename){ + function save(uri, filename) { + let link = document.createElement('a'); + if (typeof link.download === 'string') { + link.href = uri; + link.download = filename; + + //Firefox requires the link to be in the body + document.body.appendChild(link); + + //simulate click + link.click(); + + //remove the link when done + document.body.removeChild(link); + } else { + window.open(uri); + } + } + + save("data:application/octet-stream," + encodeURIComponent(text), filename); } }; diff --git a/plugins/measure/public/MeasurePopup.jsx b/plugins/measure/public/MeasurePopup.jsx index 1878847a4..1aea4ddf3 100644 --- a/plugins/measure/public/MeasurePopup.jsx +++ b/plugins/measure/public/MeasurePopup.jsx @@ -1,6 +1,7 @@ import React from 'react'; import PropTypes from 'prop-types'; import './MeasurePopup.scss'; +import Utils from 'webodm/classes/Utils'; import $ from 'jquery'; import L from 'leaflet'; @@ -23,10 +24,46 @@ export default class MeasurePopup extends React.Component { volume: null, // to be calculated error: "" }; + + this.exportMeasurement = this.exportMeasurement.bind(this); + this.getProperties = this.getProperties.bind(this); + this.getGeoJSON = this.getGeoJSON.bind(this); } componentDidMount(){ this.calculateVolume(); + this.props.resultFeature._measurePopup = this; + } + + componentWillUnmount(){ + this.props.resultFeature._measurePopup = null; + } + + getProperties(){ + const result = { + Length: this.props.model.length, + Area: this.props.model.area + }; + if (this.state.volume !== null && this.state.volume !== false){ + result.Volume = this.state.volume; + } + + return result; + } + + getGeoJSON(){ + const geoJSON = this.props.resultFeature.toGeoJSON(); + geoJSON.properties = this.getProperties(); + return geoJSON; + } + + exportMeasurement(){ + const geoJSON = { + type: "FeatureCollection", + features: [this.getGeoJSON()] + }; + + Utils.saveAs(JSON.stringify(geoJSON, null, 4), "measurement.geojson") } calculateVolume(){ @@ -96,6 +133,7 @@ export default class MeasurePopup extends React.Component { {volume === null && !error &&

Volume: computing...

} {typeof volume === "number" &&

Volume: {volume.toFixed("2")} Cubic Meters ({(volume * 35.3147).toFixed(2)} Cubic Feet)

} {error &&

Volume: 200 ? 'long' : '')}>{error}

} + Export to GeoJSON ); } } \ No newline at end of file diff --git a/plugins/measure/public/MeasurePopup.scss b/plugins/measure/public/MeasurePopup.scss index b8241e1b2..eb23b92da 100644 --- a/plugins/measure/public/MeasurePopup.scss +++ b/plugins/measure/public/MeasurePopup.scss @@ -8,4 +8,9 @@ display: block; max-height: 200px; } + + a.export-measurements{ + display: block; + margin-top: 12px; + } } \ No newline at end of file diff --git a/plugins/measure/public/app.jsx b/plugins/measure/public/app.jsx index b1c0e1ac2..d9ffbfd44 100644 --- a/plugins/measure/public/app.jsx +++ b/plugins/measure/public/app.jsx @@ -3,6 +3,7 @@ import './app.scss'; import 'leaflet-measure-ex/dist/leaflet-measure'; import 'leaflet-measure-ex/dist/leaflet-measure.css'; import MeasurePopup from './MeasurePopup'; +import Utils from 'webodm/classes/Utils'; import ReactDOM from 'ReactDOM'; import React from 'react'; import $ from 'jquery'; @@ -11,7 +12,7 @@ export default class App{ constructor(map){ this.map = map; - L.control.measure({ + const measure = L.control.measure({ labels:{ measureDistancesAndAreas: 'Measure volume, area and length', areaMeasurement: 'Measurement' @@ -22,6 +23,25 @@ export default class App{ secondaryAreaUnit: 'acres' }).addTo(map); + const $btnExport = $("
Export Measurements"); + $btnExport.appendTo($(measure.$startPrompt).children("ul.tasks")); + $btnExport.on('click', () => { + const features = []; + map.eachLayer(layer => { + const mp = layer._measurePopup; + if (mp){ + features.push(mp.getGeoJSON()); + } + }); + + const geoJSON = { + type: "FeatureCollection", + features: features + }; + + Utils.saveAs(JSON.stringify(geoJSON, null, 4), "measurements.geojson") + }); + map.on('measurepopupshown', ({popupContainer, model, resultFeature}) => { // Only modify area popup, length popup is fine as default if (model.area !== 0){ From c3f4917e9214c3293f31b4c0698b541c4e9ba111 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 19 Feb 2019 15:10:16 -0500 Subject: [PATCH 07/12] Review step prior to starting processing --- app/static/app/js/components/NewTaskPanel.jsx | 120 ++++++++++-------- .../app/js/components/ProjectListItem.jsx | 4 - .../app/js/components/UploadProgressBar.jsx | 5 +- app/static/app/js/css/NewTaskPanel.scss | 5 + app/static/app/js/css/UploadProgressBar.scss | 5 + 5 files changed, 79 insertions(+), 60 deletions(-) create mode 100644 app/static/app/js/css/UploadProgressBar.scss diff --git a/app/static/app/js/components/NewTaskPanel.jsx b/app/static/app/js/components/NewTaskPanel.jsx index c7dec6443..ea22a93f0 100644 --- a/app/static/app/js/components/NewTaskPanel.jsx +++ b/app/static/app/js/components/NewTaskPanel.jsx @@ -29,7 +29,8 @@ class NewTaskPanel extends React.Component { resizeMode: Storage.getItem('resize_mode') === null ? ResizeModes.YES : ResizeModes.fromString(Storage.getItem('resize_mode')), resizeSize: parseInt(Storage.getItem('resize_size')) || 2048, items: [], // Coming from plugins, - taskInfo: {} + taskInfo: {}, + inReview: false }; this.save = this.save.bind(this); @@ -51,17 +52,26 @@ class NewTaskPanel extends React.Component { } save(e){ - e.preventDefault(); - this.taskForm.saveLastPresetToStorage(); - Storage.setItem('resize_size', this.state.resizeSize); - Storage.setItem('resize_mode', this.state.resizeMode); - if (this.props.onSave) this.props.onSave(this.getTaskInfo()); + if (!this.state.inReview){ + this.setState({inReview: true}); + }else{ + this.setState({inReview: false}); + e.preventDefault(); + this.taskForm.saveLastPresetToStorage(); + Storage.setItem('resize_size', this.state.resizeSize); + Storage.setItem('resize_mode', this.state.resizeMode); + if (this.props.onSave) this.props.onSave(this.getTaskInfo()); + } } cancel = (e) => { - if (this.props.onCancel){ - if (window.confirm("Are you sure you want to cancel?")){ - this.props.onCancel(); + if (this.state.inReview){ + this.setState({inReview: false}); + }else{ + if (this.props.onCancel){ + if (window.confirm("Are you sure you want to cancel?")){ + this.props.onCancel(); + } } } } @@ -106,58 +116,60 @@ class NewTaskPanel extends React.Component { return (
-

{this.props.filesCount} files selected. Please check these additional options:

- { if (domNode) this.taskForm = domNode; }} - /> - - {this.state.editTaskFormLoaded ? -
-
- -
-
- - -
-
- - px -
+
+

{this.props.filesCount} files selected. Please check these additional options:

+ { if (domNode) this.taskForm = domNode; }} + /> + + {this.state.editTaskFormLoaded ? +
+
+ +
+
+ + +
+
+ + px +
+
+ {this.state.items.map((Item, i) =>
+ +
)}
- {this.state.items.map((Item, i) =>
- -
)} -
- : ""} + : ""} +
{this.state.editTaskFormLoaded ?
{this.props.onCancel !== undefined && } - +
: ""} diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 1cf7900a2..2ee1a5a6b 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -415,10 +415,6 @@ class ProjectListItem extends React.Component { /> : ""} - {this.state.upload.uploading || this.state.upload.resizing ? - - : ""} - {this.state.upload.error !== "" ?
diff --git a/app/static/app/js/components/UploadProgressBar.jsx b/app/static/app/js/components/UploadProgressBar.jsx index 39d73af0e..13ce62245 100644 --- a/app/static/app/js/components/UploadProgressBar.jsx +++ b/app/static/app/js/components/UploadProgressBar.jsx @@ -1,3 +1,4 @@ +import '../css/UploadProgressBar.scss'; import React from 'react'; import PropTypes from 'prop-types'; @@ -34,13 +35,13 @@ class UploadProgressBar extends React.Component { `${this.props.totalCount} files uploaded successfully`; return ( -
+
{percentage}%
-
+
{label}
diff --git a/app/static/app/js/css/NewTaskPanel.scss b/app/static/app/js/css/NewTaskPanel.scss index 525d299fc..947bbeca5 100644 --- a/app/static/app/js/css/NewTaskPanel.scss +++ b/app/static/app/js/css/NewTaskPanel.scss @@ -36,4 +36,9 @@ display: inline-block; } } + + .disabled{ + opacity: 0.8; + pointer-events:none; + } } \ No newline at end of file diff --git a/app/static/app/js/css/UploadProgressBar.scss b/app/static/app/js/css/UploadProgressBar.scss new file mode 100644 index 000000000..b2218897a --- /dev/null +++ b/app/static/app/js/css/UploadProgressBar.scss @@ -0,0 +1,5 @@ +.upload-progress-bar{ + .upload-label{ + margin-bottom: 6px; + } +} \ No newline at end of file From d2b49412139bb8afb03be55106fc8f46e3ce8e83 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Tue, 19 Feb 2019 16:15:28 -0500 Subject: [PATCH 08/12] Custom and None basemap layer choices --- app/static/app/js/components/Map.jsx | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/app/static/app/js/components/Map.jsx b/app/static/app/js/components/Map.jsx index a06200dd2..352c6cc3a 100644 --- a/app/static/app/js/components/Map.jsx +++ b/app/static/app/js/components/Map.jsx @@ -232,6 +232,29 @@ class Map extends React.Component { this.basemaps[props.label] = layer; }); + + const customLayer = L.layerGroup(); + customLayer.on("add", a => { + let url = window.prompt(`Enter a tile URL template. Valid tokens are: +{z}, {x}, {y} for Z/X/Y tile scheme +{-y} for flipped TMS-style Y coordinates + +Example: +https://a.tile.openstreetmap.org/{z}/{x}/{y}.png +`, 'https://a.tile.openstreetmap.org/{z}/{x}/{y}.png'); + + if (url){ + customLayer.clearLayers(); + const l = L.tileLayer(url, { + maxZoom: 21, + minZoom: 0 + }); + customLayer.addLayer(l); + l.bringToBack(); + } + }); + this.basemaps["Custom"] = customLayer; + this.basemaps["None"] = L.layerGroup(); } this.autolayers = Leaflet.control.autolayers({ From 55712f0d582926bf3f3cb3913b94b80da7a9d29f Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Wed, 20 Feb 2019 16:42:20 -0500 Subject: [PATCH 09/12] Import task functionality poc --- app/api/tasks.py | 51 +++++++- app/api/urls.py | 5 +- app/migrations/0025_auto_20190220_1854.py | 52 +++++++++ app/models/task.py | 110 ++++++++++++------ app/pending_actions.py | 1 + app/static/app/js/classes/PendingActions.js | 7 +- .../app/js/components/ImportTaskPanel.jsx | 96 +++++++++++++++ .../app/js/components/ProjectListItem.jsx | 29 ++++- app/static/app/js/components/TaskListItem.jsx | 6 +- .../components/tests/ImportTaskPanel.test.jsx | 10 ++ app/static/app/js/css/ImportTaskPanel.scss | 20 ++++ nodeodm/migrations/0006_auto_20190220_1842.py | 24 ++++ nodeodm/models.py | 2 +- 13 files changed, 361 insertions(+), 52 deletions(-) create mode 100644 app/migrations/0025_auto_20190220_1854.py create mode 100644 app/static/app/js/components/ImportTaskPanel.jsx create mode 100644 app/static/app/js/components/tests/ImportTaskPanel.test.jsx create mode 100644 app/static/app/js/css/ImportTaskPanel.scss create mode 100644 nodeodm/migrations/0006_auto_20190220_1842.py diff --git a/app/api/tasks.py b/app/api/tasks.py index 4afd6c965..32645751e 100644 --- a/app/api/tasks.py +++ b/app/api/tasks.py @@ -2,6 +2,8 @@ from wsgiref.util import FileWrapper import mimetypes + +import datetime from django.core.exceptions import ObjectDoesNotExist, SuspiciousFileOperation, ValidationError from django.db import transaction from django.http import FileResponse @@ -13,11 +15,19 @@ from rest_framework.views import APIView from app import models, pending_actions +from nodeodm import status_codes from nodeodm.models import ProcessingNode from worker import tasks as worker_tasks from .common import get_and_check_project, get_tile_json, path_traversal_check +def flatten_files(request_files): + # MultiValueDict in, flat array of files out + return [file for filesList in map( + lambda key: request_files.getlist(key), + [keys for keys in request_files]) + for file in filesList] + class TaskIDsSerializer(serializers.BaseSerializer): def to_representation(self, obj): return obj.id @@ -36,6 +46,7 @@ def get_processing_node_name(self, obj): return None def get_images_count(self, obj): + # TODO: create a field in the model for this return obj.imageupload_set.count() def get_can_rerun_from(self, obj): @@ -142,11 +153,7 @@ def retrieve(self, request, pk=None, project_pk=None): def create(self, request, project_pk=None): project = get_and_check_project(request, project_pk, ('change_project', )) - # MultiValueDict in, flat array of files out - files = [file for filesList in map( - lambda key: request.FILES.getlist(key), - [keys for keys in request.FILES]) - for file in filesList] + files = flatten_files(request.FILES) if len(files) <= 1: raise exceptions.ValidationError(detail="Cannot create task, you need at least 2 images") @@ -322,3 +329,37 @@ def get(self, request, pk=None, project_pk=None, unsafe_asset_path=""): raise exceptions.NotFound("Asset does not exist") return download_file_response(request, asset_path, 'inline') + +""" +Task assets import +""" +class TaskAssetsImport(APIView): + permission_classes = (permissions.AllowAny,) + parser_classes = (parsers.MultiPartParser, parsers.JSONParser, parsers.FormParser,) + + def post(self, request, project_pk=None): + project = get_and_check_project(request, project_pk, ('change_project',)) + + files = flatten_files(request.FILES) + + if len(files) != 1: + raise exceptions.ValidationError(detail="Cannot create task, you need to upload 1 file") + + with transaction.atomic(): + task = models.Task.objects.create(project=project, + auto_processing_node=False, + name="Imported Task", + import_url="file://all.zip", + status=status_codes.RUNNING, + pending_action=pending_actions.IMPORT) + task.create_task_directories() + + destination_file = task.assets_path("all.zip") + with open(destination_file, 'wb+') as fd: + for chunk in files[0].chunks(): + fd.write(chunk) + + worker_tasks.process_task.delay(task.id) + + serializer = TaskSerializer(task) + return Response(serializer.data, status=status.HTTP_201_CREATED) diff --git a/app/api/urls.py b/app/api/urls.py index f9e1b1d60..13f94d6c9 100644 --- a/app/api/urls.py +++ b/app/api/urls.py @@ -3,7 +3,7 @@ from app.api.presets import PresetViewSet from app.plugins import get_api_url_patterns from .projects import ProjectViewSet -from .tasks import TaskViewSet, TaskTiles, TaskTilesJson, TaskDownloads, TaskAssets +from .tasks import TaskViewSet, TaskTiles, TaskTilesJson, TaskDownloads, TaskAssets, TaskAssetsImport from .processingnodes import ProcessingNodeViewSet, ProcessingNodeOptionsView from rest_framework_nested import routers from rest_framework_jwt.views import obtain_jwt_token @@ -25,10 +25,9 @@ url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/tiles/(?P[\d]+)/(?P[\d]+)/(?P[\d]+)\.png$', TaskTiles.as_view()), url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/(?Porthophoto|dsm|dtm)/tiles\.json$', TaskTilesJson.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/download/(?P.+)$', TaskDownloads.as_view()), - url(r'projects/(?P[^/.]+)/tasks/(?P[^/.]+)/assets/(?P.+)$', TaskAssets.as_view()), + url(r'projects/(?P[^/.]+)/tasks/import$', TaskAssetsImport.as_view()), url(r'^auth/', include('rest_framework.urls')), url(r'^token-auth/', obtain_jwt_token), diff --git a/app/migrations/0025_auto_20190220_1854.py b/app/migrations/0025_auto_20190220_1854.py new file mode 100644 index 000000000..2bbf86ceb --- /dev/null +++ b/app/migrations/0025_auto_20190220_1854.py @@ -0,0 +1,52 @@ +# Generated by Django 2.1.5 on 2019-02-20 18:54 + +import app.models.task +import colorfield.fields +import django.contrib.postgres.fields +import django.contrib.postgres.fields.jsonb +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('app', '0024_update_task_assets'), + ] + + operations = [ + migrations.AddField( + model_name='task', + name='import_url', + field=models.TextField(blank=True, default='', help_text='URL this task is imported from (only for imported tasks)'), + ), + migrations.AlterField( + model_name='preset', + name='options', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=list, help_text="Options that define this preset (same format as in a Task's options).", validators=[app.models.task.validate_task_options]), + ), + migrations.AlterField( + model_name='task', + name='available_assets', + field=django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=80), blank=True, default=list, help_text='List of available assets to download', size=None), + ), + migrations.AlterField( + model_name='task', + name='options', + field=django.contrib.postgres.fields.jsonb.JSONField(blank=True, default=dict, help_text='Options that are being used to process this task', validators=[app.models.task.validate_task_options]), + ), + migrations.AlterField( + model_name='task', + name='pending_action', + field=models.IntegerField(blank=True, choices=[(1, 'CANCEL'), (2, 'REMOVE'), (3, 'RESTART'), (4, 'RESIZE'), (5, 'IMPORT')], db_index=True, help_text='A requested action to be performed on the task. The selected action will be performed by the worker at the next iteration.', null=True), + ), + migrations.AlterField( + model_name='theme', + name='header_background', + field=colorfield.fields.ColorField(default='#3498db', help_text="Background color of the site's header.", max_length=18), + ), + migrations.AlterField( + model_name='theme', + name='tertiary', + field=colorfield.fields.ColorField(default='#3498db', help_text='Navigation links.', max_length=18), + ), + ] diff --git a/app/models/task.py b/app/models/task.py index 759f4a166..a654d5b00 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -8,6 +8,7 @@ import json from shlex import quote +import errno import piexif import re @@ -170,6 +171,7 @@ class Task(models.Model): (pending_actions.REMOVE, 'REMOVE'), (pending_actions.RESTART, 'RESTART'), (pending_actions.RESIZE, 'RESIZE'), + (pending_actions.IMPORT, 'IMPORT'), ) # Not an exact science @@ -223,6 +225,8 @@ class Task(models.Model): running_progress = models.FloatField(default=0.0, help_text="Value between 0 and 1 indicating the running progress (estimated) of this task", blank=True) + import_url = models.TextField(null=False, default="", blank=True, help_text="URL this task is imported from (only for imported tasks)") + def __init__(self, *args, **kwargs): super(Task, self).__init__(*args, **kwargs) @@ -333,6 +337,14 @@ def get_asset_download_path(self, asset): else: raise FileNotFoundError("{} is not a valid asset".format(asset)) + def handle_import(self): + self.console_output += "Importing assets...\n" + self.save() + self.extract_assets_and_complete() + self.pending_action = None + self.processing_time = 0 + self.save() + def process(self): """ This method contains the logic for processing tasks asynchronously @@ -342,6 +354,9 @@ def process(self): """ try: + if self.pending_action == pending_actions.IMPORT: + self.handle_import() + if self.pending_action == pending_actions.RESIZE: resized_images = self.resize_images() self.refresh_from_db() @@ -571,42 +586,7 @@ def callback(progress): self.console_output += "Extracting results. This could take a few minutes...\n"; self.save() - # Extract from zip - with zipfile.ZipFile(zip_path, "r") as zip_h: - zip_h.extractall(assets_dir) - - logger.info("Extracted all.zip for {}".format(self)) - - # Populate *_extent fields - extent_fields = [ - (os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")), - 'orthophoto_extent'), - (os.path.realpath(self.assets_path("odm_dem", "dsm.tif")), - 'dsm_extent'), - (os.path.realpath(self.assets_path("odm_dem", "dtm.tif")), - 'dtm_extent'), - ] - - for raster_path, field in extent_fields: - if os.path.exists(raster_path): - # Read extent and SRID - raster = GDALRaster(raster_path) - extent = OGRGeometry.from_bbox(raster.extent) - - # It will be implicitly transformed into the SRID of the model’s field - # self.field = GEOSGeometry(...) - setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid)) - - logger.info("Populated extent field with {} for {}".format(raster_path, self)) - - self.update_available_assets_field() - self.running_progress = 1.0 - self.console_output += "Done!\n" - self.status = status_codes.COMPLETED - self.save() - - from app.plugins import signals as plugin_signals - plugin_signals.task_completed.send_robust(sender=self.__class__, task_id=self.id) + self.extract_assets_and_complete() else: # FAILED, CANCELED self.save() @@ -624,6 +604,51 @@ def callback(progress): # Task was interrupted during image resize / upload logger.warning("{} interrupted".format(self, str(e))) + def extract_assets_and_complete(self): + """ + Extracts assets/all.zip and populates task fields where required. + :return: + """ + assets_dir = self.assets_path("") + zip_path = self.assets_path("all.zip") + + # Extract from zip + with zipfile.ZipFile(zip_path, "r") as zip_h: + zip_h.extractall(assets_dir) + + logger.info("Extracted all.zip for {}".format(self)) + + # Populate *_extent fields + extent_fields = [ + (os.path.realpath(self.assets_path("odm_orthophoto", "odm_orthophoto.tif")), + 'orthophoto_extent'), + (os.path.realpath(self.assets_path("odm_dem", "dsm.tif")), + 'dsm_extent'), + (os.path.realpath(self.assets_path("odm_dem", "dtm.tif")), + 'dtm_extent'), + ] + + for raster_path, field in extent_fields: + if os.path.exists(raster_path): + # Read extent and SRID + raster = GDALRaster(raster_path) + extent = OGRGeometry.from_bbox(raster.extent) + + # It will be implicitly transformed into the SRID of the model’s field + # self.field = GEOSGeometry(...) + setattr(self, field, GEOSGeometry(extent.wkt, srid=raster.srid)) + + logger.info("Populated extent field with {} for {}".format(raster_path, self)) + + self.update_available_assets_field() + self.running_progress = 1.0 + self.console_output += "Done!\n" + self.status = status_codes.COMPLETED + self.save() + + from app.plugins import signals as plugin_signals + plugin_signals.task_completed.send_robust(sender=self.__class__, task_id=self.id) + def get_tile_path(self, tile_type, z, x, y): return self.assets_path("{}_tiles".format(tile_type), z, x, "{}.png".format(y)) @@ -783,3 +808,16 @@ def resize_gcp(self, resized_images): except subprocess.CalledProcessError as e: logger.warning("Could not resize GCP file {}: {}".format(gcp_path, str(e))) return None + + def create_task_directories(self): + """ + Create directories for this task (if they don't exist already) + """ + assets_dir = self.assets_path("") + try: + os.makedirs(assets_dir) + except OSError as exc: # Python >2.5 + if exc.errno == errno.EEXIST and os.path.isdir(assets_dir): + pass + else: + raise diff --git a/app/pending_actions.py b/app/pending_actions.py index 79c0cc6bb..c7f025329 100644 --- a/app/pending_actions.py +++ b/app/pending_actions.py @@ -2,3 +2,4 @@ REMOVE = 2 RESTART = 3 RESIZE = 4 +IMPORT = 5 diff --git a/app/static/app/js/classes/PendingActions.js b/app/static/app/js/classes/PendingActions.js index b69f17283..69f3c9a8d 100644 --- a/app/static/app/js/classes/PendingActions.js +++ b/app/static/app/js/classes/PendingActions.js @@ -1,7 +1,8 @@ const CANCEL = 1, REMOVE = 2, RESTART = 3, - RESIZE = 4; + RESIZE = 4, + IMPORT = 5; let pendingActions = { [CANCEL]: { @@ -15,6 +16,9 @@ let pendingActions = { }, [RESIZE]: { descr: "Resizing images..." + }, + [IMPORT]: { + descr: "Importing..." } }; @@ -23,6 +27,7 @@ export default { REMOVE: REMOVE, RESTART: RESTART, RESIZE: RESIZE, + IMPORT: IMPORT, description: function(pendingAction) { if (pendingActions[pendingAction]) return pendingActions[pendingAction].descr; diff --git a/app/static/app/js/components/ImportTaskPanel.jsx b/app/static/app/js/components/ImportTaskPanel.jsx new file mode 100644 index 000000000..d39094e63 --- /dev/null +++ b/app/static/app/js/components/ImportTaskPanel.jsx @@ -0,0 +1,96 @@ +import '../css/ImportTaskPanel.scss'; +import React from 'react'; +import PropTypes from 'prop-types'; +import Dropzone from '../vendor/dropzone'; +import csrf from '../django/csrf'; + +class ImportTaskPanel extends React.Component { + static defaultProps = { + }; + + static propTypes = { + // onSave: PropTypes.func.isRequired, + onCancel: PropTypes.func, + projectId: PropTypes.number.isRequired + }; + + constructor(props){ + super(props); + + this.state = { + }; + } + + componentDidMount(){ + Dropzone.autoDiscover = false; + + this.dz = new Dropzone(this.dropzone, { + paramName: "file", + url : `/api/projects/${this.props.projectId}/tasks/import`, + parallelUploads: 1, + uploadMultiple: false, + acceptedFiles: "application/zip", + autoProcessQueue: true, + createImageThumbnails: false, + previewTemplate: '
', + clickable: this.uploadButton, + chunkSize: 2147483647, + timeout: 2147483647, + + headers: { + [csrf.header]: csrf.token + } + }); + + this.dz.on("error", function(file){ + // Show + }) + .on("uploadprogress", function(file, progress){ + console.log(progress); + }) + .on("complete", function(file){ + if (file.status === "success"){ + }else{ + // error + } + }); + } + + cancel = (e) => { + this.props.onCancel(); + } + + setRef = (prop) => { + return (domNode) => { + if (domNode != null) this[prop] = domNode; + } + } + + render() { + return ( +
+
+ +

Import Existing Assets

+

You can import .zip files that have been exported from existing tasks via Download Assets All Assets.

+ + +
+
+ ); + } +} + +export default ImportTaskPanel; diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 2ee1a5a6b..15fece741 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -3,6 +3,7 @@ import React from 'react'; import update from 'immutability-helper'; import TaskList from './TaskList'; import NewTaskPanel from './NewTaskPanel'; +import ImportTaskPanel from './ImportTaskPanel'; import UploadProgressBar from './UploadProgressBar'; import ProgressBar from './ProgressBar'; import ErrorMessage from './ErrorMessage'; @@ -32,7 +33,8 @@ class ProjectListItem extends React.Component { upload: this.getDefaultUploadState(), error: "", data: props.data, - refreshing: false + refreshing: false, + importing: false }; this.toggleTaskList = this.toggleTaskList.bind(this); @@ -335,6 +337,14 @@ class ProjectListItem extends React.Component { location.href = `/map/project/${this.state.data.id}/`; } + handleImportTask = () => { + this.setState({importing: true}); + } + + handleCancelImportTask = () => { + this.setState({importing: false}); + } + render() { const { refreshing, data } = this.state; const numTasks = data.tasks.length; @@ -361,13 +371,17 @@ class ProjectListItem extends React.Component {
{this.hasPermission("add") ? - + +
: ""}

Import Existing Assets

You can import .zip files that have been exported from existing tasks via Download Assets All Assets.

- - + + {this.state.typeUrl ? +
+
+ + +
+
: ""} + + {this.state.uploading ?
+ + +
: ""}
); diff --git a/app/static/app/js/components/ProjectListItem.jsx b/app/static/app/js/components/ProjectListItem.jsx index 15fece741..cabce614c 100644 --- a/app/static/app/js/components/ProjectListItem.jsx +++ b/app/static/app/js/components/ProjectListItem.jsx @@ -204,13 +204,7 @@ class ProjectListItem extends React.Component { let response = JSON.parse(files[0].xhr.response); if (!response.id) throw new Error(`Expected id field, but none given (${response})`); - if (this.state.showTaskList){ - this.taskList.refresh(); - }else{ - this.setState({showTaskList: true}); - } - this.resetUploadState(); - this.refresh(); + this.newTaskAdded(); }catch(e){ this.setUploadState({error: `Invalid response from server: ${e.message}`, uploading: false}) } @@ -247,6 +241,18 @@ class ProjectListItem extends React.Component { } } + newTaskAdded = () => { + this.setState({importing: false}); + + if (this.state.showTaskList){ + this.taskList.refresh(); + }else{ + this.setState({showTaskList: true}); + } + this.resetUploadState(); + this.refresh(); + } + setRef(prop){ return (domNode) => { if (domNode != null) this[prop] = domNode; @@ -448,6 +454,7 @@ class ProjectListItem extends React.Component { {this.state.importing ? diff --git a/app/static/app/js/components/TaskListItem.jsx b/app/static/app/js/components/TaskListItem.jsx index 095c06bbf..24f29aa2e 100644 --- a/app/static/app/js/components/TaskListItem.jsx +++ b/app/static/app/js/components/TaskListItem.jsx @@ -406,12 +406,13 @@ class TaskListItem extends React.Component { } if ([statusCodes.QUEUED, statusCodes.RUNNING, null].indexOf(task.status) !== -1 && - task.processing_node){ + (task.processing_node || imported)){ addActionButton("Cancel", "btn-primary", "glyphicon glyphicon-remove-circle", this.genActionApiCall("cancel", {defaultError: "Cannot cancel task."})); } if ([statusCodes.FAILED, statusCodes.COMPLETED, statusCodes.CANCELED].indexOf(task.status) !== -1 && - task.processing_node){ + task.processing_node && + !imported){ // By default restart reruns every pipeline // step from the beginning const rerunFrom = task.can_rerun_from.length > 1 ? diff --git a/app/static/app/js/css/ImportTaskPanel.scss b/app/static/app/js/css/ImportTaskPanel.scss index a80472664..bbcc68c58 100644 --- a/app/static/app/js/css/ImportTaskPanel.scss +++ b/app/static/app/js/css/ImportTaskPanel.scss @@ -17,4 +17,13 @@ .close:hover, .close:focus{ color: inherit; } + + .upload-progress-bar{ + margin-top: 12px; + } + + .btn-import{ + margin-top: 8px; + margin-left: 8px; + } } \ No newline at end of file diff --git a/app/static/app/js/css/TaskListItem.scss b/app/static/app/js/css/TaskListItem.scss index d712c97df..a30647594 100644 --- a/app/static/app/js/css/TaskListItem.scss +++ b/app/static/app/js/css/TaskListItem.scss @@ -15,6 +15,7 @@ .name{ padding-left: 0; + margin-top: 4px; } .details{ From fcd206cf424ac8b90e150ca967771a7e4a3dc73e Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 21 Feb 2019 15:28:20 -0500 Subject: [PATCH 11/12] Front-end unit test fix --- .../app/js/components/ImportTaskPanel.jsx | 111 +++++++++--------- .../components/tests/ImportTaskPanel.test.jsx | 2 +- 2 files changed, 57 insertions(+), 56 deletions(-) diff --git a/app/static/app/js/components/ImportTaskPanel.jsx b/app/static/app/js/components/ImportTaskPanel.jsx index b7b953f7b..7800045e8 100644 --- a/app/static/app/js/components/ImportTaskPanel.jsx +++ b/app/static/app/js/components/ImportTaskPanel.jsx @@ -37,59 +37,61 @@ class ImportTaskPanel extends React.Component { componentDidMount(){ Dropzone.autoDiscover = false; - this.dz = new Dropzone(this.dropzone, { - paramName: "file", - url : `/api/projects/${this.props.projectId}/tasks/import`, - parallelUploads: 1, - uploadMultiple: false, - acceptedFiles: "application/zip", - autoProcessQueue: true, - createImageThumbnails: false, - previewTemplate: '
', - clickable: this.uploadButton, - chunkSize: 2147483647, - timeout: 2147483647, - - headers: { - [csrf.header]: csrf.token - } - }); - - this.dz.on("error", (file) => { - if (this.state.uploading) this.setState({error: "Cannot upload file. Check your internet connection and try again."}); - }) - .on("sending", () => { - this.setState({typeUrl: false, uploading: true, totalCount: 1}); - }) - .on("reset", () => { - this.setState({uploading: false, progress: 0, totalBytes: 0, totalBytesSent: 0}); - }) - .on("uploadprogress", (file, progress, bytesSent) => { - this.setState({ - progress, - totalBytes: file.size, - totalBytesSent: bytesSent - }); - }) - .on("sending", (file, xhr, formData) => { - // Safari does not have support for has on FormData - // as of December 2017 - if (!formData.has || !formData.has("name")) formData.append("name", this.defaultTaskName()); - }) - .on("complete", (file) => { - if (file.status === "success"){ - this.setState({uploading: false}); - try{ - let response = JSON.parse(file.xhr.response); - if (!response.id) throw new Error(`Expected id field, but none given (${response})`); - this.props.onImported(); - }catch(e){ - this.setState({error: `Invalid response from server: ${e.message}`}); + if (this.dropzone){ + this.dz = new Dropzone(this.dropzone, { + paramName: "file", + url : `/api/projects/${this.props.projectId}/tasks/import`, + parallelUploads: 1, + uploadMultiple: false, + acceptedFiles: "application/zip", + autoProcessQueue: true, + createImageThumbnails: false, + previewTemplate: '
', + clickable: this.uploadButton, + chunkSize: 2147483647, + timeout: 2147483647, + + headers: { + [csrf.header]: csrf.token } - }else if (this.state.uploading){ - this.setState({uploading: false, error: "An error occured while uploading the file. Please try again."}); - } }); + + this.dz.on("error", (file) => { + if (this.state.uploading) this.setState({error: "Cannot upload file. Check your internet connection and try again."}); + }) + .on("sending", () => { + this.setState({typeUrl: false, uploading: true, totalCount: 1}); + }) + .on("reset", () => { + this.setState({uploading: false, progress: 0, totalBytes: 0, totalBytesSent: 0}); + }) + .on("uploadprogress", (file, progress, bytesSent) => { + this.setState({ + progress, + totalBytes: file.size, + totalBytesSent: bytesSent + }); + }) + .on("sending", (file, xhr, formData) => { + // Safari does not have support for has on FormData + // as of December 2017 + if (!formData.has || !formData.has("name")) formData.append("name", this.defaultTaskName()); + }) + .on("complete", (file) => { + if (file.status === "success"){ + this.setState({uploading: false}); + try{ + let response = JSON.parse(file.xhr.response); + if (!response.id) throw new Error(`Expected id field, but none given (${response})`); + this.props.onImported(); + }catch(e){ + this.setState({error: `Invalid response from server: ${e.message}`}); + } + }else if (this.state.uploading){ + this.setState({uploading: false, error: "An error occured while uploading the file. Please try again."}); + } + }); + } } cancel = (e) => { @@ -125,6 +127,8 @@ class ImportTaskPanel extends React.Component { name: this.defaultTaskName() } ).done(json => { + this.setState({importingFromUrl: false}); + if (json.id){ this.props.onImported(); }else{ @@ -132,10 +136,7 @@ class ImportTaskPanel extends React.Component { } }) .fail(() => { - this.setState({error: "Cannot import from URL. Check your internet connection."}); - }) - .always(() => { - this.setState({importingFromUrl: false}); + this.setState({importingFromUrl: false, error: "Cannot import from URL. Check your internet connection."}); }); } diff --git a/app/static/app/js/components/tests/ImportTaskPanel.test.jsx b/app/static/app/js/components/tests/ImportTaskPanel.test.jsx index 1e66b93df..29bc24fe8 100644 --- a/app/static/app/js/components/tests/ImportTaskPanel.test.jsx +++ b/app/static/app/js/components/tests/ImportTaskPanel.test.jsx @@ -4,7 +4,7 @@ import ImportTaskPanel from '../ImportTaskPanel'; describe('', () => { it('renders without exploding', () => { - const wrapper = shallow(); + const wrapper = shallow( {}} />); expect(wrapper.exists()).toBe(true); }) }); \ No newline at end of file From cf7e76726c05a844e3186d58c610b9277772a2b3 Mon Sep 17 00:00:00 2001 From: Piero Toffanin Date: Thu, 21 Feb 2019 16:58:27 -0500 Subject: [PATCH 12/12] Unit tests for import feature --- app/models/task.py | 3 +- app/tests/test_api_task.py | 1 + app/tests/test_api_task_import.py | 168 ++++++++++++++++++++++++++++++ 3 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 app/tests/test_api_task_import.py diff --git a/app/models/task.py b/app/models/task.py index e05c7e1bd..cf31db68b 100644 --- a/app/models/task.py +++ b/app/models/task.py @@ -348,6 +348,7 @@ def handle_import(self): # TODO: this is potentially vulnerable to a zip bomb attack # mitigated by the fact that a valid account is needed to # import tasks + logger.info("Importing task assets from {} for {}".format(self.import_url, self)) download_stream = requests.get(self.import_url, stream=True, timeout=10) content_length = download_stream.headers.get('content-length') total_length = int(content_length) if content_length is not None else None @@ -657,7 +658,7 @@ def extract_assets_and_complete(self): with zipfile.ZipFile(zip_path, "r") as zip_h: zip_h.extractall(assets_dir) except zipfile.BadZipFile: - raise NodeServerError("Corrupted zip file") + raise NodeServerError("Invalid zip file") logger.info("Extracted all.zip for {}".format(self)) diff --git a/app/tests/test_api_task.py b/app/tests/test_api_task.py index f5e171ea7..7a93bd581 100644 --- a/app/tests/test_api_task.py +++ b/app/tests/test_api_task.py @@ -122,6 +122,7 @@ def test_task(self): multiple_param_task = Task.objects.latest('created_at') self.assertTrue(multiple_param_task.name == 'test_task') self.assertTrue(multiple_param_task.processing_node.id == pnode.id) + self.assertEqual(multiple_param_task.import_url, "") image1.seek(0) image2.seek(0) diff --git a/app/tests/test_api_task_import.py b/app/tests/test_api_task_import.py new file mode 100644 index 000000000..2db6bd5b6 --- /dev/null +++ b/app/tests/test_api_task_import.py @@ -0,0 +1,168 @@ +import os +import time + +import io +import requests +from django.contrib.auth.models import User +from guardian.shortcuts import remove_perm, assign_perm +from rest_framework import status +from rest_framework.test import APIClient + +import worker +from app.models import Project +from app.models import Task +from app.tests.classes import BootTransactionTestCase +from app.tests.utils import clear_test_media_root, start_processing_node +from nodeodm import status_codes +from nodeodm.models import ProcessingNode +from webodm import settings + + +class TestApiTask(BootTransactionTestCase): + def setUp(self): + super().setUp() + clear_test_media_root() + + def test_task(self): + client = APIClient() + + node_odm = start_processing_node() + + user = User.objects.get(username="testuser") + self.assertFalse(user.is_superuser) + project = Project.objects.create( + owner=user, + name="test project" + ) + + image1 = open("app/fixtures/tiny_drone_image.jpg", 'rb') + image2 = open("app/fixtures/tiny_drone_image_2.jpg", 'rb') + + # Create processing node + pnode = ProcessingNode.objects.create(hostname="localhost", port=11223) + client.login(username="testuser", password="test1234") + + # Create task + res = client.post("/api/projects/{}/tasks/".format(project.id), { + 'images': [image1, image2] + }, format="multipart") + image1.close() + image2.close() + task = Task.objects.get(id=res.data['id']) + + # Wait for completion + c = 0 + while c < 10: + worker.tasks.process_pending_tasks() + task.refresh_from_db() + if task.status == status_codes.COMPLETED: + break + c += 1 + time.sleep(1) + + + self.assertEqual(task.status, status_codes.COMPLETED) + + # Download task assets + task_uuid = task.uuid + res = client.get("/api/projects/{}/tasks/{}/download/all.zip".format(project.id, task.id)) + self.assertEqual(res.status_code, status.HTTP_200_OK) + + if not os.path.exists(settings.MEDIA_TMP): + os.mkdir(settings.MEDIA_TMP) + + assets_path = os.path.join(settings.MEDIA_TMP, "all.zip") + + with open(assets_path, 'wb') as f: + f.write(res.content) + + remove_perm('change_project', user, project) + + assets_file = open(assets_path, 'rb') + + # Cannot import unless we have permission + res = client.post("/api/projects/{}/tasks/import".format(project.id), { + 'file': [assets_file] + }, format="multipart") + self.assertEqual(res.status_code, status.HTTP_404_NOT_FOUND) + + assign_perm('change_project', user, project) + + # Import with file upload method + assets_file.seek(0) + res = client.post("/api/projects/{}/tasks/import".format(project.id), { + 'file': [assets_file] + }, format="multipart") + self.assertEqual(res.status_code, status.HTTP_201_CREATED) + assets_file.close() + + file_import_task = Task.objects.get(id=res.data['id']) + # Wait for completion + c = 0 + while c < 10: + worker.tasks.process_pending_tasks() + file_import_task.refresh_from_db() + if file_import_task.status == status_codes.COMPLETED: + break + c += 1 + time.sleep(1) + + self.assertEqual(file_import_task.import_url, "file://all.zip") + self.assertEqual(file_import_task.images_count, 1) + self.assertEqual(file_import_task.processing_node, None) + self.assertEqual(file_import_task.auto_processing_node, False) + + # Can access assets + res = client.get("/api/projects/{}/tasks/{}/assets/odm_orthophoto/odm_orthophoto.tif".format(project.id, file_import_task.id)) + self.assertEqual(res.status_code, status.HTTP_200_OK) + + # Set task public so we can download from it without auth + file_import_task.public = True + file_import_task.save() + + # Import with URL method + assets_import_url = "http://{}:{}/task/{}/download/all.zip".format(pnode.hostname, pnode.port, task_uuid) + res = client.post("/api/projects/{}/tasks/import".format(project.id), { + 'url': assets_import_url + }) + self.assertEqual(res.status_code, status.HTTP_201_CREATED) + url_task = Task.objects.get(id=res.data['id']) + + # Wait for completion + c = 0 + while c < 10: + worker.tasks.process_pending_tasks() + url_task.refresh_from_db() + if url_task.status == status_codes.COMPLETED: + break + c += 1 + time.sleep(1) + + self.assertEqual(url_task.import_url, assets_import_url) + self.assertEqual(url_task.images_count, 1) + + # Import corrupted file + assets_import_url = "http://{}:{}/task/{}/download/orthophoto.tif".format(pnode.hostname, pnode.port, task_uuid) + res = client.post("/api/projects/{}/tasks/import".format(project.id), { + 'url': assets_import_url + }) + self.assertEqual(res.status_code, status.HTTP_201_CREATED) + corrupted_task = Task.objects.get(id=res.data['id']) + + # Wait for completion + c = 0 + while c < 10: + worker.tasks.process_pending_tasks() + corrupted_task.refresh_from_db() + if corrupted_task.status == status_codes.FAILED: + break + c += 1 + time.sleep(1) + + self.assertEqual(corrupted_task.status, status_codes.FAILED) + self.assertTrue("Invalid" in corrupted_task.last_error) + + # Stop processing node + node_odm.terminate() + +