Merge pull request #2 from mentorpal/mentor-upload-api

Add mentor_upload_api with /ping and /upload/answer endpoint

Makefile

@@ -25,17 +25,17 @@ LICENSE_HEADER:
 .PHONY: license
 license: LICENSE LICENSE_HEADER $(VENV)
 	. $(VENV)/bin/activate \
-		&& python -m licenseheaders -t LICENSE_HEADER -d opentutor_classifier/src $(args) \
-		&& python -m licenseheaders -t LICENSE_HEADER -d opentutor_classifier/tests $(args) \
-		&& python -m licenseheaders -t LICENSE_HEADER -d opentutor_classifier_api/src $(args) \
-		&& python -m licenseheaders -t LICENSE_HEADER -d opentutor_classifier_api/tests $(args) \
+		&& python -m licenseheaders -t LICENSE_HEADER -d mentor_upload_worker/src $(args) \
+		&& python -m licenseheaders -t LICENSE_HEADER -d mentor_upload_worker/tests $(args) \
+		&& python -m licenseheaders -t LICENSE_HEADER -d mentor_upload_api/src $(args) \
+		&& python -m licenseheaders -t LICENSE_HEADER -d mentor_upload_api/tests $(args) \
 		&& python -m licenseheaders -t LICENSE_HEADER -d tools $(args) \
 		&& python -m licenseheaders -t LICENSE_HEADER -d word2vec $(args)
 
 .PHONY: test
 test:
-	cd opentutor_classifier && $(MAKE) test
-	cd opentutor_classifier_api && $(MAKE) test
+	cd mentor_upload_worker && $(MAKE) test
+	cd mentor_upload_api && $(MAKE) test
 
 .PHONY: test-all
 test-all:

mentor_upload_api/.coveragerc

@@ -0,0 +1,11 @@
+[run]
+omit = */tests/*,*/tests.py,test*.py
+source = ./src
+dynamic_context = test_function
+branch = True
+
+[report]
+skip_empty = True
+
+[html]
+show_contexts = True

mentor_upload_api/.dockerignore

@@ -0,0 +1,7 @@
+.cov*
+.pytest_cache
+.venv
+htmlcov
+tests
+.mypy_cache
+.vscode

mentor_upload_api/Dockerfile

@@ -0,0 +1,10 @@
+FROM python:3.8-slim
+ENV STATUS_URL_FORCE_HTTPS=false
+ADD requirements.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt
+RUN rm /tmp/requirements.txt
+ENV FLASK_APP=mentor_upload_api
+WORKDIR /app
+COPY src .
+RUN chmod +x /app/entrypoint.sh
+ENTRYPOINT ["/app/entrypoint.sh"]

mentor_upload_api/Makefile

@@ -0,0 +1,77 @@
+DOCKER_IMAGE?=mentor-upload-api
+DOCKER_IMAGE_ID=$(shell docker images -q ${DOCKER_IMAGE} 2> /dev/null)
+ROOT=$(shell dirname ${PWD})
+VENV=.venv
+VENV_PIP=$(VENV)/bin/pip
+$(VENV):
+	$(MAKE) $(VENV)-update
+
+.PHONY $(VENV)-installed:
+$(VENV)-installed:
+	$(ROOT)/tools/virtualenv_ensure_installed.sh
+
+.PHONY: $(VENV)-update
+$(VENV)-update: $(VENV)-installed
+	[ -d $(VENV) ] || virtualenv -p python3.8 $(VENV)
+	$(VENV_PIP) install --upgrade pip
+	$(VENV_PIP) install -r requirements.test.txt
+
+.PHONY clean:
+clean:
+	rm -rf .venv .pytest_cache .mypy_cache build
+
+.PHONY docker-build:
+docker-build: clean
+	docker build -t $(DOCKER_IMAGE) .
+
+.PHONY docker-build:
+docker-run:
+	docker run \
+		-it \
+		--rm \
+		-p 5000:5000 \
+	$(DOCKER_IMAGE)
+
+.PHONY: format
+format: $(VENV)
+	cd $(ROOT) && $(MAKE) format
+
+.PHONY: license
+license:
+	cd $(ROOT) && make license
+
+.PHONY: test
+test: $(VENV)
+	. $(VENV)/bin/activate \
+		&& export PYTHONPATH=$${PYTHONPATH}:$(PWD)/src \
+		&& coverage run \
+			--omit="$(PWD)/tests $(VENV)" \
+			-m py.test -vv $(args)
+
+.PHONY: test-all
+test-all:
+	$(MAKE) test-format
+	$(MAKE) test-lint
+	$(MAKE) test-license
+	$(MAKE) test-types
+	$(MAKE) test
+
+.PHONY: test-format
+test-format: $(VENV)
+	cd $(ROOT) && $(MAKE) test-format
+
+.PHONY: test-license
+test-license:
+	cd $(ROOT) && make test-license
+
+.PHONY: test-lint
+test-lint: $(VENV)
+	cd $(ROOT) && $(MAKE) test-lint
+
+.PHONY: test-types
+test-types: $(VENV)
+	cd $(ROOT) && $(MAKE) test-types
+
+.PHONY: update-deps
+update-deps: $(VENV)
+	. $(VENV)/bin/activate && pip-upgrade requirements*

mentor_upload_api/README.md

@@ -0,0 +1,18 @@
+# mentor-upload-api
+dockerized REST api for the mentor upload
+
+## Licensing
+
+All source code files must include a USC open license header.
+
+To check if files have a license header:
+
+```
+make test-license
+```
+
+To add license headers:
+
+```
+make license
+```

mentor_upload_api/mypy.ini

@@ -0,0 +1,14 @@
+[mypy]
+python_version = 3.8
+
+[mypy-celery.*]
+ignore_missing_imports = True
+
+[mypy-cerberus.*]
+ignore_missing_imports = True
+
+[mypy-flask_cors.*]
+ignore_missing_imports = True
+
+[mypy-pytest.*]
+ignore_missing_imports = True

mentor_upload_api/pytest.ini

@@ -0,0 +1,9 @@
+[pytest]
+norecursedirs=
+    .venv
+    .pytest_cache
+    bin
+    build
+    htmlcov
+addopts=
+    --durations=10

mentor_upload_api/requirements.test.txt

@@ -0,0 +1,9 @@
+-r requirements.txt
+coverage==5.3
+pip-upgrader
+pytest==6.1.1
+pytest-mock==3.3.1
+pytest-env==0.6.2
+pytest-flask==1.0.0
+requests==2.24.0
+responses==0.12.1

mentor_upload_api/requirements.txt

@@ -0,0 +1,7 @@
+celery==5.0.0
+cerberus==1.3.2
+flask==1.1.2
+Flask-Cors==3.0.9
+gunicorn==20.0.4
+redis==3.5.3
+werkzeug==1.0.1

mentor_upload_api/src/entrypoint.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+##
+## This software is Copyright ©️ 2020 The University of Southern California. All Rights Reserved. 
+## Permission to use, copy, modify, and distribute this software and its documentation for educational, research and non-profit purposes, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and subject to the full license file found in the root of this software deliverable. Permission to make commercial use of this software may be obtained by contacting:  USC Stevens Center for Innovation University of Southern California 1150 S. Olive Street, Suite 2300, Los Angeles, CA 90115, USA Email: [email protected]
+##
+## The full terms of this copyright and license should always be found in the root directory of this software deliverable as "license.txt" and if these terms are not found with this software, please contact the USC Stevens Center for the full license.
+##
+export FLASK_APP=/app/mentor_upload_api
+cd /app && gunicorn -b 0.0.0.0:5000 manage:app
+# -h 0.0.0.0
+#   needs ip set or will be unreachable from host
+#   regardless of docker-run port mappings
+exit 0

mentor_upload_api/src/manage.py

@@ -0,0 +1,9 @@
+#
+# This software is Copyright ©️ 2020 The University of Southern California. All Rights Reserved.
+# Permission to use, copy, modify, and distribute this software and its documentation for educational, research and non-profit purposes, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and subject to the full license file found in the root of this software deliverable. Permission to make commercial use of this software may be obtained by contacting:  USC Stevens Center for Innovation University of Southern California 1150 S. Olive Street, Suite 2300, Los Angeles, CA 90115, USA Email: [email protected]
+#
+# The full terms of this copyright and license should always be found in the root directory of this software deliverable as "license.txt" and if these terms are not found with this software, please contact the USC Stevens Center for the full license.
+#
+from mentor_upload_api import create_app
+
+app = create_app()

mentor_upload_api/src/mentor_upload_api/__init__.py

@@ -0,0 +1,40 @@
+#
+# This software is Copyright ©️ 2020 The University of Southern California. All Rights Reserved.
+# Permission to use, copy, modify, and distribute this software and its documentation for educational, research and non-profit purposes, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and subject to the full license file found in the root of this software deliverable. Permission to make commercial use of this software may be obtained by contacting:  USC Stevens Center for Innovation University of Southern California 1150 S. Olive Street, Suite 2300, Los Angeles, CA 90115, USA Email: [email protected]
+#
+# The full terms of this copyright and license should always be found in the root directory of this software deliverable as "license.txt" and if these terms are not found with this software, please contact the USC Stevens Center for the full license.
+#
+from logging.config import dictConfig
+
+from flask import Flask
+from flask_cors import CORS
+
+
+def create_app():
+    dictConfig(
+        {
+            "version": 1,
+            "formatters": {
+                "default": {
+                    "format": "[%(asctime)s] %(levelname)s in %(module)s: %(message)s"
+                }
+            },
+            "handlers": {
+                "wsgi": {
+                    "class": "logging.StreamHandler",
+                    "stream": "ext://flask.logging.wsgi_errors_stream",
+                    "formatter": "default",
+                }
+            },
+            "root": {"level": "INFO", "handlers": ["wsgi"]},
+        }
+    )
+    app = Flask(__name__)
+    CORS(app)
+    from mentor_upload_api.blueprints.ping import ping_blueprint
+
+    app.register_blueprint(ping_blueprint, url_prefix="/upload/ping")
+    from mentor_upload_api.blueprints.upload import upload_blueprint
+
+    app.register_blueprint(upload_blueprint, url_prefix="/upload/answer")
+    return app

mentor_upload_api/src/mentor_upload_api/blueprints/__init__.py

@@ -0,0 +1,6 @@
+#
+# This software is Copyright ©️ 2020 The University of Southern California. All Rights Reserved.
+# Permission to use, copy, modify, and distribute this software and its documentation for educational, research and non-profit purposes, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and subject to the full license file found in the root of this software deliverable. Permission to make commercial use of this software may be obtained by contacting:  USC Stevens Center for Innovation University of Southern California 1150 S. Olive Street, Suite 2300, Los Angeles, CA 90115, USA Email: [email protected]
+#
+# The full terms of this copyright and license should always be found in the root directory of this software deliverable as "license.txt" and if these terms are not found with this software, please contact the USC Stevens Center for the full license.
+#

mentor_upload_api/src/mentor_upload_api/blueprints/ping.py

@@ -0,0 +1,19 @@
+#
+# This software is Copyright ©️ 2020 The University of Southern California. All Rights Reserved.
+# Permission to use, copy, modify, and distribute this software and its documentation for educational, research and non-profit purposes, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and subject to the full license file found in the root of this software deliverable. Permission to make commercial use of this software may be obtained by contacting:  USC Stevens Center for Innovation University of Southern California 1150 S. Olive Street, Suite 2300, Los Angeles, CA 90115, USA Email: [email protected]
+#
+# The full terms of this copyright and license should always be found in the root directory of this software deliverable as "license.txt" and if these terms are not found with this software, please contact the USC Stevens Center for the full license.
+#
+import os
+
+from flask import Blueprint, jsonify
+
+ping_blueprint = Blueprint("ping", __name__)
+
+
+@ping_blueprint.route("", methods=["GET"])
+@ping_blueprint.route("/", methods=["GET"])
+def ping():
+    return jsonify(
+        {"status": "success", "message": "pong!", "container_id": os.uname()[1]}
+    )

mentor_upload_api/src/mentor_upload_api/blueprints/upload.py

@@ -0,0 +1,67 @@
+#
+# This software is Copyright ©️ 2020 The University of Southern California. All Rights Reserved.
+# Permission to use, copy, modify, and distribute this software and its documentation for educational, research and non-profit purposes, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and subject to the full license file found in the root of this software deliverable. Permission to make commercial use of this software may be obtained by contacting:  USC Stevens Center for Innovation University of Southern California 1150 S. Olive Street, Suite 2300, Los Angeles, CA 90115, USA Email: [email protected]
+#
+# The full terms of this copyright and license should always be found in the root directory of this software deliverable as "license.txt" and if these terms are not found with this software, please contact the USC Stevens Center for the full license.
+#
+from os import environ, path, makedirs
+from flask import Blueprint, jsonify, request
+import uuid
+
+import mentor_upload_tasks
+import mentor_upload_tasks.tasks
+
+upload_blueprint = Blueprint("answer", __name__)
+
+
+def _to_status_url(root: str, id: str) -> str:
+    return f"{request.url_root.replace('http://', 'https://', 1) if (environ.get('STATUS_URL_FORCE_HTTPS') or '').lower() in ('1', 'y', 'true', 'on') and str.startswith(request.url_root,'http://') else request.url_root}upload/answer/status/{id}"
+
+
+def get_upload_root() -> str:
+    return environ.get("UPLOAD_ROOT") or "uploads"
+
+
+@upload_blueprint.route("/", methods=["POST"])
+@upload_blueprint.route("", methods=["POST"])
+def upload():
+    mentor = request.form.get("mentor")
+    question = request.form.get("question")
+    upload_file = request.files["video"]
+
+    root_ext = path.splitext(upload_file.filename)
+    file_name = f"{uuid.uuid4()}-{mentor}-{question}{root_ext[1]}"
+    file_path = path.join(get_upload_root(), file_name)
+    makedirs(get_upload_root(), exist_ok=True)
+    upload_file.save(file_path)
+
+    req = {"mentor": mentor, "question": question, "video_path": file_path}
+    t = mentor_upload_tasks.tasks.upload_task.apply_async(req)
+    return jsonify(
+        {
+            "data": {
+                "id": t.id,
+                "statusUrl": _to_status_url(request.url_root, t.id),
+            }
+        }
+    )
+
+
+@upload_blueprint.route("/status/<task_id>/", methods=["GET"])
+@upload_blueprint.route("/status/<task_id>", methods=["GET"])
+def upload_status(task_id: str):
+    t = mentor_upload_tasks.tasks.upload_task.AsyncResult(task_id)
+    return jsonify(
+        {
+            "data": {
+                "id": task_id,
+                "state": t.state or "NONE",
+                "status": t.status,
+                "info": None
+                if not t.info
+                else t.info
+                if isinstance(t.info, dict) or isinstance(t.info, list)
+                else str(t.info),
+            }
+        }
+    )

mentor_upload_api/src/mentor_upload_tasks/__init__.py

@@ -0,0 +1,6 @@
+#
+# This software is Copyright ©️ 2020 The University of Southern California. All Rights Reserved.
+# Permission to use, copy, modify, and distribute this software and its documentation for educational, research and non-profit purposes, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and subject to the full license file found in the root of this software deliverable. Permission to make commercial use of this software may be obtained by contacting:  USC Stevens Center for Innovation University of Southern California 1150 S. Olive Street, Suite 2300, Los Angeles, CA 90115, USA Email: [email protected]
+#
+# The full terms of this copyright and license should always be found in the root directory of this software deliverable as "license.txt" and if these terms are not found with this software, please contact the USC Stevens Center for the full license.
+#

mentor_upload_api/src/mentor_upload_tasks/tasks.py

@@ -0,0 +1,25 @@
+#
+# This software is Copyright ©️ 2020 The University of Southern California. All Rights Reserved.
+# Permission to use, copy, modify, and distribute this software and its documentation for educational, research and non-profit purposes, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and subject to the full license file found in the root of this software deliverable. Permission to make commercial use of this software may be obtained by contacting:  USC Stevens Center for Innovation University of Southern California 1150 S. Olive Street, Suite 2300, Los Angeles, CA 90115, USA Email: [email protected]
+#
+# The full terms of this copyright and license should always be found in the root directory of this software deliverable as "license.txt" and if these terms are not found with this software, please contact the USC Stevens Center for the full license.
+#
+import os
+
+from celery import Celery
+
+config = {
+    "broker_url": os.environ.get("CELERY_BROKER_URL", "redis://redis:6379/0"),
+    "result_backend": os.environ.get("CELERY_RESULT_BACKEND", "redis://redis:6379/0"),
+    "accept_content": ["json"],
+    "task_serializer": os.environ.get("CELERY_TASK_SERIALIZER", "json"),
+    "event_serializer": os.environ.get("CELERY_EVENT_SERIALIZER", "json"),
+    "result_serializer": os.environ.get("CELERY_RESULT_SERIALIZER", "json"),
+}
+celery = Celery("mentor_upload_tasks", broker=config["broker_url"])
+celery.conf.update(config)
+
+
+@celery.task()
+def upload_task(req):
+    pass