From 9ece0e924a397c4c1ba808acfd9aa0b711e188f1 Mon Sep 17 00:00:00 2001 From: Yusuf Zainee Date: Mon, 24 Jun 2019 16:40:08 +0530 Subject: [PATCH 1/5] sync endpoint added --- requirements.in | 1 + requirements.txt | 1 + src/rest_api.py | 51 +++++++++++++++++++++++++++++++++++++++++++++++- swagger.yaml | 51 +++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 102 insertions(+), 2 deletions(-) diff --git a/requirements.in b/requirements.in index a79963e..ab57876 100644 --- a/requirements.in +++ b/requirements.in @@ -12,3 +12,4 @@ codecov radon boto3 sentry-sdk +requests-futures diff --git a/requirements.txt b/requirements.txt index d231671..f6fdd46 100644 --- a/requirements.txt +++ b/requirements.txt @@ -39,6 +39,7 @@ pytest-mock==1.10.0 pytest==3.8.1 python-dateutil==2.8.0 # via botocore radon==3.0.1 +requests-futures==1.0.0 requests==2.19.1 s3transfer==0.2.0 # via boto3 six==1.11.0 # via flask-cors, mando, more-itertools, pytest, python-dateutil diff --git a/src/rest_api.py b/src/rest_api.py index 6decb4e..d6572b8 100644 --- a/src/rest_api.py +++ b/src/rest_api.py @@ -1,6 +1,6 @@ """Definition of the routes for gemini server.""" -import os import flask +import os import requests from flask import Flask, request from flask_cors import CORS @@ -15,6 +15,7 @@ from notification.user_notification import UserNotification from fabric8a_auth.errors import AuthError import sentry_sdk +from requests_futures.sessions import FuturesSession app = Flask(__name__) @@ -23,6 +24,12 @@ sentry_sdk.init(os.environ.get("SENTRY_DSN")) init_selinon() +_session = FuturesSession(max_workers=3) + +_SERVICE_HOST = os.environ.get("BAYESIAN_DATA_IMPORTER_SERVICE_HOST", "bayesian-data-importer") +_SERVICE_PORT = os.environ.get("BAYESIAN_DATA_IMPORTER_SERVICE_PORT", "9192") +_CVE_SYNC_ENDPOINT = "api/v1/sync_latest_non_cve_version" +_LATEST_VERSION_SYNC_ENDPOINT = "api/v1/sync_latest_version" SERVICE_TOKEN = 'token' try: @@ -44,6 +51,48 @@ def liveness(): return flask.jsonify({}), 200 +@app.route('/api/v1/sync-graph-data', methods=['POST']) +@login_required +def sync_data(): + """ + Endpoint for carrying a sync of graph db data. + + Takes in value to either call sync of non cve version + or latest version or both. + + valid input: { + "non_cve_sync": True / False + "latest_version_sync: True / false + "cve_ecosystem": ['maven', 'pypi, 'npm'] + } + + """ + resp = { + "success": True, + "message": "Sync operation started." + } + + input_json = request.get_json() + non_cve_sync = input_json.get('non_cve_sync', False) + if non_cve_sync: + cve_ecosystem = input_json.get('cve_ecosystem', []) + if len(cve_ecosystem) == 0: + resp['success'] = False + resp['message'] = "Incorrect data.. Send cve_ecosystem for non_cve_sync operation" + return flask.jsonify(resp), 400 + url = "http://{host}:{port}/{endpoint}".format(host=_SERVICE_HOST, + port=_SERVICE_PORT, + endpoint=_CVE_SYNC_ENDPOINT) + _session.post(url, json=cve_ecosystem) + latest_version_sync = input_json.get('latest_version_sync', False) + if latest_version_sync: + url = "http://{host}:{port}/{endpoint}".format(host=_SERVICE_HOST, + port=_SERVICE_PORT, + endpoint=_LATEST_VERSION_SYNC_ENDPOINT) + _session.post(url, json=['all']) + return flask.jsonify(resp), 200 + + @app.route('/api/v1/register', methods=['POST']) @login_required def register(): diff --git a/swagger.yaml b/swagger.yaml index ee7dbe7..4f25915 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -26,6 +26,36 @@ paths: responses: '200': description: Service is ready + /sync-graph-data: + post: + tags: + - Sync Graph DB + operationId: f8a_scanner.api_v1.sync-graph-data + summary: Sync the graph db data + consumes: + - application/json + produces: + - application/json + parameters: + - in: body + name: repo + description: repository details and email id + required: true + schema: + $ref: '#/definitions/sync' + responses: + '200': + schema: + $ref: "#/definitions/syncresp" + description: Sync operation started + '400': + description: Bad request from the client + '401': + description: Request unauthorized + '404': + description: Data not found + '500': + description: Internal server error /register: post: tags: @@ -177,7 +207,7 @@ paths: summary: Lists the available reports. description: > Lists the available stack analyses reports that have been generated offline. The frequency of these generated reports are weekly and monthly. - + For listing monthly reports call `stacks-report/list/monthly` and for getting the list of weekly reports call `stacks-report/list/weekly` operationId: f8a_scanner.api_v1.list_reports produces: @@ -241,6 +271,25 @@ paths: '404': description: No comparison data could be found definitions: + sync: + title: Sync operation request + properties: + non_cve_sync: + type: boolean + cve_ecosystem: + type: array + items: + type: string + latest_version_sync: + type: boolean + syncresp: + title: Sync operation response + description: Sync non cve version and latest version response + properties: + success: + type: boolean + summary: + type: string ComparisonReport: title: Comparison Report description: Comparison Report structure From 46ce9d8c9ec48fe56759e6902ac6c70febf7c57b Mon Sep 17 00:00:00 2001 From: Yusuf Zainee Date: Mon, 24 Jun 2019 19:36:25 +0530 Subject: [PATCH 2/5] test cases added --- src/rest_api.py | 21 +++++++++++----- tests/test_rest_api.py | 54 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 65 insertions(+), 10 deletions(-) diff --git a/src/rest_api.py b/src/rest_api.py index d6572b8..d372bf0 100644 --- a/src/rest_api.py +++ b/src/rest_api.py @@ -16,11 +16,13 @@ from fabric8a_auth.errors import AuthError import sentry_sdk from requests_futures.sessions import FuturesSession +import logging app = Flask(__name__) CORS(app) - +logging.basicConfig(level=logging.INFO) +logger = logging.getLogger(__name__) sentry_sdk.init(os.environ.get("SENTRY_DSN")) init_selinon() @@ -61,35 +63,42 @@ def sync_data(): or latest version or both. valid input: { - "non_cve_sync": True / False - "latest_version_sync: True / false + "non_cve_sync": True / False, + "latest_version_sync": True / false, "cve_ecosystem": ['maven', 'pypi, 'npm'] } """ resp = { "success": True, - "message": "Sync operation started." + "message": "Sync operation started ->" } input_json = request.get_json() + logger.info("sync-graph-data called with the input {i}".format(i=input_json)) non_cve_sync = input_json.get('non_cve_sync', False) - if non_cve_sync: + if non_cve_sync == "true": cve_ecosystem = input_json.get('cve_ecosystem', []) if len(cve_ecosystem) == 0: resp['success'] = False resp['message'] = "Incorrect data.. Send cve_ecosystem for non_cve_sync operation" + logger.error("Incorrect data.. Send cve_ecosystem for non_cve_sync operation") return flask.jsonify(resp), 400 url = "http://{host}:{port}/{endpoint}".format(host=_SERVICE_HOST, port=_SERVICE_PORT, endpoint=_CVE_SYNC_ENDPOINT) + logger.info("Calling non cve sync with {i}".format(i=cve_ecosystem)) _session.post(url, json=cve_ecosystem) + resp['message'] = resp['message'] + " for non cve version" latest_version_sync = input_json.get('latest_version_sync', False) - if latest_version_sync: + if latest_version_sync == "true": url = "http://{host}:{port}/{endpoint}".format(host=_SERVICE_HOST, port=_SERVICE_PORT, endpoint=_LATEST_VERSION_SYNC_ENDPOINT) + logger.info("Calling latest version sync with 'all'") _session.post(url, json=['all']) + resp['message'] = resp['message'] + " for latest version" + logger.info("Sync operation called.. Message->", resp['message']) return flask.jsonify(resp), 200 diff --git a/tests/test_rest_api.py b/tests/test_rest_api.py index 966dd53..e46f37b 100644 --- a/tests/test_rest_api.py +++ b/tests/test_rest_api.py @@ -141,10 +141,10 @@ def test_liveness_endpoint(client): assert json_data == {} -def test_liveness_endpoint_wrong_http_method(client): - """Test the /api/v1/liveness endpoint by calling it with wrong HTTP method.""" - url = api_route_for("liveness") - response = client.post(url) +def test_sync_data(client): + """Test the /api/v1/sync-graph-data endpoint.""" + url = api_route_for("sync-graph-data") + response = client.get(url) assert response.status_code == 405 response = client.put(url) assert response.status_code == 405 @@ -154,6 +154,52 @@ def test_liveness_endpoint_wrong_http_method(client): assert response.status_code == 405 +@patch("requests.Session.post", return_value="") +def test_sync_data1(_mock1, client): + """Test the /api/v1/sync-graph-data endpoint.""" + url = api_route_for("sync-graph-data") + inp = { + "non_cve_sync": "false", + "latest_version_sync": "true", + "cve_ecosystem": ["npm"] + } + response = client.post(url, + data=json.dumps(inp), + content_type='application/json') + assert response.status_code == 200 + + inp = { + "non_cve_sync": "true", + "latest_version_sync": "true" + } + response = client.post(url, + data=json.dumps(inp), + content_type='application/json') + assert response.status_code == 400 + + inp = { + "non_cve_sync": "true", + "latest_version_sync": "true", + "cve_ecosystem": ["npm"] + } + response = client.post(url, + data=json.dumps(inp), + content_type='application/json') + assert response.status_code == 200 + + +def test_register_endpoint_1(client): + """Test the /api/v1/register endpoint.""" + reg_resp = client.post(api_route_for('register'), + data=json.dumps(payload)) + assert reg_resp.status_code == 400 + reg_resp_json = get_json_from_response(reg_resp) + assert reg_resp_json == { + "success": False, + "summary": "Set content type to application/json" + } + + @patch("src.rest_api.retrieve_worker_result") def test_report_endpoint(mocker, client): """Test the /api/v1/report endpoint.""" From 96227f0d008d72cfb5472d656429338d2f38ca22 Mon Sep 17 00:00:00 2001 From: Yusuf Zainee Date: Mon, 24 Jun 2019 19:58:00 +0530 Subject: [PATCH 3/5] test rectified --- tests/test_rest_api.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/test_rest_api.py b/tests/test_rest_api.py index e46f37b..18af3c4 100644 --- a/tests/test_rest_api.py +++ b/tests/test_rest_api.py @@ -188,18 +188,6 @@ def test_sync_data1(_mock1, client): assert response.status_code == 200 -def test_register_endpoint_1(client): - """Test the /api/v1/register endpoint.""" - reg_resp = client.post(api_route_for('register'), - data=json.dumps(payload)) - assert reg_resp.status_code == 400 - reg_resp_json = get_json_from_response(reg_resp) - assert reg_resp_json == { - "success": False, - "summary": "Set content type to application/json" - } - - @patch("src.rest_api.retrieve_worker_result") def test_report_endpoint(mocker, client): """Test the /api/v1/report endpoint.""" From 974f09f28c405efb86d6c2fd3f1aff118eb622f1 Mon Sep 17 00:00:00 2001 From: Yusuf Zainee Date: Mon, 1 Jul 2019 19:47:53 +0530 Subject: [PATCH 4/5] review comments --- src/rest_api.py | 4 ++-- swagger.yaml | 4 ++-- tests/test_rest_api.py | 20 ++++++++++---------- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/src/rest_api.py b/src/rest_api.py index d372bf0..d68f990 100644 --- a/src/rest_api.py +++ b/src/rest_api.py @@ -77,7 +77,7 @@ def sync_data(): input_json = request.get_json() logger.info("sync-graph-data called with the input {i}".format(i=input_json)) non_cve_sync = input_json.get('non_cve_sync', False) - if non_cve_sync == "true": + if non_cve_sync: cve_ecosystem = input_json.get('cve_ecosystem', []) if len(cve_ecosystem) == 0: resp['success'] = False @@ -91,7 +91,7 @@ def sync_data(): _session.post(url, json=cve_ecosystem) resp['message'] = resp['message'] + " for non cve version" latest_version_sync = input_json.get('latest_version_sync', False) - if latest_version_sync == "true": + if latest_version_sync: url = "http://{host}:{port}/{endpoint}".format(host=_SERVICE_HOST, port=_SERVICE_PORT, endpoint=_LATEST_VERSION_SYNC_ENDPOINT) diff --git a/swagger.yaml b/swagger.yaml index 4f25915..8c6213d 100644 --- a/swagger.yaml +++ b/swagger.yaml @@ -38,8 +38,8 @@ paths: - application/json parameters: - in: body - name: repo - description: repository details and email id + name: sync + description: parameters to perform sync operation required: true schema: $ref: '#/definitions/sync' diff --git a/tests/test_rest_api.py b/tests/test_rest_api.py index 18af3c4..db0254d 100644 --- a/tests/test_rest_api.py +++ b/tests/test_rest_api.py @@ -158,32 +158,32 @@ def test_sync_data(client): def test_sync_data1(_mock1, client): """Test the /api/v1/sync-graph-data endpoint.""" url = api_route_for("sync-graph-data") - inp = { - "non_cve_sync": "false", + inp = '''{ + "non_cve_sync": true, "latest_version_sync": "true", "cve_ecosystem": ["npm"] - } + }''' response = client.post(url, - data=json.dumps(inp), + data=inp, content_type='application/json') assert response.status_code == 200 - inp = { + inp = '''{ "non_cve_sync": "true", "latest_version_sync": "true" - } + }''' response = client.post(url, - data=json.dumps(inp), + data=inp, content_type='application/json') assert response.status_code == 400 - inp = { + inp = '''{ "non_cve_sync": "true", "latest_version_sync": "true", "cve_ecosystem": ["npm"] - } + }''' response = client.post(url, - data=json.dumps(inp), + data=inp, content_type='application/json') assert response.status_code == 200 From 08b014048059415b352196d7e90e98dded5e2f14 Mon Sep 17 00:00:00 2001 From: Yusuf Zainee Date: Tue, 2 Jul 2019 12:35:07 +0530 Subject: [PATCH 5/5] review comments --- src/rest_api.py | 4 ++-- tests/test_rest_api.py | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/rest_api.py b/src/rest_api.py index d68f990..6f747e2 100644 --- a/src/rest_api.py +++ b/src/rest_api.py @@ -63,8 +63,8 @@ def sync_data(): or latest version or both. valid input: { - "non_cve_sync": True / False, - "latest_version_sync": True / false, + "non_cve_sync": true/false, + "latest_version_sync": true/false, "cve_ecosystem": ['maven', 'pypi, 'npm'] } diff --git a/tests/test_rest_api.py b/tests/test_rest_api.py index db0254d..92f8aed 100644 --- a/tests/test_rest_api.py +++ b/tests/test_rest_api.py @@ -160,7 +160,7 @@ def test_sync_data1(_mock1, client): url = api_route_for("sync-graph-data") inp = '''{ "non_cve_sync": true, - "latest_version_sync": "true", + "latest_version_sync": true, "cve_ecosystem": ["npm"] }''' response = client.post(url, @@ -169,8 +169,8 @@ def test_sync_data1(_mock1, client): assert response.status_code == 200 inp = '''{ - "non_cve_sync": "true", - "latest_version_sync": "true" + "non_cve_sync": true, + "latest_version_sync": true }''' response = client.post(url, data=inp, @@ -178,8 +178,8 @@ def test_sync_data1(_mock1, client): assert response.status_code == 400 inp = '''{ - "non_cve_sync": "true", - "latest_version_sync": "true", + "non_cve_sync": true, + "latest_version_sync": true, "cve_ecosystem": ["npm"] }''' response = client.post(url,