Merge pull request #195 from yzainee/gemini_yzainee

Introduction of new Gemini API to sync graph data

requirements.in

@@ -12,3 +12,4 @@ codecov
 radon
 boto3
 sentry-sdk
+requests-futures

requirements.txt

@@ -39,6 +39,7 @@ pytest-mock==1.10.0
 pytest==3.8.1
 python-dateutil==2.8.0    # via botocore
 radon==3.0.1
+requests-futures==1.0.0
 requests==2.19.1
 s3transfer==0.2.0         # via boto3
 six==1.11.0               # via flask-cors, mando, more-itertools, pytest, python-dateutil

src/rest_api.py

@@ -1,6 +1,6 @@
 """Definition of the routes for gemini server."""
-import os
 import flask
+import os
 import requests
 from flask import Flask, request
 from flask_cors import CORS
@@ -15,14 +15,23 @@
 from notification.user_notification import UserNotification
 from fabric8a_auth.errors import AuthError
 import sentry_sdk
+from requests_futures.sessions import FuturesSession
+import logging
 
 
 app = Flask(__name__)
 CORS(app)
-
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
 sentry_sdk.init(os.environ.get("SENTRY_DSN"))
 
 init_selinon()
+_session = FuturesSession(max_workers=3)
+
+_SERVICE_HOST = os.environ.get("BAYESIAN_DATA_IMPORTER_SERVICE_HOST", "bayesian-data-importer")
+_SERVICE_PORT = os.environ.get("BAYESIAN_DATA_IMPORTER_SERVICE_PORT", "9192")
+_CVE_SYNC_ENDPOINT = "api/v1/sync_latest_non_cve_version"
+_LATEST_VERSION_SYNC_ENDPOINT = "api/v1/sync_latest_version"
 
 SERVICE_TOKEN = 'token'
 try:
@@ -44,6 +53,55 @@ def liveness():
     return flask.jsonify({}), 200
 
 
+@app.route('/api/v1/sync-graph-data', methods=['POST'])
+@login_required
+def sync_data():
+    """
+    Endpoint for carrying a sync of graph db data.
+
+    Takes in value to either call sync of non cve version
+    or latest version or both.
+
+    valid input: {
+        "non_cve_sync": true/false,
+        "latest_version_sync": true/false,
+        "cve_ecosystem": ['maven', 'pypi, 'npm']
+    }
+
+    """
+    resp = {
+        "success": True,
+        "message": "Sync operation started ->"
+    }
+
+    input_json = request.get_json()
+    logger.info("sync-graph-data called with the input {i}".format(i=input_json))
+    non_cve_sync = input_json.get('non_cve_sync', False)
+    if non_cve_sync:
+        cve_ecosystem = input_json.get('cve_ecosystem', [])
+        if len(cve_ecosystem) == 0:
+            resp['success'] = False
+            resp['message'] = "Incorrect data.. Send cve_ecosystem for non_cve_sync operation"
+            logger.error("Incorrect data.. Send cve_ecosystem for non_cve_sync operation")
+            return flask.jsonify(resp), 400
+        url = "http://{host}:{port}/{endpoint}".format(host=_SERVICE_HOST,
+                                                       port=_SERVICE_PORT,
+                                                       endpoint=_CVE_SYNC_ENDPOINT)
+        logger.info("Calling non cve sync with {i}".format(i=cve_ecosystem))
+        _session.post(url, json=cve_ecosystem)
+        resp['message'] = resp['message'] + " for non cve version"
+    latest_version_sync = input_json.get('latest_version_sync', False)
+    if latest_version_sync:
+        url = "http://{host}:{port}/{endpoint}".format(host=_SERVICE_HOST,
+                                                       port=_SERVICE_PORT,
+                                                       endpoint=_LATEST_VERSION_SYNC_ENDPOINT)
+        logger.info("Calling latest version sync with 'all'")
+        _session.post(url, json=['all'])
+        resp['message'] = resp['message'] + " for latest version"
+    logger.info("Sync operation called.. Message->", resp['message'])
+    return flask.jsonify(resp), 200
+
+
 @app.route('/api/v1/register', methods=['POST'])
 @login_required
 def register():

swagger.yaml

@@ -26,6 +26,36 @@ paths:
       responses:
         '200':
           description: Service is ready
+  /sync-graph-data:
+    post:
+      tags:
+        - Sync Graph DB
+      operationId: f8a_scanner.api_v1.sync-graph-data
+      summary: Sync the graph db data
+      consumes:
+        - application/json
+      produces:
+        - application/json
+      parameters:
+        - in: body
+          name: sync
+          description: parameters to perform sync operation
+          required: true
+          schema:
+            $ref: '#/definitions/sync'
+      responses:
+        '200':
+          schema:
+            $ref: "#/definitions/syncresp"
+          description: Sync operation started
+        '400':
+          description: Bad request from the client
+        '401':
+          description: Request unauthorized
+        '404':
+          description: Data not found
+        '500':
+          description: Internal server error
   /register:
     post:
       tags:
@@ -177,7 +207,7 @@ paths:
       summary: Lists the available reports.
       description: >
         Lists the available stack analyses reports that have been generated offline. The frequency of these generated reports are weekly and monthly.
-        
+
         For listing monthly reports call `stacks-report/list/monthly` and for getting the list of weekly reports call `stacks-report/list/weekly`
       operationId: f8a_scanner.api_v1.list_reports
       produces:
@@ -241,6 +271,25 @@ paths:
         '404':
           description: No comparison data could be found
 definitions:
+  sync:
+    title: Sync operation request
+    properties:
+      non_cve_sync:
+        type: boolean
+      cve_ecosystem:
+        type: array
+        items:
+          type: string
+      latest_version_sync:
+        type: boolean
+  syncresp:
+    title: Sync operation response
+    description: Sync non cve version and latest version response
+    properties:
+      success:
+        type: boolean
+      summary:
+        type: string
   ComparisonReport:
     title: Comparison Report
     description: Comparison Report structure

tests/test_rest_api.py

@@ -141,10 +141,10 @@ def test_liveness_endpoint(client):
     assert json_data == {}
 
 
-def test_liveness_endpoint_wrong_http_method(client):
-    """Test the /api/v1/liveness endpoint by calling it with wrong HTTP method."""
-    url = api_route_for("liveness")
-    response = client.post(url)
+def test_sync_data(client):
+    """Test the /api/v1/sync-graph-data endpoint."""
+    url = api_route_for("sync-graph-data")
+    response = client.get(url)
     assert response.status_code == 405
     response = client.put(url)
     assert response.status_code == 405
@@ -154,6 +154,40 @@ def test_liveness_endpoint_wrong_http_method(client):
     assert response.status_code == 405
 
 
+@patch("requests.Session.post", return_value="")
+def test_sync_data1(_mock1, client):
+    """Test the /api/v1/sync-graph-data endpoint."""
+    url = api_route_for("sync-graph-data")
+    inp = '''{
+        "non_cve_sync": true,
+        "latest_version_sync": true,
+        "cve_ecosystem": ["npm"]
+    }'''
+    response = client.post(url,
+                           data=inp,
+                           content_type='application/json')
+    assert response.status_code == 200
+
+    inp = '''{
+        "non_cve_sync": true,
+        "latest_version_sync": true
+    }'''
+    response = client.post(url,
+                           data=inp,
+                           content_type='application/json')
+    assert response.status_code == 400
+
+    inp = '''{
+        "non_cve_sync": true,
+        "latest_version_sync": true,
+        "cve_ecosystem": ["npm"]
+    }'''
+    response = client.post(url,
+                           data=inp,
+                           content_type='application/json')
+    assert response.status_code == 200
+
+
 @patch("src.rest_api.retrieve_worker_result")
 def test_report_endpoint(mocker, client):
     """Test the /api/v1/report endpoint."""