Starting unknown flow for the packages missing latest version in dail…

…y ingestion report

Dockerfile

@@ -11,8 +11,6 @@ RUN yum install -y epel-release &&\
     mkdir -p ${APP_DIR}
 
 RUN pip3 install --upgrade pip
-RUN pip3 install git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@${F8A_UTIL_VERSION}
-RUN pip3 install git+https://[email protected]/fabric8-analytics/fabric8-analytics-version-comparator.git#egg=f8a_version_comparator
 
 COPY f8a_report/ ${APP_DIR}/f8a_report
 COPY requirements.txt ${APP_DIR}

f8a_report/ingestion_helper.py

@@ -0,0 +1,46 @@
+import logging
+from selinon import run_flow
+from f8a_worker.utils import MavenCoordinates
+import os
+
+logger = logging.getLogger(__name__)
+
+
+def server_create_analysis(ecosystem, package, version, force=False, force_graph_sync=False):
+    """Create bayesianApiFlow handling analyses for specified EPV.
+
+    :param ecosystem: ecosystem for which the flow should be run
+    :param package: package for which should be flow run
+    :param version: package version
+    :param force: force run flow even specified EPV exists
+    :param force_graph_sync: force synchronization to graph
+    :return: dispatcher ID handling flow
+    """
+    component = MavenCoordinates.normalize_str(package) if ecosystem == 'maven' else package
+
+    args = {
+            "ecosystem": ecosystem,
+            "force": force,
+            "force_graph_sync": force_graph_sync,
+            "name": component,
+            "recursive_limit": 0,
+            "version": version
+        }
+
+    if os.environ.get("WORKER_ADMINISTRATION_REGION", "") == "api":
+        return server_run_flow('bayesianApiFlow', args)
+    else:
+        return server_run_flow('bayesianFlow', args)
+
+
+def server_run_flow(flow_name, flow_args):
+    """Run a flow.
+
+    :param flow_name: name of flow to be run as stated in YAML config file
+    :param flow_args: arguments for the flow
+    :return: dispatcher ID handling flow
+    """
+    logger.info('Running flow {} for args {}'.format(flow_name, flow_args))
+    dispacher_id = run_flow(flow_name, flow_args)
+
+    return dispacher_id

f8a_report/main.py

@@ -6,6 +6,7 @@
 from v2.report_generator import StackReportBuilder
 from manifest_helper import manifest_interface
 import os
+from f8a_worker.setup_celery import init_selinon, init_celery
 
 logger = logging.getLogger(__file__)
 
@@ -79,4 +80,7 @@ def main():
 
 
 if __name__ == '__main__':
+    if os.environ.get("INVOKE_API_WORKERS", "") == "1":
+        init_selinon()
+        init_celery(result_backend=False)
     main()

f8a_report/report_helper.py

@@ -19,6 +19,8 @@
 from unknown_deps_report_helper import UnknownDepsReportHelper
 from sentry_report_helper import SentryReportHelper
 from cve_helper import CVE
+import os
+from ingestion_helper import server_create_analysis
 
 logger = logging.getLogger(__file__)
 logging.basicConfig(level=logging.INFO)
@@ -724,6 +726,12 @@ def check_latest_node(self, latest_epvs, template):
                     "version": ver
                 }
                 missing_latest[eco].append(tmp)
+
+                if os.environ.get("DISABLE_UNKNOWN_PACKAGE_FLOW", "") == "0" and os.environ.get("INVOKE_API_WORKERS", "") == "1":
+                    logger.info("Starting ingestion flow for eco= {}, pkg= {}, ver= {}".format(eco, pkg, ver))
+                    dispatcher_id = server_create_analysis(eco, pkg, ver, force=True, force_graph_sync=False)
+                    logger.info("Dispatcher_ID= {}".format(dispatcher_id))
+
         template['ingestion_summary']['missing_latest_node'] = missing_latest
         return template
 

openshift/template.yaml

@@ -181,6 +181,22 @@ objects:
                   value: ${PYPI_TRAINING_REPO}
                 - name: GREMLIN_QUERY_SIZE
                   value: "25"
+                - name: WORKER_ADMINISTRATION_REGION
+                  value: "api"
+                - name: INVOKE_API_WORKERS
+                  value: "1"
+                - name: DISABLE_UNKNOWN_PACKAGE_FLOW
+                  value: "0"
+                - name: AWS_SQS_ACCESS_KEY_ID
+                  valueFrom:
+                    secretKeyRef:
+                      name: aws-sqs
+                      key: aws_access_key_id
+                - name: AWS_SQS_SECRET_ACCESS_KEY
+                  valueFrom:
+                    secretKeyRef:
+                      name: aws-sqs
+                      key: aws_secret_access_key
               resources:
                 requests:
                   memory: ${MEMORY_REQUEST}
@@ -231,7 +247,7 @@ parameters:
   name: MEMORY_REQUEST
   value: "1024Mi"
 
-- description: Memory limit 
+- description: Memory limit
   displayName: Memory limit
   required: true
   name: MEMORY_LIMIT

requirements.txt

@@ -61,5 +61,9 @@ xmltodict==0.12.0         # via moto
 zipp==3.1.0               # via importlib-metadata
 zope.interface==4.6.0     # via datetime
 
+git+https://github.com/fabric8-analytics/fabric8-analytics-worker.git@58d3025
+git+https://github.com/fabric8-analytics/fabric8-analytics-utils.git@3bca34e
+git+https://github.com/fabric8-analytics/fabric8-analytics-version-comparator.git#egg=f8a_version_comparator
+
 # The following packages are considered to be unsafe in a requirements file:
 # setuptools