CVE support fo BIG-IP data collector added, NIST 2.0 REST API, Bugfixes

F5TT.md

@@ -47,16 +47,6 @@ Refer to [installation instructions](/contrib/docker-compose)
 
 ## As a native python application
 
-Second Sight data collector requires:
-
-- Any Linux distribution
-- Python 3 (tested on 3.9+)
-- [FastAPI](https://fastapi.tiangolo.com/)
-- [Uvicorn](https://www.uvicorn.org/)
-- [Requests](https://docs.python-requests.org/en/master/)
-- [json2html](https://pypi.org/project/json2html/)
-- [clickhouse-driver](https://pypi.org/project/clickhouse-driver/)
-
 Dependencies can be installed using:
 
 ```

contrib/bigip-collect/README.md

@@ -2,8 +2,7 @@
 
 ## Description
 
-The `bigIPCollect.sh` script must be copied and run on a BIG-IP instance (hardware-based or Virtual Edition). It will collect raw JSON files and package them into a single .tgz file:
-the .tgz file can then be processed offline by Second Sight to build all target JSON files
+The `bigIPCollect.sh` script must be copied and run on a BIG-IP instance (hardware-based or Virtual Edition). It will collect raw data and package them into a JSON file that can be processed offline by Second Sight.
 
 ## Installation on BIG-IP
 
@@ -88,16 +87,16 @@ Password:
 -> Collecting hardware details
 -> Collecting provisioned modules
 -> Collecting APM usage
--> Data collection completed, building tarfile
--> All done, copy /tmp/20221122-2351-bigIPCollect.tgz to your local host using scp
+-> Data collection completed, building JSON payload
+-> All done, copy /tmp/20230626-0002-bigIPCollect.json to your local host using scp
 [root@bigip1:Active:Disconnected] tmp #
 ```
 
-- Retrieve the tgz file
+- Retrieve the JSON file
 
 ```
-$ scp [email protected]:/tmp/20221122-2351-bigIPCollect.tgz .
+$ scp [email protected]:/tmp/20230626-0002-bigIPCollect.json .
 ([email protected]) Password: 
-20221122-2351-bigIPCollect.tgz               100%   14KB   5.4MB/s   00:00    
+20230626-0002-bigIPCollect.json               100%   14KB   5.4MB/s   00:00    
 $ 
 ```

contrib/bigiq-collect/f5ttfs.py

@@ -183,6 +183,7 @@ def bigiqLogin():
     "lastUpdateMicros": 1636742559283127
 });
 
+# Upload a BIG-IQ tgz file for postprocessing
 @app.route('/upload', methods = ['POST'])
 def upload_file():
    if request.method == 'POST':

f5tt/app.py

@@ -23,6 +23,7 @@
 # All modules
 import bigiq
 import nms
+import cveDB
 
 requests.packages.urllib3.disable_warnings(InsecureRequestWarning)
 
@@ -246,6 +247,24 @@ def getMetrics():
     return Response(content=reply,media_type="text/plain")
 
 
+# Post a BIG-IP JSON file, adds CVE information and returns the postprocessed JSON
+# POST body JSON format
+# curl -iX POST localhost:5000/getCVE/TMOS -d '{"tmos": { "version": "16.1.3", "modules": [ "apm", "ltm" ] } }'
+@app.post('/getCVE/TMOS')
+async def getCVE_bigip(request: Request):
+  if request.method == 'POST':
+    allCVE = {}
+    body = json.loads(await request.body())
+
+    if 'tmos' in body:
+      if 'version' in body['tmos']:
+        if 'modules' in body['tmos']:
+          for m in body['tmos']['modules']:
+            allCVE.update(cveDB.getF5(product=m,version=body['tmos']['version']))
+
+    return allCVE
+
+
 @app.get("/{uri}")
 @app.post("/{uri}")
 @app.put("/{uri}")

f5tt/cveDB.py

@@ -49,7 +49,7 @@ def init(nistURL='https://services.nvd.nist.gov',nistApiKey='',proxy={}):
 
 # Returns the cpeMatchString to query NIST REST API
 def __mkCpeMatchString(vendor="*",product="*",version="*"):
-  return 'cpe:2.3:*:'+vendor+':'+product+':'+version
+  return 'cpe:2.3:a:'+vendor+':'+product+':'+version
 
 
 # Fetches all CVE for the given vendor/product/version
@@ -58,30 +58,33 @@ def __getFromNist(vendor,product="*",version="*"):
 
   if cpeMatchString not in this.cveCachedDB:
     # If we don't have a local cached copy of the JSON, fetch it from NIST and cache it to speed up lookup
-    headers = { 'Content-Type': 'application/json' }
+
+    params = {
+      'resultsPerPage': 2000,
+      'cpeName': cpeMatchString,
+      'startIndex': 0
+    }
 
     if this.nistApiKey == '':
-      params = {
-        'resultsPerPage': 2000,
-        'cpeMatchString': cpeMatchString
+      headers = {
+        'Content-Type': 'application/json',
       }
     else:
-      params = {
-        'resultsPerPage': 2000,
-        'cpeMatchString': cpeMatchString,
-        'apiKey': this.nistApiKey
+      headers = {
+        'Content-Type': 'application/json',
+        'api-key': this.nistApiKey
       }
 
     s = Session()
-    req = Request('GET',this.nistURL+"/rest/json/cves/1.0",params=params,headers=headers)
+    req = Request('GET',this.nistURL+"/rest/json/cves/2.0",params=params,headers=headers)
 
     p = s.prepare_request(req)
     s.proxies = proxy
     try:
       res = s.send(p,verify=False)
       if res.status_code == 200:
         this.cveCachedDB[cpeMatchString]=res.json()
-    except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError):
+    except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError, KeyError):
         this.cveCachedDB[cpeMatchString]={}
 
   return this.cveCachedDB[cpeMatchString]
@@ -91,59 +94,91 @@ def __getFromNist(vendor,product="*",version="*"):
 def getF5(product="*",version="*"):
   matchingCVE={}
 
-  try:
-    allCVE = __getFromNist(vendor="f5",product="*",version=version)
-  except:
+  if product not in tmosModules2NIST:
     return matchingCVE
 
-  if product in tmosModules2NIST:
-    matchingProducts=tmosModules2NIST[product]
-
-    for product in matchingProducts:
-      if 'result' in allCVE:
-        for cve in allCVE['result']['CVE_Items']:
-          allCPEMatches=cve['configurations']['nodes'][0]['cpe_match']
+  for nistProduct in tmosModules2NIST[product]:
+    try:
+      allCVE = __getFromNist(vendor="f5",product=nistProduct,version=version)
+
+      if 'totalResults' in allCVE:
+        for cveTopLevel in allCVE['vulnerabilities']:
+          cve=cveTopLevel['cve']
+          cveId=cve['id']
+          cveUrl = []
+
+          if 'references' in cve:
+            for reference in cve['references']:
+              cveUrl.append(reference)
+
+          if 'descriptions' in cve:
+            for desc in cve['descriptions']:
+              if desc['lang'] == 'en':
+                cveDesc = desc['value']
+
+          if 'metrics' in cve:
+            if 'cvssMetricV31' in cve['metrics']:
+              cveExplScore=cve['metrics']['cvssMetricV31'][0]['exploitabilityScore']
+              cveBaseSeverity=cve['metrics']['cvssMetricV31'][0]['cvssData']['baseSeverity']
+              cveBaseScore=cve['metrics']['cvssMetricV31'][0]['cvssData']['baseScore']
+            elif 'cvssMetricV2' in cve['metrics']:
+              cveExplScore=cve['metrics']['cvssMetricV2'][0]['exploitabilityScore']
+              cveBaseSeverity=''
+              cveBaseScore=cve['metrics']['cvssMetricV2'][0]['cvssData']['baseScore']
+            else:
+              cveBaseSeverity=''
+              cveBaseScore=''
+              cveExplScore=''
 
-          for cpeMatch in allCPEMatches:
-            if product in cpeMatch['cpe23Uri']:
-              # Found CVE match
-              cveId=cve['cve']['CVE_data_meta']['ID']
-              cveUrl=cve['cve']['references']['reference_data'][0]['url']
-              cveDesc=cve['cve']['description']['description_data'][0]['value']
-              cveBaseSeverity=cve['impact']['baseMetricV3']['cvssV3']['baseSeverity'] if 'baseMetricV3' in cve['impact'] else ''
-              cveBaseScore=cve['impact']['baseMetricV3']['cvssV3']['baseScore'] if 'baseMetricV3' in cve['impact'] else ''
-              cveExplScore=cve['impact']['baseMetricV2']['exploitabilityScore'] if 'baseMetricV2' in cve['impact'] else ''
+          if cveId not in matchingCVE:
+            matchingCVE[cveId]={"id":cveId,"url":cveUrl,"description":cveDesc,"baseSeverity":cveBaseSeverity,"baseScore":cveBaseScore,"exploitabilityScore":cveExplScore}
 
-              if cveId not in matchingCVE:
-                matchingCVE[cveId]={"id":cveId,"url":cveUrl,"description":cveDesc,"baseSeverity":cveBaseSeverity,"baseScore":cveBaseScore,"exploitabilityScore":cveExplScore}
+    except:
+      return matchingCVE
 
   return matchingCVE
 
 
 # Returns all CVE for the given NGINX instance version
 def getNGINX(version="*"):
-  allCVE = __getFromNist(vendor="f5",product="nginx",version=version)
   matchingCVE={}
 
-  if version != '':
-    if 'result' in allCVE:
-      for cve in allCVE['result']['CVE_Items']:
-        allCPEMatches=cve['configurations']['nodes'][0]['cpe_match']
-
-        for cpeMatch in allCPEMatches:
-          # Found CVE match
-          cveId=cve['cve']['CVE_data_meta']['ID']
-          cveUrl=cve['cve']['references']['reference_data'][0]['url']
-          cveDesc=cve['cve']['description']['description_data'][0]['value']
-          cveBaseSeverity=cve['impact']['baseMetricV3']['cvssV3']['baseSeverity'] if 'baseMetricV3' in cve['impact'] else ''
-          cveBaseScore=cve['impact']['baseMetricV3']['cvssV3']['baseScore'] if 'baseMetricV3' in cve['impact'] else ''
-          cveExplScore=cve['impact']['baseMetricV2']['exploitabilityScore'] if 'baseMetricV2' in cve['impact'] else ''
-
-          if cveId not in matchingCVE:
-            matchingCVE[cveId]={"id":cveId,"url":cveUrl,"description":cveDesc,"baseSeverity":cveBaseSeverity,"baseScore":cveBaseScore,"exploitabilityScore":cveExplScore}
-
-    cveF5 = getF5(product="nginx",version=version)
+  try:
+    allCVE = __getFromNist(vendor="f5",product="nginx",version=version)
+  except:
+    return matchingCVE
 
-    matchingCVE.update(cveF5)
+  if version != '':
+    if 'totalResults' in allCVE:
+      for cveTopLevel in allCVE['vulnerabilities']:
+        cve=cveTopLevel['cve']
+        cveId=cve['id']
+        cveUrl = []
+
+        if 'references' in cve:
+          for reference in cve['references']:
+            cveUrl.append(reference)
+
+        if 'descriptions' in cve:
+          for desc in cve['descriptions']:
+            if desc['lang'] == 'en':
+              cveDesc = desc['value']
+
+        if 'metrics' in cve:
+          if 'cvssMetricV31' in cve['metrics']:
+            cveExplScore=cve['metrics']['cvssMetricV31'][0]['exploitabilityScore']
+            cveBaseSeverity=cve['metrics']['cvssMetricV31'][0]['cvssData']['baseSeverity']
+            cveBaseScore=cve['metrics']['cvssMetricV31'][0]['cvssData']['baseScore']
+          elif 'cvssMetricV2' in cve['metrics']:
+            cveExplScore=cve['metrics']['cvssMetricV2'][0]['exploitabilityScore']
+            cveBaseSeverity=''
+            cveBaseScore=cve['metrics']['cvssMetricV2'][0]['cvssData']['baseScore']
+          else:
+            cveBaseSeverity=''
+            cveBaseScore=''
+            cveExplScore=''
+
+        if cveId not in matchingCVE:
+          matchingCVE[cveId]={"id":cveId,"url":cveUrl,"description":cveDesc,"baseSeverity":cveBaseSeverity,"baseScore":cveBaseScore,"exploitabilityScore":cveExplScore}
 
   return matchingCVE