diff --git a/F5TT.md b/F5TT.md index e8158a7..acae932 100644 --- a/F5TT.md +++ b/F5TT.md @@ -47,16 +47,6 @@ Refer to [installation instructions](/contrib/docker-compose) ## As a native python application -Second Sight data collector requires: - -- Any Linux distribution -- Python 3 (tested on 3.9+) -- [FastAPI](https://fastapi.tiangolo.com/) -- [Uvicorn](https://www.uvicorn.org/) -- [Requests](https://docs.python-requests.org/en/master/) -- [json2html](https://pypi.org/project/json2html/) -- [clickhouse-driver](https://pypi.org/project/clickhouse-driver/) - Dependencies can be installed using: ``` diff --git a/contrib/bigip-collect/README.md b/contrib/bigip-collect/README.md index 59ba8bc..7b32a92 100644 --- a/contrib/bigip-collect/README.md +++ b/contrib/bigip-collect/README.md @@ -2,8 +2,7 @@ ## Description -The `bigIPCollect.sh` script must be copied and run on a BIG-IP instance (hardware-based or Virtual Edition). It will collect raw JSON files and package them into a single .tgz file: -the .tgz file can then be processed offline by Second Sight to build all target JSON files +The `bigIPCollect.sh` script must be copied and run on a BIG-IP instance (hardware-based or Virtual Edition). It will collect raw data and package them into a JSON file that can be processed offline by Second Sight. ## Installation on BIG-IP @@ -88,16 +87,16 @@ Password: -> Collecting hardware details -> Collecting provisioned modules -> Collecting APM usage --> Data collection completed, building tarfile --> All done, copy /tmp/20221122-2351-bigIPCollect.tgz to your local host using scp +-> Data collection completed, building JSON payload +-> All done, copy /tmp/20230626-0002-bigIPCollect.json to your local host using scp [root@bigip1:Active:Disconnected] tmp # ``` -- Retrieve the tgz file +- Retrieve the JSON file ``` -$ scp root@bigip.f5:/tmp/20221122-2351-bigIPCollect.tgz . +$ scp root@bigip.f5:/tmp/20230626-0002-bigIPCollect.json . (root@bigip.f5) Password: -20221122-2351-bigIPCollect.tgz 100% 14KB 5.4MB/s 00:00 +20230626-0002-bigIPCollect.json 100% 14KB 5.4MB/s 00:00 $ ``` diff --git a/contrib/bigiq-collect/f5ttfs.py b/contrib/bigiq-collect/f5ttfs.py index 6367215..82f8700 100755 --- a/contrib/bigiq-collect/f5ttfs.py +++ b/contrib/bigiq-collect/f5ttfs.py @@ -183,6 +183,7 @@ def bigiqLogin(): "lastUpdateMicros": 1636742559283127 }); +# Upload a BIG-IQ tgz file for postprocessing @app.route('/upload', methods = ['POST']) def upload_file(): if request.method == 'POST': diff --git a/f5tt/app.py b/f5tt/app.py index 7fe93c6..f6eb978 100644 --- a/f5tt/app.py +++ b/f5tt/app.py @@ -23,6 +23,7 @@ # All modules import bigiq import nms +import cveDB requests.packages.urllib3.disable_warnings(InsecureRequestWarning) @@ -246,6 +247,24 @@ def getMetrics(): return Response(content=reply,media_type="text/plain") +# Post a BIG-IP JSON file, adds CVE information and returns the postprocessed JSON +# POST body JSON format +# curl -iX POST localhost:5000/getCVE/TMOS -d '{"tmos": { "version": "16.1.3", "modules": [ "apm", "ltm" ] } }' +@app.post('/getCVE/TMOS') +async def getCVE_bigip(request: Request): + if request.method == 'POST': + allCVE = {} + body = json.loads(await request.body()) + + if 'tmos' in body: + if 'version' in body['tmos']: + if 'modules' in body['tmos']: + for m in body['tmos']['modules']: + allCVE.update(cveDB.getF5(product=m,version=body['tmos']['version'])) + + return allCVE + + @app.get("/{uri}") @app.post("/{uri}") @app.put("/{uri}") diff --git a/f5tt/cveDB.py b/f5tt/cveDB.py index cf7975f..9662ade 100644 --- a/f5tt/cveDB.py +++ b/f5tt/cveDB.py @@ -49,7 +49,7 @@ def init(nistURL='https://services.nvd.nist.gov',nistApiKey='',proxy={}): # Returns the cpeMatchString to query NIST REST API def __mkCpeMatchString(vendor="*",product="*",version="*"): - return 'cpe:2.3:*:'+vendor+':'+product+':'+version + return 'cpe:2.3:a:'+vendor+':'+product+':'+version # Fetches all CVE for the given vendor/product/version @@ -58,22 +58,25 @@ def __getFromNist(vendor,product="*",version="*"): if cpeMatchString not in this.cveCachedDB: # If we don't have a local cached copy of the JSON, fetch it from NIST and cache it to speed up lookup - headers = { 'Content-Type': 'application/json' } + + params = { + 'resultsPerPage': 2000, + 'cpeName': cpeMatchString, + 'startIndex': 0 + } if this.nistApiKey == '': - params = { - 'resultsPerPage': 2000, - 'cpeMatchString': cpeMatchString + headers = { + 'Content-Type': 'application/json', } else: - params = { - 'resultsPerPage': 2000, - 'cpeMatchString': cpeMatchString, - 'apiKey': this.nistApiKey + headers = { + 'Content-Type': 'application/json', + 'api-key': this.nistApiKey } s = Session() - req = Request('GET',this.nistURL+"/rest/json/cves/1.0",params=params,headers=headers) + req = Request('GET',this.nistURL+"/rest/json/cves/2.0",params=params,headers=headers) p = s.prepare_request(req) s.proxies = proxy @@ -81,7 +84,7 @@ def __getFromNist(vendor,product="*",version="*"): res = s.send(p,verify=False) if res.status_code == 200: this.cveCachedDB[cpeMatchString]=res.json() - except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError): + except (ConnectTimeout, HTTPError, ReadTimeout, Timeout, ConnectionError, KeyError): this.cveCachedDB[cpeMatchString]={} return this.cveCachedDB[cpeMatchString] @@ -91,59 +94,91 @@ def __getFromNist(vendor,product="*",version="*"): def getF5(product="*",version="*"): matchingCVE={} - try: - allCVE = __getFromNist(vendor="f5",product="*",version=version) - except: + if product not in tmosModules2NIST: return matchingCVE - if product in tmosModules2NIST: - matchingProducts=tmosModules2NIST[product] - - for product in matchingProducts: - if 'result' in allCVE: - for cve in allCVE['result']['CVE_Items']: - allCPEMatches=cve['configurations']['nodes'][0]['cpe_match'] + for nistProduct in tmosModules2NIST[product]: + try: + allCVE = __getFromNist(vendor="f5",product=nistProduct,version=version) + + if 'totalResults' in allCVE: + for cveTopLevel in allCVE['vulnerabilities']: + cve=cveTopLevel['cve'] + cveId=cve['id'] + cveUrl = [] + + if 'references' in cve: + for reference in cve['references']: + cveUrl.append(reference) + + if 'descriptions' in cve: + for desc in cve['descriptions']: + if desc['lang'] == 'en': + cveDesc = desc['value'] + + if 'metrics' in cve: + if 'cvssMetricV31' in cve['metrics']: + cveExplScore=cve['metrics']['cvssMetricV31'][0]['exploitabilityScore'] + cveBaseSeverity=cve['metrics']['cvssMetricV31'][0]['cvssData']['baseSeverity'] + cveBaseScore=cve['metrics']['cvssMetricV31'][0]['cvssData']['baseScore'] + elif 'cvssMetricV2' in cve['metrics']: + cveExplScore=cve['metrics']['cvssMetricV2'][0]['exploitabilityScore'] + cveBaseSeverity='' + cveBaseScore=cve['metrics']['cvssMetricV2'][0]['cvssData']['baseScore'] + else: + cveBaseSeverity='' + cveBaseScore='' + cveExplScore='' - for cpeMatch in allCPEMatches: - if product in cpeMatch['cpe23Uri']: - # Found CVE match - cveId=cve['cve']['CVE_data_meta']['ID'] - cveUrl=cve['cve']['references']['reference_data'][0]['url'] - cveDesc=cve['cve']['description']['description_data'][0]['value'] - cveBaseSeverity=cve['impact']['baseMetricV3']['cvssV3']['baseSeverity'] if 'baseMetricV3' in cve['impact'] else '' - cveBaseScore=cve['impact']['baseMetricV3']['cvssV3']['baseScore'] if 'baseMetricV3' in cve['impact'] else '' - cveExplScore=cve['impact']['baseMetricV2']['exploitabilityScore'] if 'baseMetricV2' in cve['impact'] else '' + if cveId not in matchingCVE: + matchingCVE[cveId]={"id":cveId,"url":cveUrl,"description":cveDesc,"baseSeverity":cveBaseSeverity,"baseScore":cveBaseScore,"exploitabilityScore":cveExplScore} - if cveId not in matchingCVE: - matchingCVE[cveId]={"id":cveId,"url":cveUrl,"description":cveDesc,"baseSeverity":cveBaseSeverity,"baseScore":cveBaseScore,"exploitabilityScore":cveExplScore} + except: + return matchingCVE return matchingCVE # Returns all CVE for the given NGINX instance version def getNGINX(version="*"): - allCVE = __getFromNist(vendor="f5",product="nginx",version=version) matchingCVE={} - if version != '': - if 'result' in allCVE: - for cve in allCVE['result']['CVE_Items']: - allCPEMatches=cve['configurations']['nodes'][0]['cpe_match'] - - for cpeMatch in allCPEMatches: - # Found CVE match - cveId=cve['cve']['CVE_data_meta']['ID'] - cveUrl=cve['cve']['references']['reference_data'][0]['url'] - cveDesc=cve['cve']['description']['description_data'][0]['value'] - cveBaseSeverity=cve['impact']['baseMetricV3']['cvssV3']['baseSeverity'] if 'baseMetricV3' in cve['impact'] else '' - cveBaseScore=cve['impact']['baseMetricV3']['cvssV3']['baseScore'] if 'baseMetricV3' in cve['impact'] else '' - cveExplScore=cve['impact']['baseMetricV2']['exploitabilityScore'] if 'baseMetricV2' in cve['impact'] else '' - - if cveId not in matchingCVE: - matchingCVE[cveId]={"id":cveId,"url":cveUrl,"description":cveDesc,"baseSeverity":cveBaseSeverity,"baseScore":cveBaseScore,"exploitabilityScore":cveExplScore} - - cveF5 = getF5(product="nginx",version=version) + try: + allCVE = __getFromNist(vendor="f5",product="nginx",version=version) + except: + return matchingCVE - matchingCVE.update(cveF5) + if version != '': + if 'totalResults' in allCVE: + for cveTopLevel in allCVE['vulnerabilities']: + cve=cveTopLevel['cve'] + cveId=cve['id'] + cveUrl = [] + + if 'references' in cve: + for reference in cve['references']: + cveUrl.append(reference) + + if 'descriptions' in cve: + for desc in cve['descriptions']: + if desc['lang'] == 'en': + cveDesc = desc['value'] + + if 'metrics' in cve: + if 'cvssMetricV31' in cve['metrics']: + cveExplScore=cve['metrics']['cvssMetricV31'][0]['exploitabilityScore'] + cveBaseSeverity=cve['metrics']['cvssMetricV31'][0]['cvssData']['baseSeverity'] + cveBaseScore=cve['metrics']['cvssMetricV31'][0]['cvssData']['baseScore'] + elif 'cvssMetricV2' in cve['metrics']: + cveExplScore=cve['metrics']['cvssMetricV2'][0]['exploitabilityScore'] + cveBaseSeverity='' + cveBaseScore=cve['metrics']['cvssMetricV2'][0]['cvssData']['baseScore'] + else: + cveBaseSeverity='' + cveBaseScore='' + cveExplScore='' + + if cveId not in matchingCVE: + matchingCVE[cveId]={"id":cveId,"url":cveUrl,"description":cveDesc,"baseSeverity":cveBaseSeverity,"baseScore":cveBaseScore,"exploitabilityScore":cveExplScore} return matchingCVE