From ed93a7a681aeb42f7156cfd32ab78f4f3da7d014 Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Wed, 8 Jan 2020 17:06:05 +0600 Subject: [PATCH 01/18] Add empty CVEView and empty test. --- backend/device_registry/tests/test_all.py | 27 ++++++++++++++++++- backend/device_registry/urls.py | 2 ++ backend/device_registry/views.py | 32 +++++++++++++++++++++-- 3 files changed, 58 insertions(+), 3 deletions(-) diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index e6ab5b812..4324a6bb4 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -1407,4 +1407,29 @@ def test_incomplete(self): response = self.client.get(self.url) self.assertEqual(response.status_code, 200) self.assertListEqual([(0.4 + 0.5) / 2], response.context_data['trust_score_history']) - self.assertListEqual([1], response.context_data['ra_solved_history']) \ No newline at end of file + self.assertListEqual([1], response.context_data['ra_solved_history']) + + +class CVEViewTests(TestCase): + def setUp(self): + User = get_user_model() + self.user = User.objects.create_user('test') + self.user.set_password('123') + self.user.save() + self.client.login(username='test', password='123') + self.url = reverse('dashboard') + self.profile = Profile.objects.create(user=self.user) + + self.device0 = Device.objects.create( + device_id='device0.d.wott-dev.local', + owner=self.user + ) + self.url = reverse('cve') + self.device_url = reverse('device_cve', kwargs={'device_pk': self.device0.pk}) + + def test_empty(self): + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + + response = self.client.get(self.device_url) + self.assertEqual(response.status_code, 200) diff --git a/backend/device_registry/urls.py b/backend/device_registry/urls.py index 7ad248abf..255a655a8 100644 --- a/backend/device_registry/urls.py +++ b/backend/device_registry/urls.py @@ -108,6 +108,8 @@ name='ajax_policy_device_nr'), path('actions/', views.RecommendedActionsView.as_view(), name='actions'), path('devices//actions/', views.RecommendedActionsView.as_view(), name='device_actions'), + path('cve/', views.CVEView.as_view(), name='cve'), + path('devices//cve/', views.CVEView.as_view(), name='device_cve'), path( 'ajax/tags/autocomplete/', api_views.autocomplete_tags, diff --git a/backend/device_registry/views.py b/backend/device_registry/views.py index f34ee7f27..91f0b8305 100644 --- a/backend/device_registry/views.py +++ b/backend/device_registry/views.py @@ -1,11 +1,14 @@ +from __future__ import annotations + import json import uuid from collections import defaultdict +from typing import NamedTuple, List from django.contrib.auth.decorators import login_required from django.contrib.auth.mixins import LoginRequiredMixin from django.db import transaction -from django.db.models import Case, When +from django.db.models import Case, When, Count, F from django.db.models import Q, Sum, Avg, IntegerField from django.http import HttpResponseRedirect, HttpResponse, HttpResponseBadRequest, HttpResponseForbidden from django.shortcuts import get_object_or_404 @@ -19,7 +22,7 @@ from .forms import ClaimDeviceForm, DeviceAttrsForm, PortsForm, ConnectionsForm, DeviceMetadataForm from .forms import FirewallStateGlobalPolicyForm, GlobalPolicyForm from .models import Device, PortScan, FirewallState, get_bootstrap_color, PairingKey, \ - RecommendedAction, HistoryRecord + RecommendedAction, HistoryRecord, Vulnerability, DebPackage from .models import GlobalPolicy from .recommended_actions import ActionMeta, FirewallDisabledAction, Action, Severity @@ -585,3 +588,28 @@ def get_context_data(self, **kwargs): context['actions'] = actions context['device_name'] = device_name return context + + +class CVEView(LoginRequiredMixin, LoginTrackMixin, TemplateView): + template_name = 'dashboard.html' + + class AffectedPackage(NamedTuple): + name: str + hosts_affected: int + + class TableRow(NamedTuple): + cve_name: str + cve_url: str + cve_date: timezone.datetime + severity: str + packages: List[AffectedPackage] + + def get_context_data(self, **kwargs): + context = super().get_context_data(**kwargs) + + DebPackage.objects.filter(vulnerabilities__isnull=False)\ + .annotate(cve_name=F('vulnerabilities__name'), + cve_severity=F('vulnerabilities__urgency'), + hosts_affected=Count('device')) + + return context From d3b439374f10936226ef5f1d2aa08ce1a66ea52f Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Wed, 8 Jan 2020 20:32:47 +0600 Subject: [PATCH 02/18] Meaningful tests, grouping, sorting. --- backend/device_registry/tests/test_all.py | 53 +++++++++++++++++++++++ backend/device_registry/views.py | 51 ++++++++++++++++++---- 2 files changed, 95 insertions(+), 9 deletions(-) diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index 4324a6bb4..501ec5690 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -23,6 +23,7 @@ GlobalPolicy, PairingKey, Vulnerability, RecommendedAction from device_registry.forms import DeviceAttrsForm, PortsForm, ConnectionsForm, FirewallStateGlobalPolicyForm from device_registry.forms import GlobalPolicyForm +from device_registry.views import CVEView from profile_page.models import Profile @@ -1424,12 +1425,64 @@ def setUp(self): device_id='device0.d.wott-dev.local', owner=self.user ) + self.device1 = Device.objects.create( + device_id='device1.d.wott-dev.local', + owner=self.user + ) self.url = reverse('cve') self.device_url = reverse('device_cve', kwargs={'device_pk': self.device0.pk}) def test_empty(self): + packages = [ + DebPackage(name='one_first', version='version_one', source_name='one_source', source_version='one_version', + arch=DebPackage.Arch.i386), + DebPackage(name='one_second', version='version_one', source_name='one_source', source_version='one_version', + arch=DebPackage.Arch.i386), + DebPackage(name='two_first', version='version_two', source_name='two_source', source_version='two_version', + arch=DebPackage.Arch.i386), + DebPackage(name='two_second', version='version_two', source_name='two_source', source_version='two_version', + arch=DebPackage.Arch.i386), + ] + vulns = [ + Vulnerability(os_release_codename='stretch', name='CVE-2018-1', package='one_source', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.LOW.value, fix_available=True), + Vulnerability(os_release_codename='stretch', name='CVE-2018-2', package='one_source', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.LOW.value, fix_available=True), + Vulnerability(os_release_codename='stretch', name='CVE-2018-3', package='one_source', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.LOW.value, fix_available=False) + ] + DebPackage.objects.bulk_create(packages) + Vulnerability.objects.bulk_create(vulns) + packages[0].vulnerabilities.set(vulns) + packages[1].vulnerabilities.set(vulns[1:]) + + self.device0.deb_packages.set(packages) + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + self.assertListEqual(response.context_data['table_rows'], [ + CVEView.TableRow('CVE-2018-2', '', Vulnerability.Urgency.LOW, [ + CVEView.AffectedPackage('one_first', 1), + CVEView.AffectedPackage('one_second', 1) + ]), + CVEView.TableRow('CVE-2018-1', '', Vulnerability.Urgency.LOW, [ + CVEView.AffectedPackage('one_first', 1), + # CVEView.AffectedPackage('one_second', 1) + ]), + ]) + + self.device1.deb_packages.set(packages[1:]) response = self.client.get(self.url) self.assertEqual(response.status_code, 200) + self.assertListEqual(response.context_data['table_rows'], [ + CVEView.TableRow('CVE-2018-2', '', Vulnerability.Urgency.LOW, [ + CVEView.AffectedPackage('one_first', 1), + CVEView.AffectedPackage('one_second', 2) + ]), + CVEView.TableRow('CVE-2018-1', '', Vulnerability.Urgency.LOW, [ + CVEView.AffectedPackage('one_first', 1), + # CVEView.AffectedPackage('one_second', 0) + ]), + ]) response = self.client.get(self.device_url) self.assertEqual(response.status_code, 200) diff --git a/backend/device_registry/views.py b/backend/device_registry/views.py index 91f0b8305..0064ace87 100644 --- a/backend/device_registry/views.py +++ b/backend/device_registry/views.py @@ -1,5 +1,3 @@ -from __future__ import annotations - import json import uuid from collections import defaultdict @@ -600,16 +598,51 @@ class AffectedPackage(NamedTuple): class TableRow(NamedTuple): cve_name: str cve_url: str - cve_date: timezone.datetime - severity: str - packages: List[AffectedPackage] + # cve_date: timezone.datetime + urgency: Vulnerability.Urgency + packages: List[NamedTuple] + + @property + def key(self): + urgencies = { + Vulnerability.Urgency.HIGH: 3, + Vulnerability.Urgency.MEDIUM: 2, + Vulnerability.Urgency.LOW: 1, + Vulnerability.Urgency.NONE: 0 + } + return urgencies[self.urgency], sum([p.hosts_affected for p in self.packages]) + + @property + def severity(self): + severities = { + Vulnerability.Urgency.HIGH: 'High', + Vulnerability.Urgency.MEDIUM: 'Medium', + Vulnerability.Urgency.LOW: 'Low', + Vulnerability.Urgency.NONE: '' + } + return severities[self.urgency] def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - DebPackage.objects.filter(vulnerabilities__isnull=False)\ - .annotate(cve_name=F('vulnerabilities__name'), - cve_severity=F('vulnerabilities__urgency'), - hosts_affected=Count('device')) + vulns = Vulnerability.objects.filter(debpackage__device__owner=self.request.user, fix_available=True) + vulns = {v.name: v for v in vulns} + packages = DebPackage.objects.filter(vulnerabilities__isnull=False, vulnerabilities__fix_available=True)\ + .annotate(cve_name=F('vulnerabilities__name'), hosts_affected=Count('device')) + + rows = defaultdict(set) + for p in packages: + rows[p.cve_name].add(p) + + # TODO: sort by CVE severity and sum of hosts_affected + + table_rows = [] + for cve_name, r in rows.items(): + plist = [self.AffectedPackage(p.name, p.hosts_affected) for p in r] # TODO: sort by hosts_affected + table_rows.append(self.TableRow(cve_name, '', Vulnerability.Urgency(vulns[cve_name].urgency), plist)) + + context['table_rows'] = sorted(table_rows, + key=lambda r: r.key, + reverse=True) return context From 2af3933df35efccbb78fa0cf613488240aed3ec1 Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Thu, 9 Jan 2020 16:52:06 +0600 Subject: [PATCH 03/18] Sort by CVE urgency. Split tests into smaller pieces. --- backend/device_registry/tests/test_all.py | 54 ++++++++++++++++------- backend/device_registry/views.py | 38 +++++++--------- 2 files changed, 54 insertions(+), 38 deletions(-) diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index 501ec5690..ecfe0a3c2 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -1418,7 +1418,6 @@ def setUp(self): self.user.set_password('123') self.user.save() self.client.login(username='test', password='123') - self.url = reverse('dashboard') self.profile = Profile.objects.create(user=self.user) self.device0 = Device.objects.create( @@ -1432,8 +1431,7 @@ def setUp(self): self.url = reverse('cve') self.device_url = reverse('device_cve', kwargs={'device_pk': self.device0.pk}) - def test_empty(self): - packages = [ + self.packages = [ DebPackage(name='one_first', version='version_one', source_name='one_source', source_version='one_version', arch=DebPackage.Arch.i386), DebPackage(name='one_second', version='version_one', source_name='one_source', source_version='one_version', @@ -1443,7 +1441,7 @@ def test_empty(self): DebPackage(name='two_second', version='version_two', source_name='two_source', source_version='two_version', arch=DebPackage.Arch.i386), ] - vulns = [ + self.vulns = [ Vulnerability(os_release_codename='stretch', name='CVE-2018-1', package='one_source', is_binary=False, other_versions=[], urgency=Vulnerability.Urgency.LOW.value, fix_available=True), Vulnerability(os_release_codename='stretch', name='CVE-2018-2', package='one_source', is_binary=False, @@ -1451,38 +1449,64 @@ def test_empty(self): Vulnerability(os_release_codename='stretch', name='CVE-2018-3', package='one_source', is_binary=False, other_versions=[], urgency=Vulnerability.Urgency.LOW.value, fix_available=False) ] - DebPackage.objects.bulk_create(packages) - Vulnerability.objects.bulk_create(vulns) - packages[0].vulnerabilities.set(vulns) - packages[1].vulnerabilities.set(vulns[1:]) + DebPackage.objects.bulk_create(self.packages) + Vulnerability.objects.bulk_create(self.vulns) + self.device0.deb_packages.set(self.packages) + + def test_sort_package_hosts_affected(self): + self.packages[0].vulnerabilities.set(self.vulns) + self.packages[1].vulnerabilities.set(self.vulns[1:]) + self.device1.deb_packages.set(self.packages[1:]) - self.device0.deb_packages.set(packages) response = self.client.get(self.url) self.assertEqual(response.status_code, 200) self.assertListEqual(response.context_data['table_rows'], [ CVEView.TableRow('CVE-2018-2', '', Vulnerability.Urgency.LOW, [ + # These two AffectedPackage's should be sorted by hosts_affected + CVEView.AffectedPackage('one_second', 2), + CVEView.AffectedPackage('one_first', 1) + ]), + CVEView.TableRow('CVE-2018-1', '', Vulnerability.Urgency.LOW, [ + CVEView.AffectedPackage('one_first', 1), + ]) + ]) + + def test_sort_urgency(self): + self.vulns[1].urgency = Vulnerability.Urgency.HIGH.value + self.vulns[1].save() + self.packages[0].vulnerabilities.set(self.vulns) + self.packages[1].vulnerabilities.set(self.vulns) + response = self.client.get(self.url) + self.assertEqual(response.status_code, 200) + self.assertListEqual(response.context_data['table_rows'], [ + # These two TableRow's should be sorted by urgency + CVEView.TableRow('CVE-2018-2', '', Vulnerability.Urgency.HIGH, [ CVEView.AffectedPackage('one_first', 1), CVEView.AffectedPackage('one_second', 1) ]), CVEView.TableRow('CVE-2018-1', '', Vulnerability.Urgency.LOW, [ CVEView.AffectedPackage('one_first', 1), - # CVEView.AffectedPackage('one_second', 1) - ]), + CVEView.AffectedPackage('one_second', 1) + ]) ]) - self.device1.deb_packages.set(packages[1:]) + def test_sort_total_hosts_affected(self): + self.packages[0].vulnerabilities.set(self.vulns) + self.packages[1].vulnerabilities.set(self.vulns[1:]) + response = self.client.get(self.url) self.assertEqual(response.status_code, 200) self.assertListEqual(response.context_data['table_rows'], [ + # These two TableRow's should be sorted by the sum of hosts_affected CVEView.TableRow('CVE-2018-2', '', Vulnerability.Urgency.LOW, [ CVEView.AffectedPackage('one_first', 1), - CVEView.AffectedPackage('one_second', 2) + CVEView.AffectedPackage('one_second', 1) ]), CVEView.TableRow('CVE-2018-1', '', Vulnerability.Urgency.LOW, [ CVEView.AffectedPackage('one_first', 1), - # CVEView.AffectedPackage('one_second', 0) - ]), + ]) ]) + def test_empty(self): response = self.client.get(self.device_url) self.assertEqual(response.status_code, 200) diff --git a/backend/device_registry/views.py b/backend/device_registry/views.py index 0064ace87..240d2f13f 100644 --- a/backend/device_registry/views.py +++ b/backend/device_registry/views.py @@ -600,27 +600,21 @@ class TableRow(NamedTuple): cve_url: str # cve_date: timezone.datetime urgency: Vulnerability.Urgency - packages: List[NamedTuple] + packages: List[NamedTuple] # Actually it's List[AffectedPackage] + urgencies = { + Vulnerability.Urgency.HIGH: (3, 'High'), + Vulnerability.Urgency.MEDIUM: (2, 'Medium'), + Vulnerability.Urgency.LOW: (1, 'Low'), + Vulnerability.Urgency.NONE: (0, '') + } @property def key(self): - urgencies = { - Vulnerability.Urgency.HIGH: 3, - Vulnerability.Urgency.MEDIUM: 2, - Vulnerability.Urgency.LOW: 1, - Vulnerability.Urgency.NONE: 0 - } - return urgencies[self.urgency], sum([p.hosts_affected for p in self.packages]) + return self.urgencies[self.urgency][0], sum([p.hosts_affected for p in self.packages]) @property def severity(self): - severities = { - Vulnerability.Urgency.HIGH: 'High', - Vulnerability.Urgency.MEDIUM: 'Medium', - Vulnerability.Urgency.LOW: 'Low', - Vulnerability.Urgency.NONE: '' - } - return severities[self.urgency] + return self.urgencies[self.urgency][1] def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -630,19 +624,17 @@ def get_context_data(self, **kwargs): packages = DebPackage.objects.filter(vulnerabilities__isnull=False, vulnerabilities__fix_available=True)\ .annotate(cve_name=F('vulnerabilities__name'), hosts_affected=Count('device')) - rows = defaultdict(set) + packages_by_cve = defaultdict(set) for p in packages: - rows[p.cve_name].add(p) - - # TODO: sort by CVE severity and sum of hosts_affected + packages_by_cve[p.cve_name].add(p) table_rows = [] - for cve_name, r in rows.items(): - plist = [self.AffectedPackage(p.name, p.hosts_affected) for p in r] # TODO: sort by hosts_affected + for cve_name, package in packages_by_cve.items(): + plist = sorted([self.AffectedPackage(p.name, p.hosts_affected) for p in package], + key=lambda p: p.hosts_affected, reverse=True) table_rows.append(self.TableRow(cve_name, '', Vulnerability.Urgency(vulns[cve_name].urgency), plist)) context['table_rows'] = sorted(table_rows, - key=lambda r: r.key, - reverse=True) + key=lambda r: r.key, reverse=True) return context From 25290cbf0342a770014b43cd7f69fc3450bb5ea7 Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Thu, 9 Jan 2020 18:36:20 +0600 Subject: [PATCH 04/18] Convert Vulnerability.Urgency into integer, add pub_date. Select max vulnerability urgency and pub_date in CVEView. --- .../celery_tasks/ubuntu_cve.py | 1 + .../migrations/0059_auto_20190920_1011.py | 2 +- .../migrations/0079_auto_20200109_1137.py | 34 +++++++++++++++++++ .../migrations/0080_vulnerability_pub_date.py | 18 ++++++++++ backend/device_registry/models.py | 13 +++---- backend/device_registry/tests/test_all.py | 24 ++++++------- backend/device_registry/views.py | 33 ++++++++++-------- 7 files changed, 92 insertions(+), 33 deletions(-) create mode 100644 backend/device_registry/migrations/0079_auto_20200109_1137.py create mode 100644 backend/device_registry/migrations/0080_vulnerability_pub_date.py diff --git a/backend/device_registry/celery_tasks/ubuntu_cve.py b/backend/device_registry/celery_tasks/ubuntu_cve.py index a432a2946..0facab0e9 100644 --- a/backend/device_registry/celery_tasks/ubuntu_cve.py +++ b/backend/device_registry/celery_tasks/ubuntu_cve.py @@ -291,6 +291,7 @@ def fetch_vulnerabilities(): 'medium': Vulnerability.Urgency.MEDIUM, 'high': Vulnerability.Urgency.HIGH, }.get(header['Priority'], Vulnerability.Urgency.NONE), + pub_date=header['PublicDate'], remote=None, fix_available=(status == 'fixed'), os_release_codename=codename diff --git a/backend/device_registry/migrations/0059_auto_20190920_1011.py b/backend/device_registry/migrations/0059_auto_20190920_1011.py index 8d17a499d..6b5d1d38f 100644 --- a/backend/device_registry/migrations/0059_auto_20190920_1011.py +++ b/backend/device_registry/migrations/0059_auto_20190920_1011.py @@ -21,7 +21,7 @@ class Migration(migrations.Migration): ('is_binary', models.BooleanField()), ('unstable_version', models.CharField(blank=True, max_length=64)), ('other_versions', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=64), blank=True, size=None)), - ('urgency', models.CharField(choices=[(device_registry.models.Vulnerability.Urgency(' '), ' '), (device_registry.models.Vulnerability.Urgency('L'), 'L'), (device_registry.models.Vulnerability.Urgency('M'), 'M'), (device_registry.models.Vulnerability.Urgency('H'), 'H')], max_length=64)), + ('urgency', models.CharField(max_length=64)), ('remote', models.BooleanField(null=True)), ('fix_available', models.BooleanField()), ], diff --git a/backend/device_registry/migrations/0079_auto_20200109_1137.py b/backend/device_registry/migrations/0079_auto_20200109_1137.py new file mode 100644 index 000000000..a3e1c7942 --- /dev/null +++ b/backend/device_registry/migrations/0079_auto_20200109_1137.py @@ -0,0 +1,34 @@ +# Generated by Django 2.2.6 on 2020-01-09 11:37 + +import device_registry.models +from django.db import migrations, models +from django.db.models import Case, CharField, Value, When + + +def convert_urgencies(apps, schema_editor): + urgencies = { + 'Urgency.NONE': 0, + 'Urgency.LOW': 1, + 'Urgency.MEDIUM': 2, + 'Urgency.HIGH': 3, + } + Vulnerability = apps.get_model('device_registry', 'Vulnerability') + Vulnerability.objects.update(urgency=Case( + *[When(urgency=k, then=Value(v)) for k, v in urgencies.items()] + )) + + +class Migration(migrations.Migration): + + dependencies = [ + ('device_registry', '0078_merge_20200107_1719'), + ] + + operations = [ + migrations.RunPython(convert_urgencies), + migrations.AlterField( + model_name='vulnerability', + name='urgency', + field=models.PositiveSmallIntegerField(choices=[(device_registry.models.Vulnerability.Urgency(0), 0), (device_registry.models.Vulnerability.Urgency(1), 1), (device_registry.models.Vulnerability.Urgency(2), 2), (device_registry.models.Vulnerability.Urgency(3), 3)]), + ), + ] diff --git a/backend/device_registry/migrations/0080_vulnerability_pub_date.py b/backend/device_registry/migrations/0080_vulnerability_pub_date.py new file mode 100644 index 000000000..60d33920b --- /dev/null +++ b/backend/device_registry/migrations/0080_vulnerability_pub_date.py @@ -0,0 +1,18 @@ +# Generated by Django 2.2.6 on 2020-01-09 12:25 + +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ('device_registry', '0079_auto_20200109_1137'), + ] + + operations = [ + migrations.AddField( + model_name='vulnerability', + name='pub_date', + field=models.DateField(null=True), + ), + ] diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py index b35705be5..f38973c76 100644 --- a/backend/device_registry/models.py +++ b/backend/device_registry/models.py @@ -826,11 +826,11 @@ def __lt__(self, other): def __eq__(self, other): return apt_pkg.version_compare(self.__asString, other.__asString) == 0 - class Urgency(Enum): - NONE = ' ' - LOW = 'L' - MEDIUM = 'M' - HIGH = 'H' + class Urgency(IntEnum): + NONE = 0 + LOW = 1 + MEDIUM = 2 + HIGH = 3 os_release_codename = models.CharField(max_length=64, db_index=True) name = models.CharField(max_length=64) @@ -838,9 +838,10 @@ class Urgency(Enum): is_binary = models.BooleanField() unstable_version = models.CharField(max_length=64, blank=True) other_versions = ArrayField(models.CharField(max_length=64), blank=True) - urgency = models.CharField(max_length=64, choices=[(tag, tag.value) for tag in Urgency]) + urgency = models.PositiveSmallIntegerField(choices=[(tag, tag.value) for tag in Urgency]) remote = models.BooleanField(null=True) fix_available = models.BooleanField() + pub_date = models.DateField(null=True) def is_vulnerable(self, src_ver): if self.unstable_version: diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index ecfe0a3c2..2eea9af6c 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -1443,11 +1443,11 @@ def setUp(self): ] self.vulns = [ Vulnerability(os_release_codename='stretch', name='CVE-2018-1', package='one_source', is_binary=False, - other_versions=[], urgency=Vulnerability.Urgency.LOW.value, fix_available=True), + other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=True), Vulnerability(os_release_codename='stretch', name='CVE-2018-2', package='one_source', is_binary=False, - other_versions=[], urgency=Vulnerability.Urgency.LOW.value, fix_available=True), + other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=True), Vulnerability(os_release_codename='stretch', name='CVE-2018-3', package='one_source', is_binary=False, - other_versions=[], urgency=Vulnerability.Urgency.LOW.value, fix_available=False) + other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=False) ] DebPackage.objects.bulk_create(self.packages) Vulnerability.objects.bulk_create(self.vulns) @@ -1461,18 +1461,18 @@ def test_sort_package_hosts_affected(self): response = self.client.get(self.url) self.assertEqual(response.status_code, 200) self.assertListEqual(response.context_data['table_rows'], [ - CVEView.TableRow('CVE-2018-2', '', Vulnerability.Urgency.LOW, [ + CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ # These two AffectedPackage's should be sorted by hosts_affected CVEView.AffectedPackage('one_second', 2), CVEView.AffectedPackage('one_first', 1) ]), - CVEView.TableRow('CVE-2018-1', '', Vulnerability.Urgency.LOW, [ - CVEView.AffectedPackage('one_first', 1), + CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ + CVEView.AffectedPackage('one_first', 1) ]) ]) def test_sort_urgency(self): - self.vulns[1].urgency = Vulnerability.Urgency.HIGH.value + self.vulns[1].urgency = Vulnerability.Urgency.HIGH self.vulns[1].save() self.packages[0].vulnerabilities.set(self.vulns) self.packages[1].vulnerabilities.set(self.vulns) @@ -1480,11 +1480,11 @@ def test_sort_urgency(self): self.assertEqual(response.status_code, 200) self.assertListEqual(response.context_data['table_rows'], [ # These two TableRow's should be sorted by urgency - CVEView.TableRow('CVE-2018-2', '', Vulnerability.Urgency.HIGH, [ + CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.HIGH, packages=[ CVEView.AffectedPackage('one_first', 1), CVEView.AffectedPackage('one_second', 1) ]), - CVEView.TableRow('CVE-2018-1', '', Vulnerability.Urgency.LOW, [ + CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ CVEView.AffectedPackage('one_first', 1), CVEView.AffectedPackage('one_second', 1) ]) @@ -1498,12 +1498,12 @@ def test_sort_total_hosts_affected(self): self.assertEqual(response.status_code, 200) self.assertListEqual(response.context_data['table_rows'], [ # These two TableRow's should be sorted by the sum of hosts_affected - CVEView.TableRow('CVE-2018-2', '', Vulnerability.Urgency.LOW, [ + CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ CVEView.AffectedPackage('one_first', 1), CVEView.AffectedPackage('one_second', 1) ]), - CVEView.TableRow('CVE-2018-1', '', Vulnerability.Urgency.LOW, [ - CVEView.AffectedPackage('one_first', 1), + CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ + CVEView.AffectedPackage('one_first', 1) ]) ]) diff --git a/backend/device_registry/views.py b/backend/device_registry/views.py index 240d2f13f..0f1ce6ec6 100644 --- a/backend/device_registry/views.py +++ b/backend/device_registry/views.py @@ -7,7 +7,7 @@ from django.contrib.auth.mixins import LoginRequiredMixin from django.db import transaction from django.db.models import Case, When, Count, F -from django.db.models import Q, Sum, Avg, IntegerField +from django.db.models import Q, Sum, Avg, IntegerField, Max from django.http import HttpResponseRedirect, HttpResponse, HttpResponseBadRequest, HttpResponseForbidden from django.shortcuts import get_object_or_404 from django.shortcuts import render @@ -597,30 +597,33 @@ class AffectedPackage(NamedTuple): class TableRow(NamedTuple): cve_name: str - cve_url: str - # cve_date: timezone.datetime urgency: Vulnerability.Urgency packages: List[NamedTuple] # Actually it's List[AffectedPackage] + cve_url: str + cve_date: timezone.datetime = None + urgencies = { - Vulnerability.Urgency.HIGH: (3, 'High'), - Vulnerability.Urgency.MEDIUM: (2, 'Medium'), - Vulnerability.Urgency.LOW: (1, 'Low'), - Vulnerability.Urgency.NONE: (0, '') + Vulnerability.Urgency.HIGH: 'High', + Vulnerability.Urgency.MEDIUM: 'Medium', + Vulnerability.Urgency.LOW: 'Low', + Vulnerability.Urgency.NONE: ' ' } @property def key(self): - return self.urgencies[self.urgency][0], sum([p.hosts_affected for p in self.packages]) + return self.urgency, sum([p.hosts_affected for p in self.packages]) @property def severity(self): - return self.urgencies[self.urgency][1] + return self.urgencies[self.urgency] def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) - vulns = Vulnerability.objects.filter(debpackage__device__owner=self.request.user, fix_available=True) - vulns = {v.name: v for v in vulns} + vulns = Vulnerability.objects.filter(debpackage__device__owner=self.request.user, fix_available=True) \ + .values('name').distinct().annotate(max_urgency=Max('urgency'), + pubdate=Max('pub_date')) + vuln_urgencies = {v['name']: (v['max_urgency'], v['pubdate']) for v in vulns} packages = DebPackage.objects.filter(vulnerabilities__isnull=False, vulnerabilities__fix_available=True)\ .annotate(cve_name=F('vulnerabilities__name'), hosts_affected=Count('device')) @@ -629,10 +632,12 @@ def get_context_data(self, **kwargs): packages_by_cve[p.cve_name].add(p) table_rows = [] - for cve_name, package in packages_by_cve.items(): - plist = sorted([self.AffectedPackage(p.name, p.hosts_affected) for p in package], + for cve_name, cve_packages in packages_by_cve.items(): + plist = sorted([self.AffectedPackage(p.name, p.hosts_affected) for p in cve_packages], key=lambda p: p.hosts_affected, reverse=True) - table_rows.append(self.TableRow(cve_name, '', Vulnerability.Urgency(vulns[cve_name].urgency), plist)) + urgency, cve_date = vuln_urgencies[cve_name] + table_rows.append(self.TableRow(cve_name=cve_name, cve_url='', urgency=urgency, + cve_date=cve_date, packages=plist)) context['table_rows'] = sorted(table_rows, key=lambda r: r.key, reverse=True) From d3d2d7d43f07c8b98658dfc5882b3bf11c4b8ede Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Thu, 9 Jan 2020 18:59:25 +0600 Subject: [PATCH 05/18] CVEView: filter packages by device owner. --- backend/device_registry/views.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/backend/device_registry/views.py b/backend/device_registry/views.py index 0f1ce6ec6..7cd9e0338 100644 --- a/backend/device_registry/views.py +++ b/backend/device_registry/views.py @@ -624,7 +624,9 @@ def get_context_data(self, **kwargs): .values('name').distinct().annotate(max_urgency=Max('urgency'), pubdate=Max('pub_date')) vuln_urgencies = {v['name']: (v['max_urgency'], v['pubdate']) for v in vulns} - packages = DebPackage.objects.filter(vulnerabilities__isnull=False, vulnerabilities__fix_available=True)\ + packages = DebPackage.objects.filter(device__owner=self.request.user, + vulnerabilities__isnull=False, + vulnerabilities__fix_available=True)\ .annotate(cve_name=F('vulnerabilities__name'), hosts_affected=Count('device')) packages_by_cve = defaultdict(set) From 10090b456aa2dffca49f0a8bc004fba751171efa Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Thu, 9 Jan 2020 19:21:05 +0600 Subject: [PATCH 06/18] Add affected devices. --- backend/device_registry/tests/test_all.py | 20 ++++++++++---------- backend/device_registry/views.py | 15 ++++++++------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index 2eea9af6c..8b9eb930c 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -1463,11 +1463,11 @@ def test_sort_package_hosts_affected(self): self.assertListEqual(response.context_data['table_rows'], [ CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ # These two AffectedPackage's should be sorted by hosts_affected - CVEView.AffectedPackage('one_second', 2), - CVEView.AffectedPackage('one_first', 1) + CVEView.AffectedPackage('one_second', [self.device0.pk, self.device1.pk]), + CVEView.AffectedPackage('one_first', [self.device0.pk]) ]), CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ - CVEView.AffectedPackage('one_first', 1) + CVEView.AffectedPackage('one_first', [self.device0.pk]) ]) ]) @@ -1481,12 +1481,12 @@ def test_sort_urgency(self): self.assertListEqual(response.context_data['table_rows'], [ # These two TableRow's should be sorted by urgency CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.HIGH, packages=[ - CVEView.AffectedPackage('one_first', 1), - CVEView.AffectedPackage('one_second', 1) + CVEView.AffectedPackage('one_first', [self.device0.pk]), + CVEView.AffectedPackage('one_second', [self.device0.pk]) ]), CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ - CVEView.AffectedPackage('one_first', 1), - CVEView.AffectedPackage('one_second', 1) + CVEView.AffectedPackage('one_first', [self.device0.pk]), + CVEView.AffectedPackage('one_second', [self.device0.pk]) ]) ]) @@ -1499,11 +1499,11 @@ def test_sort_total_hosts_affected(self): self.assertListEqual(response.context_data['table_rows'], [ # These two TableRow's should be sorted by the sum of hosts_affected CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ - CVEView.AffectedPackage('one_first', 1), - CVEView.AffectedPackage('one_second', 1) + CVEView.AffectedPackage('one_first', [self.device0.pk]), + CVEView.AffectedPackage('one_second', [self.device0.pk]) ]), CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ - CVEView.AffectedPackage('one_first', 1) + CVEView.AffectedPackage('one_first', [self.device0.pk]) ]) ]) diff --git a/backend/device_registry/views.py b/backend/device_registry/views.py index 7cd9e0338..2d44d1ad6 100644 --- a/backend/device_registry/views.py +++ b/backend/device_registry/views.py @@ -593,7 +593,7 @@ class CVEView(LoginRequiredMixin, LoginTrackMixin, TemplateView): class AffectedPackage(NamedTuple): name: str - hosts_affected: int + devices: List[int] class TableRow(NamedTuple): cve_name: str @@ -611,7 +611,7 @@ class TableRow(NamedTuple): @property def key(self): - return self.urgency, sum([p.hosts_affected for p in self.packages]) + return self.urgency, sum([len(p.devices) for p in self.packages]) @property def severity(self): @@ -623,11 +623,11 @@ def get_context_data(self, **kwargs): vulns = Vulnerability.objects.filter(debpackage__device__owner=self.request.user, fix_available=True) \ .values('name').distinct().annotate(max_urgency=Max('urgency'), pubdate=Max('pub_date')) - vuln_urgencies = {v['name']: (v['max_urgency'], v['pubdate']) for v in vulns} + vuln_info = {v['name']: (v['max_urgency'], v['pubdate']) for v in vulns} packages = DebPackage.objects.filter(device__owner=self.request.user, vulnerabilities__isnull=False, vulnerabilities__fix_available=True)\ - .annotate(cve_name=F('vulnerabilities__name'), hosts_affected=Count('device')) + .annotate(cve_name=F('vulnerabilities__name')) packages_by_cve = defaultdict(set) for p in packages: @@ -635,9 +635,10 @@ def get_context_data(self, **kwargs): table_rows = [] for cve_name, cve_packages in packages_by_cve.items(): - plist = sorted([self.AffectedPackage(p.name, p.hosts_affected) for p in cve_packages], - key=lambda p: p.hosts_affected, reverse=True) - urgency, cve_date = vuln_urgencies[cve_name] + plist = sorted([self.AffectedPackage(p.name, list(p.device_set.values_list('pk', flat=True))) + for p in cve_packages], + key=lambda p: len(p.devices), reverse=True) + urgency, cve_date = vuln_info[cve_name] table_rows.append(self.TableRow(cve_name=cve_name, cve_url='', urgency=urgency, cve_date=cve_date, packages=plist)) From b0eae8000adc68f5f299b5db95b8d8a3910e0285 Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Thu, 9 Jan 2020 19:38:00 +0600 Subject: [PATCH 07/18] Add AffectedPackage.device_urls and upgrade_command to be used in template. --- backend/device_registry/tests/test_all.py | 21 +++++++++++---------- backend/device_registry/views.py | 15 +++++++++++++-- 2 files changed, 24 insertions(+), 12 deletions(-) diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index 8b9eb930c..fae0fa2af 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -1463,11 +1463,11 @@ def test_sort_package_hosts_affected(self): self.assertListEqual(response.context_data['table_rows'], [ CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ # These two AffectedPackage's should be sorted by hosts_affected - CVEView.AffectedPackage('one_second', [self.device0.pk, self.device1.pk]), - CVEView.AffectedPackage('one_first', [self.device0.pk]) + CVEView.AffectedPackage('one_second', [self.device0, self.device1]), + CVEView.AffectedPackage('one_first', [self.device0]) ]), CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ - CVEView.AffectedPackage('one_first', [self.device0.pk]) + CVEView.AffectedPackage('one_first', [self.device0]) ]) ]) @@ -1481,14 +1481,15 @@ def test_sort_urgency(self): self.assertListEqual(response.context_data['table_rows'], [ # These two TableRow's should be sorted by urgency CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.HIGH, packages=[ - CVEView.AffectedPackage('one_first', [self.device0.pk]), - CVEView.AffectedPackage('one_second', [self.device0.pk]) + CVEView.AffectedPackage('one_first', [self.device0]), + CVEView.AffectedPackage('one_second', [self.device0]) ]), CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ - CVEView.AffectedPackage('one_first', [self.device0.pk]), - CVEView.AffectedPackage('one_second', [self.device0.pk]) + CVEView.AffectedPackage('one_first', [self.device0]), + CVEView.AffectedPackage('one_second', [self.device0]) ]) ]) + self.assertEqual(len(response.context_data['table_rows'][0].packages[0].device_urls), 1) def test_sort_total_hosts_affected(self): self.packages[0].vulnerabilities.set(self.vulns) @@ -1499,11 +1500,11 @@ def test_sort_total_hosts_affected(self): self.assertListEqual(response.context_data['table_rows'], [ # These two TableRow's should be sorted by the sum of hosts_affected CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ - CVEView.AffectedPackage('one_first', [self.device0.pk]), - CVEView.AffectedPackage('one_second', [self.device0.pk]) + CVEView.AffectedPackage('one_first', [self.device0]), + CVEView.AffectedPackage('one_second', [self.device0]) ]), CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ - CVEView.AffectedPackage('one_first', [self.device0.pk]) + CVEView.AffectedPackage('one_first', [self.device0]) ]) ]) diff --git a/backend/device_registry/views.py b/backend/device_registry/views.py index 2d44d1ad6..fc8199607 100644 --- a/backend/device_registry/views.py +++ b/backend/device_registry/views.py @@ -593,7 +593,18 @@ class CVEView(LoginRequiredMixin, LoginTrackMixin, TemplateView): class AffectedPackage(NamedTuple): name: str - devices: List[int] + devices: List[Device] + + @property + def device_urls(self): + return [( + d.get_name(), + reverse('device_cve', kwargs={'device_pk': d.pk}) + ) for d in self.devices] + + @property + def upgrade_command(self): + return f'apt-get update && apt-get install -y {self.name}' class TableRow(NamedTuple): cve_name: str @@ -635,7 +646,7 @@ def get_context_data(self, **kwargs): table_rows = [] for cve_name, cve_packages in packages_by_cve.items(): - plist = sorted([self.AffectedPackage(p.name, list(p.device_set.values_list('pk', flat=True))) + plist = sorted([self.AffectedPackage(p.name, list(p.device_set.filter(owner=self.request.user))) for p in cve_packages], key=lambda p: len(p.devices), reverse=True) urgency, cve_date = vuln_info[cve_name] From 9f5c4b9953d723d8372f851322772c70ff8d8f96 Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Thu, 9 Jan 2020 19:43:56 +0600 Subject: [PATCH 08/18] Test that cve_date is returned. --- backend/device_registry/tests/test_all.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index fae0fa2af..c2a2fa634 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -1441,11 +1441,12 @@ def setUp(self): DebPackage(name='two_second', version='version_two', source_name='two_source', source_version='two_version', arch=DebPackage.Arch.i386), ] + self.today = timezone.now().date() self.vulns = [ Vulnerability(os_release_codename='stretch', name='CVE-2018-1', package='one_source', is_binary=False, other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=True), - Vulnerability(os_release_codename='stretch', name='CVE-2018-2', package='one_source', is_binary=False, - other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=True), + Vulnerability(os_release_codename='buster', name='CVE-2018-2', package='one_source', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=True, pub_date=self.today), Vulnerability(os_release_codename='stretch', name='CVE-2018-3', package='one_source', is_binary=False, other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=False) ] @@ -1465,7 +1466,7 @@ def test_sort_package_hosts_affected(self): # These two AffectedPackage's should be sorted by hosts_affected CVEView.AffectedPackage('one_second', [self.device0, self.device1]), CVEView.AffectedPackage('one_first', [self.device0]) - ]), + ], cve_date=self.today), CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ CVEView.AffectedPackage('one_first', [self.device0]) ]) @@ -1483,7 +1484,7 @@ def test_sort_urgency(self): CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.HIGH, packages=[ CVEView.AffectedPackage('one_first', [self.device0]), CVEView.AffectedPackage('one_second', [self.device0]) - ]), + ], cve_date=self.today), CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ CVEView.AffectedPackage('one_first', [self.device0]), CVEView.AffectedPackage('one_second', [self.device0]) @@ -1502,7 +1503,7 @@ def test_sort_total_hosts_affected(self): CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ CVEView.AffectedPackage('one_first', [self.device0]), CVEView.AffectedPackage('one_second', [self.device0]) - ]), + ], cve_date=self.today), CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ CVEView.AffectedPackage('one_first', [self.device0]) ]) From f5c2573279b5db802cab6880568f3c93a97acce1 Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Fri, 10 Jan 2020 11:15:07 +0600 Subject: [PATCH 09/18] CVE page template, filter by device. --- backend/device_registry/templates/cve.html | 17 ++++++++++++++++ backend/device_registry/tests/test_all.py | 23 +++++++++++++++++++++- backend/device_registry/views.py | 19 ++++++++++++++---- 3 files changed, 54 insertions(+), 5 deletions(-) create mode 100644 backend/device_registry/templates/cve.html diff --git a/backend/device_registry/templates/cve.html b/backend/device_registry/templates/cve.html new file mode 100644 index 000000000..feee0273e --- /dev/null +++ b/backend/device_registry/templates/cve.html @@ -0,0 +1,17 @@ +{% extends "admin_base.html" %} + +{% block title %}WoTT - Claim Node{% endblock title %} + +{% block dashboard_title %} +

Claim Node

+{% endblock dashboard_title %} + +{% block admin_content %} + + This is CVE page. +{% endblock admin_content %} + +{% block scripts %} +{{ block.super }} + +{% endblock scripts %} \ No newline at end of file diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index c2a2fa634..e5a58b73b 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -1417,6 +1417,7 @@ def setUp(self): self.user = User.objects.create_user('test') self.user.set_password('123') self.user.save() + self.user_unrelated = User.objects.create_user('unrelated') self.client.login(username='test', password='123') self.profile = Profile.objects.create(user=self.user) @@ -1428,6 +1429,10 @@ def setUp(self): device_id='device1.d.wott-dev.local', owner=self.user ) + self.device_unrelated = Device.objects.create( + device_id='device-unrelated.d.wott-dev.local', + owner=self.user_unrelated + ) self.url = reverse('cve') self.device_url = reverse('device_cve', kwargs={'device_pk': self.device0.pk}) @@ -1453,6 +1458,7 @@ def setUp(self): DebPackage.objects.bulk_create(self.packages) Vulnerability.objects.bulk_create(self.vulns) self.device0.deb_packages.set(self.packages) + self.device_unrelated.deb_packages.set(self.packages) def test_sort_package_hosts_affected(self): self.packages[0].vulnerabilities.set(self.vulns) @@ -1509,6 +1515,21 @@ def test_sort_total_hosts_affected(self): ]) ]) - def test_empty(self): + def test_filter_device(self): + # The setup is the same as in test_sort_package_hosts_affected. + # The result should also be the same except without device1. + self.packages[0].vulnerabilities.set(self.vulns) + self.packages[1].vulnerabilities.set(self.vulns[1:]) + self.device1.deb_packages.set(self.packages[1:]) + response = self.client.get(self.device_url) self.assertEqual(response.status_code, 200) + self.assertListEqual(response.context_data['table_rows'], [ + CVEView.TableRow(cve_name='CVE-2018-2', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ + CVEView.AffectedPackage('one_first', [self.device0]), + CVEView.AffectedPackage('one_second', [self.device0]) + ], cve_date=self.today), + CVEView.TableRow(cve_name='CVE-2018-1', cve_url='', urgency=Vulnerability.Urgency.LOW, packages=[ + CVEView.AffectedPackage('one_first', [self.device0]) + ]) + ]) diff --git a/backend/device_registry/views.py b/backend/device_registry/views.py index fc8199607..b951229d1 100644 --- a/backend/device_registry/views.py +++ b/backend/device_registry/views.py @@ -589,7 +589,7 @@ def get_context_data(self, **kwargs): class CVEView(LoginRequiredMixin, LoginTrackMixin, TemplateView): - template_name = 'dashboard.html' + template_name = 'cve.html' class AffectedPackage(NamedTuple): name: str @@ -630,12 +630,22 @@ def severity(self): def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) + user = self.request.user + + device_pk = kwargs.get('device_pk') + if device_pk is not None: + device = get_object_or_404(Device, pk=device_pk, owner=user) + vuln_query = Q(debpackage__device__pk=device_pk) + packages_query = Q(device__pk=device_pk) + else: + vuln_query = Q(debpackage__device__owner=user) + packages_query = Q(device__owner=user) - vulns = Vulnerability.objects.filter(debpackage__device__owner=self.request.user, fix_available=True) \ + vulns = Vulnerability.objects.filter(vuln_query, fix_available=True) \ .values('name').distinct().annotate(max_urgency=Max('urgency'), pubdate=Max('pub_date')) vuln_info = {v['name']: (v['max_urgency'], v['pubdate']) for v in vulns} - packages = DebPackage.objects.filter(device__owner=self.request.user, + packages = DebPackage.objects.filter(packages_query, vulnerabilities__isnull=False, vulnerabilities__fix_available=True)\ .annotate(cve_name=F('vulnerabilities__name')) @@ -646,7 +656,8 @@ def get_context_data(self, **kwargs): table_rows = [] for cve_name, cve_packages in packages_by_cve.items(): - plist = sorted([self.AffectedPackage(p.name, list(p.device_set.filter(owner=self.request.user))) + plist = sorted([self.AffectedPackage(p.name, + [device] if device_pk else list(p.device_set.filter(owner=user))) for p in cve_packages], key=lambda p: len(p.devices), reverse=True) urgency, cve_date = vuln_info[cve_name] From d171ce830681467d4abe339d00c5012aefccee09 Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Fri, 10 Jan 2020 12:58:17 +0600 Subject: [PATCH 10/18] Working CVE page template. --- backend/device_registry/templates/cve.html | 56 +++++++++++++++++++++- backend/device_registry/views.py | 17 +++++-- 2 files changed, 68 insertions(+), 5 deletions(-) diff --git a/backend/device_registry/templates/cve.html b/backend/device_registry/templates/cve.html index feee0273e..6d77c64d0 100644 --- a/backend/device_registry/templates/cve.html +++ b/backend/device_registry/templates/cve.html @@ -3,12 +3,64 @@ {% block title %}WoTT - Claim Node{% endblock title %} {% block dashboard_title %} -

Claim Node

+

CVE list{% if device_name %} for {{ device_name }}{% endif %}

{% endblock dashboard_title %} {% block admin_content %} - This is CVE page. + + + + + + + {% if not device_name %} + + {% endif %} + + + + {% for row in table_rows %} + + + + + + {% if not device_name %} + + {% endif %} + + + {% endfor %} + +
CVEDateSeverityPackages AffectedNodes AffectedSolve
+ {{ row.cve_link.text }} + {{ row.cve_date|default_if_none:"N/A" }}{{ row.severity }} + {% for p in row.packages %} + {{ p.name }} +
+ {% endfor %} + 		+ {% for p in row.packages %} + + {{ p.device_urls|length }} + + +
+ {% endfor %} + 		+ {% for p in row.packages %} + Instructions + +
+ {% endfor %} +
{% endblock admin_content %} {% block scripts %} diff --git a/backend/device_registry/views.py b/backend/device_registry/views.py index b951229d1..af65bceb1 100644 --- a/backend/device_registry/views.py +++ b/backend/device_registry/views.py @@ -591,13 +591,17 @@ def get_context_data(self, **kwargs): class CVEView(LoginRequiredMixin, LoginTrackMixin, TemplateView): template_name = 'cve.html' + class Hyperlink(NamedTuple): + text: str + href: str + class AffectedPackage(NamedTuple): name: str devices: List[Device] @property def device_urls(self): - return [( + return [CVEView.Hyperlink( d.get_name(), reverse('device_cve', kwargs={'device_pk': d.pk}) ) for d in self.devices] @@ -617,7 +621,7 @@ class TableRow(NamedTuple): Vulnerability.Urgency.HIGH: 'High', Vulnerability.Urgency.MEDIUM: 'Medium', Vulnerability.Urgency.LOW: 'Low', - Vulnerability.Urgency.NONE: ' ' + Vulnerability.Urgency.NONE: 'N/A' } @property @@ -628,6 +632,10 @@ def key(self): def severity(self): return self.urgencies[self.urgency] + @property + def cve_link(self): + return CVEView.Hyperlink(self.cve_name, 'http://cve.mitre.org/cgi-bin/cvename.cgi?name='+self.cve_name) + def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) user = self.request.user @@ -638,6 +646,7 @@ def get_context_data(self, **kwargs): vuln_query = Q(debpackage__device__pk=device_pk) packages_query = Q(device__pk=device_pk) else: + device = None vuln_query = Q(debpackage__device__owner=user) packages_query = Q(device__owner=user) @@ -657,7 +666,7 @@ def get_context_data(self, **kwargs): table_rows = [] for cve_name, cve_packages in packages_by_cve.items(): plist = sorted([self.AffectedPackage(p.name, - [device] if device_pk else list(p.device_set.filter(owner=user))) + [device] if device else list(p.device_set.filter(owner=user))) for p in cve_packages], key=lambda p: len(p.devices), reverse=True) urgency, cve_date = vuln_info[cve_name] @@ -666,5 +675,7 @@ def get_context_data(self, **kwargs): context['table_rows'] = sorted(table_rows, key=lambda r: r.key, reverse=True) + if device: + context['device_name'] = device.get_name() return context From cc8479a1da2fe0d7a416ac681285bffefc13bc66 Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Fri, 10 Jan 2020 15:54:34 +0600 Subject: [PATCH 11/18] Move CVE table into a card container, template wording. --- backend/device_registry/templates/cve.html | 114 +++++++++++---------- 1 file changed, 61 insertions(+), 53 deletions(-) diff --git a/backend/device_registry/templates/cve.html b/backend/device_registry/templates/cve.html index 6d77c64d0..7b8c6b0f5 100644 --- a/backend/device_registry/templates/cve.html +++ b/backend/device_registry/templates/cve.html @@ -1,6 +1,6 @@ {% extends "admin_base.html" %} -{% block title %}WoTT - Claim Node{% endblock title %} +{% block title %}WoTT - CVE list{% endblock title %} {% block dashboard_title %}

CVE list{% if device_name %} for {{ device_name }}{% endif %}

@@ -8,59 +8,67 @@

CVE list{% if device_name %} for {{ device_name }}{ {% block admin_content %} - - - - - - - {% if not device_name %} - - {% endif %} - - - - {% for row in table_rows %} - - - - - - {% if not device_name %} - - {% endif %} - - - {% endfor %} - -
CVEDateSeverityPackages AffectedNodes AffectedSolve
- {{ row.cve_link.text }} - {{ row.cve_date|default_if_none:"N/A" }}{{ row.severity }} - {% for p in row.packages %} - {{ p.name }} -
- {% endfor %} - 		- {% for p in row.packages %} - - {{ p.device_urls|length }} - - -
- {% endfor %} - 		- {% for p in row.packages %} - Instructions - -
- {% endfor %} -
+ + {% if not device_name %} + + {% for p in row.packages %} + + {{ p.device_urls|length }} + + +
+ {% endfor %} + + {% endif %} + + {% for p in row.packages %} + + Instructions + + +
+ {% endfor %} + + + {% endfor %} + + + + + {% endblock admin_content %} {% block scripts %} From cc3a21cac9937a86d0f8ddbf683ccf70ba09b119 Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Mon, 13 Jan 2020 18:13:51 +0600 Subject: [PATCH 12/18] CVE count grouped by severity. --- backend/device_registry/models.py | 18 ++++++++++++- .../templates/device_info_security.html | 18 +++++-------- backend/device_registry/tests/test_all.py | 26 ++++++++++++++++++- 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py index f38973c76..ea70e8048 100644 --- a/backend/device_registry/models.py +++ b/backend/device_registry/models.py @@ -7,7 +7,7 @@ from django.conf import settings from django.db import models, transaction -from django.db.models import Q, Avg +from django.db.models import Q, Avg, Count from django.utils import timezone from django.contrib.postgres.fields import ArrayField, JSONField from django.core.exceptions import ObjectDoesNotExist @@ -473,6 +473,22 @@ def vulnerable_packages(self): self.os_release.get('codename') in DEBIAN_SUITES + UBUNTU_SUITES: return self.deb_packages.filter(vulnerabilities__isnull=False).distinct().order_by('name') + @property + def cve_count(self): + if not(self.deb_packages_hash and self.deb_packages.exists() and self.os_release and \ + self.os_release.get('codename') in DEBIAN_SUITES + UBUNTU_SUITES): + return + pks = Vulnerability.objects.filter(urgency__gte=Vulnerability.Urgency.LOW, + debpackage__device__owner=self.owner).distinct().values('pk') + urgencies = Vulnerability.objects.filter(pk__in=pks).values('urgency')\ + .annotate(urg_count=Count('urgency')) + severities = { + Vulnerability.Urgency.HIGH: 'high', + Vulnerability.Urgency.MEDIUM: 'med', + Vulnerability.Urgency.LOW: 'low' + } + return {severities[u['urgency']]: u['urg_count'] for u in urgencies} + def generate_recommended_actions(self, classes=None): """ Generate RAs for this device and store them as RecommendedAction objects in database. diff --git a/backend/device_registry/templates/device_info_security.html b/backend/device_registry/templates/device_info_security.html index 530fae51a..f5ad2c782 100644 --- a/backend/device_registry/templates/device_info_security.html +++ b/backend/device_registry/templates/device_info_security.html @@ -23,22 +23,16 @@

Security

Vulnerable Packages - {% with object.vulnerable_packages as vulnerable_packages %} - {% if vulnerable_packages is None %} + {% with object.cve_count as cve_count %} + {% if cve_count is None %} N/A - {% elif vulnerable_packages.exists %} -
    - {% for package in vulnerable_packages %} -
  • {{ package.name }} / {{ package.version }} - ({% for v in package.vulnerabilities.all %}{{ v.name }}{% if not forloop.last %}, {% endif %}{% endfor %}) -
    • - {% endfor %} -
{% else %} - {% include "badge.html" with icon="check" color="success" %} + High: {{ cve_count.high|default:"0" }}
+ Medium: {{ cve_count.med|default:"0" }}
+ Low: {{ cve_count.low|default:"0" }}
{% endif %} {% endwith %} + Detailed View {% with object.heartbleed_vulnerable as heartbleed_vulnerable %} diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index e5a58b73b..58f992799 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -1423,7 +1423,9 @@ def setUp(self): self.device0 = Device.objects.create( device_id='device0.d.wott-dev.local', - owner=self.user + owner=self.user, + deb_packages_hash='abcd', + os_release={'codename': 'stretch'} ) self.device1 = Device.objects.create( device_id='device1.d.wott-dev.local', @@ -1533,3 +1535,25 @@ def test_filter_device(self): CVEView.AffectedPackage('one_first', [self.device0]) ]) ]) + + def test_cve_count(self): + vulns = [ + Vulnerability(os_release_codename='stretch', name='CVE-2018-1', package='one_source', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.HIGH, fix_available=True), + Vulnerability(os_release_codename='buster', name='CVE-2018-2', package='one_source', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.MEDIUM, fix_available=True, + pub_date=self.today), + Vulnerability(os_release_codename='buster', name='CVE-2018-3', package='one_source', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.MEDIUM, fix_available=True, + pub_date=self.today), + Vulnerability(os_release_codename='stretch', name='CVE-2018-4', package='one_source', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=False), + Vulnerability(os_release_codename='stretch', name='CVE-2018-5', package='one_source', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=False), + Vulnerability(os_release_codename='stretch', name='CVE-2018-6', package='one_source', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=False) + ] + Vulnerability.objects.bulk_create(vulns) + self.packages[0].vulnerabilities.set(vulns) + self.packages[1].vulnerabilities.set(vulns) + self.assertDictEqual(self.device0.cve_count, {'high': 1, 'med': 2, 'low': 3}) From 6dda2b9a6008163f8ae97e81ca0fbfefa913dd42 Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Mon, 13 Jan 2020 21:38:43 +0600 Subject: [PATCH 13/18] Cont CVEs in simplier and more reliable way. --- backend/device_registry/models.py | 16 ++++++++-------- .../templates/device_info_security.html | 6 +++--- backend/device_registry/tests/test_all.py | 18 ++++++++++-------- 3 files changed, 21 insertions(+), 19 deletions(-) diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py index ea70e8048..581f89802 100644 --- a/backend/device_registry/models.py +++ b/backend/device_registry/models.py @@ -7,7 +7,7 @@ from django.conf import settings from django.db import models, transaction -from django.db.models import Q, Avg, Count +from django.db.models import Q from django.utils import timezone from django.contrib.postgres.fields import ArrayField, JSONField from django.core.exceptions import ObjectDoesNotExist @@ -475,19 +475,19 @@ def vulnerable_packages(self): @property def cve_count(self): - if not(self.deb_packages_hash and self.deb_packages.exists() and self.os_release and \ - self.os_release.get('codename') in DEBIAN_SUITES + UBUNTU_SUITES): + if not(self.deb_packages_hash and self.deb_packages.exists() and self.os_release + and self.os_release.get('codename') in DEBIAN_SUITES + UBUNTU_SUITES): return - pks = Vulnerability.objects.filter(urgency__gte=Vulnerability.Urgency.LOW, - debpackage__device__owner=self.owner).distinct().values('pk') - urgencies = Vulnerability.objects.filter(pk__in=pks).values('urgency')\ - .annotate(urg_count=Count('urgency')) + + vuln_qs = Vulnerability.objects.filter(urgency__gte=Vulnerability.Urgency.LOW, debpackage__device=self, + fix_available=True) severities = { Vulnerability.Urgency.HIGH: 'high', Vulnerability.Urgency.MEDIUM: 'med', Vulnerability.Urgency.LOW: 'low' } - return {severities[u['urgency']]: u['urg_count'] for u in urgencies} + return {severities[urgency]: vuln_qs.filter(urgency=urgency).values('name').distinct().count() + for urgency in severities} def generate_recommended_actions(self, classes=None): """ diff --git a/backend/device_registry/templates/device_info_security.html b/backend/device_registry/templates/device_info_security.html index f5ad2c782..91134f07f 100644 --- a/backend/device_registry/templates/device_info_security.html +++ b/backend/device_registry/templates/device_info_security.html @@ -27,9 +27,9 @@

Security

{% if cve_count is None %} N/A {% else %} - High: {{ cve_count.high|default:"0" }}
- Medium: {{ cve_count.med|default:"0" }}
- Low: {{ cve_count.low|default:"0" }}
+ High: {{ cve_count.high }}
+ Medium: {{ cve_count.med }}
+ Low: {{ cve_count.low }}
{% endif %} {% endwith %} Detailed View diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index 58f992799..b65396c61 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -1538,19 +1538,21 @@ def test_filter_device(self): def test_cve_count(self): vulns = [ - Vulnerability(os_release_codename='stretch', name='CVE-2018-1', package='one_source', is_binary=False, + Vulnerability(os_release_codename='stretch', name='CVE-2018-1', package='', is_binary=False, other_versions=[], urgency=Vulnerability.Urgency.HIGH, fix_available=True), - Vulnerability(os_release_codename='buster', name='CVE-2018-2', package='one_source', is_binary=False, + Vulnerability(os_release_codename='buster', name='CVE-2018-2', package='', is_binary=False, other_versions=[], urgency=Vulnerability.Urgency.MEDIUM, fix_available=True, pub_date=self.today), - Vulnerability(os_release_codename='buster', name='CVE-2018-3', package='one_source', is_binary=False, + Vulnerability(os_release_codename='buster', name='CVE-2018-3', package='', is_binary=False, other_versions=[], urgency=Vulnerability.Urgency.MEDIUM, fix_available=True, pub_date=self.today), - Vulnerability(os_release_codename='stretch', name='CVE-2018-4', package='one_source', is_binary=False, - other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=False), - Vulnerability(os_release_codename='stretch', name='CVE-2018-5', package='one_source', is_binary=False, - other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=False), - Vulnerability(os_release_codename='stretch', name='CVE-2018-6', package='one_source', is_binary=False, + Vulnerability(os_release_codename='stretch', name='CVE-2018-4', package='', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=True), + Vulnerability(os_release_codename='stretch', name='CVE-2018-5', package='', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=True), + Vulnerability(os_release_codename='stretch', name='CVE-2018-6', package='', is_binary=False, + other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=True), + Vulnerability(os_release_codename='stretch', name='CVE-2018-6', package='', is_binary=False, other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=False) ] Vulnerability.objects.bulk_create(vulns) From a4a6f8dd90d48629af7c3dc1de98e36a66c9d526 Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Mon, 13 Jan 2020 21:48:07 +0600 Subject: [PATCH 14/18] Fix tests. --- backend/device_registry/tests/test_all.py | 49 ------------------- .../tests/test_recommended_actions.py | 3 +- backend/device_registry/views.py | 2 + 3 files changed, 4 insertions(+), 50 deletions(-) diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index b65396c61..5e961060d 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -685,55 +685,6 @@ def test_insecure_services(self): {'name': 'fingerd', 'version': 'VERSION', 'arch': 'i386', 'os_release_codename': 'jessie'}]) - def test_vulnerable_packages_render(self): - self.client.login(username='test', password='123') - url = reverse('device-detail-security', kwargs={'pk': self.device.pk}) - # No packages - should render N/A - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertInHTML(''' - - Vulnerable Packages - - - N/A - ''', response.rendered_content) - - self.device.deb_packages_hash = 'aabbccdd' - self.device.save() - self.device.set_deb_packages([ - {'name': 'python2', 'version': 'VERSION', 'source_name': 'python2', 'source_version': 'abcd', - 'arch': 'i386'}, - {'name': 'python3', 'version': 'VERSION', 'source_name': 'python3', 'source_version': 'abcd', - 'arch': 'i386'} - ], os_info={'codename': 'stretch'}) - # No vulnerable packages - green check mark - self.device.os_release = {'distro': 'debian', 'version': '9', 'codename': 'stretch', 'distro_root': 'debian', - 'full_version': '9 (stretch)'} - self.device.save(update_fields=['os_release']) - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertInHTML(''' - - Vulnerable Packages - - - - - - - ''', response.rendered_content) - - v = Vulnerability.objects.create(name='CVE-123', package='python2', is_binary=False, other_versions=[], - urgency=Vulnerability.Urgency.NONE, fix_available=True) - self.device.deb_packages.get(name='python2').vulnerabilities.add(v) - - # Has vulnerable packages - should render them - response = self.client.get(url) - self.assertEqual(response.status_code, 200) - self.assertContains(response, 'python2') - self.assertContains(response, 'CVE-123') - def test_heartbleed_render(self): self.device.deb_packages_hash = 'aabbccdd' self.client.login(username='test', password='123') diff --git a/backend/device_registry/tests/test_recommended_actions.py b/backend/device_registry/tests/test_recommended_actions.py index b6fe3516d..2e5a0307e 100644 --- a/backend/device_registry/tests/test_recommended_actions.py +++ b/backend/device_registry/tests/test_recommended_actions.py @@ -457,7 +457,8 @@ def enable_action(self): deb_package = DebPackage.objects.create(name='package', version='version1', source_name='package', source_version='sversion1', arch='amd64', os_release_codename='jessie') vulnerability = Vulnerability.objects.create(name='name', package='package', is_binary=True, other_versions=[], - urgency='L', fix_available=True, os_release_codename='jessie') + urgency=Vulnerability.Urgency.LOW, fix_available=True, + os_release_codename='jessie') deb_package.vulnerabilities.add(vulnerability) self.device.deb_packages.add(deb_package) diff --git a/backend/device_registry/views.py b/backend/device_registry/views.py index af65bceb1..94a71bd9a 100644 --- a/backend/device_registry/views.py +++ b/backend/device_registry/views.py @@ -666,6 +666,8 @@ def get_context_data(self, **kwargs): table_rows = [] for cve_name, cve_packages in packages_by_cve.items(): plist = sorted([self.AffectedPackage(p.name, + # FIXME: this line may need additional optimisation to avoid calling + # device_set.filter() every time. [device] if device else list(p.device_set.filter(owner=user))) for p in cve_packages], key=lambda p: len(p.devices), reverse=True) From c4fe7b38364e5a4f2ca87abddc6dc97da581327f Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Tue, 14 Jan 2020 12:26:53 +0600 Subject: [PATCH 15/18] Implement popovers. --- backend/device_registry/templates/cve.html | 23 ++++++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/backend/device_registry/templates/cve.html b/backend/device_registry/templates/cve.html index 7b8c6b0f5..9edda4ebb 100644 --- a/backend/device_registry/templates/cve.html +++ b/backend/device_registry/templates/cve.html @@ -39,11 +39,11 @@

CVE list{% if device_name %} for {{ device_name }}{ {% if not device_name %} {% for p in row.packages %} - + {{ p.device_urls|length }} @@ -53,10 +53,11 @@

CVE list{% if device_name %} for {{ device_name }}{ {% endif %} {% for p in row.packages %} - + Instructions
@@ -74,4 +75,18 @@

CVE list{% if device_name %} for {{ device_name }}{ {% block scripts %} {{ block.super }} + {% endblock scripts %} \ No newline at end of file From 9f15fc07f69c7cd68ea3b6fb3d29375f1edfad0c Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Thu, 16 Jan 2020 12:27:41 +0600 Subject: [PATCH 16/18] Merge migrations into one. Add comments to Device.cve_count. Add unique constraint to Vulnerability. Add default case for migration convert_urgencies(). PEP8 fixes. --- ..._20200109_1137.py => 0079_vulnerability.py} | 12 +++++++++++- .../migrations/0080_vulnerability_pub_date.py | 18 ------------------ backend/device_registry/models.py | 11 +++++++++-- backend/device_registry/views.py | 10 +++++----- 4 files changed, 25 insertions(+), 26 deletions(-) rename backend/device_registry/migrations/{0079_auto_20200109_1137.py => 0079_vulnerability.py} (76%) delete mode 100644 backend/device_registry/migrations/0080_vulnerability_pub_date.py diff --git a/backend/device_registry/migrations/0079_auto_20200109_1137.py b/backend/device_registry/migrations/0079_vulnerability.py similarity index 76% rename from backend/device_registry/migrations/0079_auto_20200109_1137.py rename to backend/device_registry/migrations/0079_vulnerability.py index a3e1c7942..7aa85bac9 100644 --- a/backend/device_registry/migrations/0079_auto_20200109_1137.py +++ b/backend/device_registry/migrations/0079_vulnerability.py @@ -14,7 +14,8 @@ def convert_urgencies(apps, schema_editor): } Vulnerability = apps.get_model('device_registry', 'Vulnerability') Vulnerability.objects.update(urgency=Case( - *[When(urgency=k, then=Value(v)) for k, v in urgencies.items()] + *[When(urgency=k, then=Value(v)) for k, v in urgencies.items()], + default=Value(0) )) @@ -31,4 +32,13 @@ class Migration(migrations.Migration): name='urgency', field=models.PositiveSmallIntegerField(choices=[(device_registry.models.Vulnerability.Urgency(0), 0), (device_registry.models.Vulnerability.Urgency(1), 1), (device_registry.models.Vulnerability.Urgency(2), 2), (device_registry.models.Vulnerability.Urgency(3), 3)]), ), + migrations.AddField( + model_name='vulnerability', + name='pub_date', + field=models.DateField(null=True), + ), + migrations.AlterUniqueTogether( + name='vulnerability', + unique_together={('os_release_codename', 'name', 'package')}, + ) ] diff --git a/backend/device_registry/migrations/0080_vulnerability_pub_date.py b/backend/device_registry/migrations/0080_vulnerability_pub_date.py deleted file mode 100644 index 60d33920b..000000000 --- a/backend/device_registry/migrations/0080_vulnerability_pub_date.py +++ /dev/null @@ -1,18 +0,0 @@ -# Generated by Django 2.2.6 on 2020-01-09 12:25 - -from django.db import migrations, models - - -class Migration(migrations.Migration): - - dependencies = [ - ('device_registry', '0079_auto_20200109_1137'), - ] - - operations = [ - migrations.AddField( - model_name='vulnerability', - name='pub_date', - field=models.DateField(null=True), - ), - ] diff --git a/backend/device_registry/models.py b/backend/device_registry/models.py index 581f89802..f4940863f 100644 --- a/backend/device_registry/models.py +++ b/backend/device_registry/models.py @@ -1,6 +1,5 @@ from enum import Enum, IntEnum import datetime -from statistics import mean import json import uuid from typing import NamedTuple @@ -475,10 +474,15 @@ def vulnerable_packages(self): @property def cve_count(self): + """ + Count the number of high, medium and low severity CVEs for the device. + :return: A dict of {'high': N1, 'med': N2, 'low': N3} or None if no deb packages or unsupported OS. + """ + + # We have no vulnerability data for OS other than Debian and Ubuntu flavors. if not(self.deb_packages_hash and self.deb_packages.exists() and self.os_release and self.os_release.get('codename') in DEBIAN_SUITES + UBUNTU_SUITES): return - vuln_qs = Vulnerability.objects.filter(urgency__gte=Vulnerability.Urgency.LOW, debpackage__device=self, fix_available=True) severities = { @@ -822,6 +826,9 @@ class Meta: class Vulnerability(models.Model): + class Meta: + unique_together = ['os_release_codename', 'name', 'package'] + class Version: """Version class which uses the original APT comparison algorithm.""" diff --git a/backend/device_registry/views.py b/backend/device_registry/views.py index 94a71bd9a..ee09998e0 100644 --- a/backend/device_registry/views.py +++ b/backend/device_registry/views.py @@ -634,7 +634,7 @@ def severity(self): @property def cve_link(self): - return CVEView.Hyperlink(self.cve_name, 'http://cve.mitre.org/cgi-bin/cvename.cgi?name='+self.cve_name) + return CVEView.Hyperlink(self.cve_name, 'http://cve.mitre.org/cgi-bin/cvename.cgi?name=' + self.cve_name) def get_context_data(self, **kwargs): context = super().get_context_data(**kwargs) @@ -665,12 +665,12 @@ def get_context_data(self, **kwargs): table_rows = [] for cve_name, cve_packages in packages_by_cve.items(): - plist = sorted([self.AffectedPackage(p.name, + plist = sorted([self.AffectedPackage(package.name, # FIXME: this line may need additional optimisation to avoid calling # device_set.filter() every time. - [device] if device else list(p.device_set.filter(owner=user))) - for p in cve_packages], - key=lambda p: len(p.devices), reverse=True) + [device] if device else list(package.device_set.filter(owner=user))) + for package in cve_packages], + key=lambda package: len(package.devices), reverse=True) urgency, cve_date = vuln_info[cve_name] table_rows.append(self.TableRow(cve_name=cve_name, cve_url='', urgency=urgency, cve_date=cve_date, packages=plist)) From c543d7b5468e062c12c1d4c8170ba63cad2bdd8d Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Thu, 16 Jan 2020 12:43:15 +0600 Subject: [PATCH 17/18] ubuntu_cve: parse pub_date. --- backend/device_registry/celery_tasks/ubuntu_cve.py | 3 ++- backend/device_registry/tests/test_all.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/device_registry/celery_tasks/ubuntu_cve.py b/backend/device_registry/celery_tasks/ubuntu_cve.py index 0facab0e9..c3ccb4ed5 100644 --- a/backend/device_registry/celery_tasks/ubuntu_cve.py +++ b/backend/device_registry/celery_tasks/ubuntu_cve.py @@ -5,6 +5,7 @@ from itertools import chain import time +import dateutil.parser from django.conf import settings import redis @@ -291,7 +292,7 @@ def fetch_vulnerabilities(): 'medium': Vulnerability.Urgency.MEDIUM, 'high': Vulnerability.Urgency.HIGH, }.get(header['Priority'], Vulnerability.Urgency.NONE), - pub_date=header['PublicDate'], + pub_date=dateutil.parser.parse(header['PublicDate']), remote=None, fix_available=(status == 'fixed'), os_release_codename=codename diff --git a/backend/device_registry/tests/test_all.py b/backend/device_registry/tests/test_all.py index 5e961060d..c708d3bfd 100644 --- a/backend/device_registry/tests/test_all.py +++ b/backend/device_registry/tests/test_all.py @@ -1503,7 +1503,7 @@ def test_cve_count(self): other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=True), Vulnerability(os_release_codename='stretch', name='CVE-2018-6', package='', is_binary=False, other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=True), - Vulnerability(os_release_codename='stretch', name='CVE-2018-6', package='', is_binary=False, + Vulnerability(os_release_codename='stretch', name='CVE-2018-7', package='', is_binary=False, other_versions=[], urgency=Vulnerability.Urgency.LOW, fix_available=False) ] Vulnerability.objects.bulk_create(vulns) From b66031bea6b6ba76bb076a6f3c2e18796a27f3ad Mon Sep 17 00:00:00 2001 From: Artem Martynovich Date: Thu, 16 Jan 2020 22:40:34 +0600 Subject: [PATCH 18/18] Move import in celery_tasks.ubuntu_cve --- backend/device_registry/celery_tasks/ubuntu_cve.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/backend/device_registry/celery_tasks/ubuntu_cve.py b/backend/device_registry/celery_tasks/ubuntu_cve.py index c3ccb4ed5..05862e774 100644 --- a/backend/device_registry/celery_tasks/ubuntu_cve.py +++ b/backend/device_registry/celery_tasks/ubuntu_cve.py @@ -5,9 +5,9 @@ from itertools import chain import time -import dateutil.parser from django.conf import settings +import dateutil.parser import redis from git import Repo