addition of sla expiration date field without signals

dojo/api_v2/serializers.py

@@ -2004,8 +2004,20 @@ class Meta:
         exclude = (
             "tid",
             "updated",
+            "async_updating"
         )
 
+    def validate(self, data):
+        async_updating = getattr(self.instance, 'async_updating', None)
+        if async_updating:
+            new_sla_config = data.get('sla_configuration', None)
+            old_sla_config = getattr(self.instance, 'sla_configuration', None)
+            if new_sla_config and old_sla_config and new_sla_config != old_sla_config:
+                raise serializers.ValidationError(
+                    'Finding SLA expiration dates are currently being recalculated. The SLA configuration for this product cannot be changed until the calculation is complete.'
+                )
+        return data
+
     def get_findings_count(self, obj) -> int:
         return obj.findings_count
 
@@ -3031,7 +3043,21 @@ class Meta:
 class SLAConfigurationSerializer(serializers.ModelSerializer):
     class Meta:
         model = SLA_Configuration
-        fields = "__all__"
+        exclude = (
+            "async_updating",
+        )
+
+    def validate(self, data):
+        async_updating = getattr(self.instance, 'async_updating', None)
+        if async_updating:
+            for field in ['critical', 'high', 'medium', 'low']:
+                old_days = getattr(self.instance, field, None)
+                new_days = data.get(field, None)
+                if old_days and new_days and (old_days != new_days):
+                    raise serializers.ValidationError(
+                        'Finding SLA expiration dates are currently being calculated. The SLA days for this SLA configuration cannot be changed until the calculation is complete.'
+                    )
+        return data
 
 
 class UserProfileSerializer(serializers.Serializer):

dojo/apps.py

@@ -74,6 +74,7 @@ def ready(self):
         import dojo.announcement.signals  # noqa
         import dojo.product.signals  # noqa
         import dojo.test.signals  # noqa
+        import dojo.sla_config.helpers  # noqa
 
 
 def get_model_fields_with_extra(model, extra_fields=()):

dojo/db_migrations/0200_finding_sla_expiration_date_product_async_updating_and_more.py

@@ -0,0 +1,131 @@
+# Generated by Django 4.1.13 on 2024-01-17 03:07
+
+from django.db import migrations, models
+from django.utils import timezone
+from datetime import datetime, timedelta
+from django.conf import settings
+from dateutil.relativedelta import relativedelta
+import logging
+
+logger = logging.getLogger(__name__)
+
+
+def get_work_days(start, end):
+    """
+    Duplicate of utility function 'get_work_days' at the time of migration creation.
+    """
+    if start.weekday() > 4:
+        start = start + timedelta(days=7 - start.weekday())
+
+    if end.weekday() > 4:
+        end = end - timedelta(days=end.weekday() - 4)
+
+    if start > end:
+        return 0
+
+    diff_days = (end - start).days + 1
+    weeks = int(diff_days / 7)
+
+    remainder = end.weekday() - start.weekday() + 1
+
+    if remainder != 0 and end.weekday() < start.weekday():
+        remainder = 5 + remainder
+
+    return weeks * 5 + remainder
+
+
+def calculate_sla_expiration_dates(apps, schema_editor):
+    System_Settings = apps.get_model('dojo', 'System_Settings')
+
+    ss, _ = System_Settings.objects.get_or_create()
+    if ss.enable_finding_sla:
+        logger.info('Calculating SLA expiration dates for all findings')
+
+        SLA_Configuration = apps.get_model('dojo', 'SLA_Configuration')
+        Product = apps.get_model('dojo', 'Product')
+        Finding = apps.get_model('dojo', 'Finding')
+
+        findings = Finding.objects.order_by('id').only('id', 'sla_start_date', 'date', 'severity', 'test', 'mitigated')
+
+        page_size = 1000
+        total_count = Finding.objects.filter(id__gt=0).count()
+        logger.debug('Found %d findings to be updated', total_count)
+
+        i = 0
+        batch = []
+        last_id = 0
+        total_pages = (total_count // page_size) + 2
+        for p in range(1, total_pages):
+            page = findings.filter(id__gt=last_id)[:page_size]
+            for find in page:
+                i += 1
+                last_id = find.id
+
+                start_date = find.sla_start_date if find.sla_start_date else find.date
+
+                sla_config = SLA_Configuration.objects.filter(id=find.test.engagement.product.sla_configuration_id).first()
+                sla_period = getattr(sla_config, find.severity.lower(), None)
+
+                days = None
+                if settings.SLA_BUSINESS_DAYS:
+                    if find.mitigated:
+                        days = get_work_days(find.date, find.mitigated.date())
+                    else:
+                        days = get_work_days(find.date, timezone.now().date())
+                else:
+                    if isinstance(start_date, datetime):
+                        start_date = start_date.date()
+
+                    if find.mitigated:
+                        days = (find.mitigated.date() - start_date).days
+                    else:
+                        days = (timezone.now().date() - start_date).days
+
+                days = days if days > 0 else 0
+
+                days_remaining = None
+                if sla_period:
+                    days_remaining = sla_period - days
+
+                if days_remaining:
+                    if find.mitigated:
+                        find.sla_expiration_date = find.mitigated.date() + relativedelta(days=days_remaining)
+                    else:
+                        find.sla_expiration_date = timezone.now().date() + relativedelta(days=days_remaining)
+
+                batch.append(find)
+
+                if (i > 0 and i % page_size == 0):
+                    Finding.objects.bulk_update(batch, ['sla_expiration_date'])
+                    batch = []
+                    logger.info('%s out of %s findings processed...', i, total_count)
+
+        Finding.objects.bulk_update(batch, ['sla_expiration_date'])
+        batch = []
+        logger.info('%s out of %s findings processed...', i, total_count)
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('dojo', '0199_whitesource_to_mend'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='finding',
+            name='sla_expiration_date',
+            field=models.DateField(blank=True, help_text="(readonly)The date SLA expires for this finding. Empty by default, causing a fallback to 'date'.", null=True, verbose_name='SLA Expiration Date'),
+        ),
+        migrations.RunPython(calculate_sla_expiration_dates, migrations.RunPython.noop),
+        migrations.AddField(
+            model_name='product',
+            name='async_updating',
+            field=models.BooleanField(default=False, help_text='Findings under this SLA configuration are asynchronously being updated'),
+        ),
+        migrations.AddField(
+            model_name='sla_configuration',
+            name='async_updating',
+            field=models.BooleanField(default=False, help_text='Findings under this SLA configuration are asynchronously being updated'),
+        ),
+    ]

dojo/filters.py

@@ -11,6 +11,7 @@
 from django.conf import settings
 import six
 from django.utils.translation import gettext_lazy as _
+from django.utils import timezone
 from django_filters import FilterSet, CharFilter, OrderingFilter, \
     ModelMultipleChoiceFilter, ModelChoiceFilter, MultipleChoiceFilter, \
     BooleanFilter, NumberFilter, DateFilter
@@ -147,22 +148,18 @@ class FindingSLAFilter(ChoiceFilter):
     def any(self, qs, name):
         return qs
 
-    def satisfies_sla(self, qs, name):
-        for finding in qs:
-            if finding.violates_sla:
-                qs = qs.exclude(id=finding.id)
-        return qs
+    def sla_satisfied(self, qs, name):
+        # return findings that have an sla expiration date after today or no sla expiration date
+        return qs.filter(Q(sla_expiration_date__isnull=True) | Q(sla_expiration_date__gt=timezone.now().date()))
 
-    def violates_sla(self, qs, name):
-        for finding in qs:
-            if not finding.violates_sla:
-                qs = qs.exclude(id=finding.id)
-        return qs
+    def sla_violated(self, qs, name):
+        # return active findings that have an sla expiration date before today
+        return qs.filter(Q(active=True) & Q(sla_expiration_date__lt=timezone.now().date()))
 
     options = {
         None: (_('Any'), any),
-        0: (_('False'), satisfies_sla),
-        1: (_('True'), violates_sla),
+        0: (_('False'), sla_satisfied),
+        1: (_('True'), sla_violated),
     }
 
     def __init__(self, *args, **kwargs):
@@ -182,22 +179,22 @@ class ProductSLAFilter(ChoiceFilter):
     def any(self, qs, name):
         return qs
 
-    def satisfies_sla(self, qs, name):
+    def sla_satisifed(self, qs, name):
         for product in qs:
-            if product.violates_sla:
+            if product.violates_sla():
                 qs = qs.exclude(id=product.id)
         return qs
 
-    def violates_sla(self, qs, name):
+    def sla_violated(self, qs, name):
         for product in qs:
-            if not product.violates_sla:
+            if not product.violates_sla():
                 qs = qs.exclude(id=product.id)
         return qs
 
     options = {
         None: (_('Any'), any),
-        0: (_('False'), satisfies_sla),
-        1: (_('True'), violates_sla),
+        0: (_('False'), sla_satisifed),
+        1: (_('True'), sla_violated),
     }
 
     def __init__(self, *args, **kwargs):
@@ -1465,9 +1462,8 @@ class Meta:
                    'endpoints', 'references',
                    'thread_id', 'notes', 'scanner_confidence',
                    'numerical_severity', 'line', 'duplicate_finding',
-                   'hash_code',
-                   'reviewers',
-                   'created', 'files', 'sla_start_date', 'cvssv3',
+                   'hash_code', 'reviewers', 'created', 'files',
+                   'sla_start_date', 'sla_expiration_date', 'cvssv3',
                    'severity_justification', 'steps_to_reproduce']
 
     def __init__(self, *args, **kwargs):

dojo/forms.py

@@ -263,6 +263,12 @@ def __init__(self, *args, **kwargs):
         super(ProductForm, self).__init__(*args, **kwargs)
         self.fields['prod_type'].queryset = get_authorized_product_types(Permissions.Product_Type_Add_Product)
 
+        # if this product has findings being asynchronously updated, disable the sla config field
+        if self.instance.async_updating:
+            self.fields['sla_configuration'].disabled = True
+            self.fields['sla_configuration'].widget.attrs['message'] = 'Finding SLA expiration dates are currently being recalculated. ' + \
+                                                                       'This field cannot be changed until the calculation is complete.'
+
     class Meta:
         model = Product
         fields = ['name', 'description', 'tags', 'product_manager', 'technical_contact', 'team_manager', 'prod_type', 'sla_configuration', 'regulations',
@@ -1073,7 +1079,7 @@ class AdHocFindingForm(forms.ModelForm):
     # the only reliable way without hacking internal fields to get predicatble ordering is to make it explicit
     field_order = ('title', 'date', 'cwe', 'vulnerability_ids', 'severity', 'cvssv3', 'description', 'mitigation', 'impact', 'request', 'response', 'steps_to_reproduce',
                    'severity_justification', 'endpoints', 'endpoints_to_add', 'references', 'active', 'verified', 'false_p', 'duplicate', 'out_of_scope',
-                   'risk_accepted', 'under_defect_review', 'sla_start_date')
+                   'risk_accepted', 'under_defect_review', 'sla_start_date', 'sla_expiration_date')
 
     def __init__(self, *args, **kwargs):
         req_resp = kwargs.pop('req_resp')
@@ -1113,7 +1119,8 @@ def clean(self):
     class Meta:
         model = Finding
         exclude = ('reporter', 'url', 'numerical_severity', 'under_review', 'reviewers', 'cve', 'inherited_tags',
-                   'review_requested_by', 'is_mitigated', 'jira_creation', 'jira_change', 'endpoint_status', 'sla_start_date')
+                   'review_requested_by', 'is_mitigated', 'jira_creation', 'jira_change', 'endpoints', 'sla_start_date',
+                   'sla_expiration_date')
 
 
 class PromoteFindingForm(forms.ModelForm):
@@ -1139,9 +1146,9 @@ class PromoteFindingForm(forms.ModelForm):
     references = forms.CharField(widget=forms.Textarea, required=False)
 
     # the onyl reliable way without hacking internal fields to get predicatble ordering is to make it explicit
-    field_order = ('title', 'group', 'date', 'sla_start_date', 'cwe', 'vulnerability_ids', 'severity', 'cvssv3', 'cvssv3_score', 'description', 'mitigation', 'impact',
-                   'request', 'response', 'steps_to_reproduce', 'severity_justification', 'endpoints', 'endpoints_to_add', 'references',
-                   'active', 'mitigated', 'mitigated_by', 'verified', 'false_p', 'duplicate',
+    field_order = ('title', 'group', 'date', 'sla_start_date', 'sla_expiration_date', 'cwe', 'vulnerability_ids', 'severity', 'cvssv3',
+                   'cvssv3_score', 'description', 'mitigation', 'impact', 'request', 'response', 'steps_to_reproduce', 'severity_justification',
+                   'endpoints', 'endpoints_to_add', 'references', 'active', 'mitigated', 'mitigated_by', 'verified', 'false_p', 'duplicate',
                    'out_of_scope', 'risk_accept', 'under_defect_review')
 
     def __init__(self, *args, **kwargs):
@@ -1211,9 +1218,9 @@ class FindingForm(forms.ModelForm):
             'invalid_choice': EFFORT_FOR_FIXING_INVALID_CHOICE})
 
     # the only reliable way without hacking internal fields to get predicatble ordering is to make it explicit
-    field_order = ('title', 'group', 'date', 'sla_start_date', 'cwe', 'vulnerability_ids', 'severity', 'cvssv3', 'cvssv3_score', 'description', 'mitigation', 'impact',
-                   'request', 'response', 'steps_to_reproduce', 'severity_justification', 'endpoints', 'endpoints_to_add', 'references',
-                   'active', 'mitigated', 'mitigated_by', 'verified', 'false_p', 'duplicate',
+    field_order = ('title', 'group', 'date', 'sla_start_date', 'sla_expiration_date', 'cwe', 'vulnerability_ids', 'severity', 'cvssv3',
+                   'cvssv3_score', 'description', 'mitigation', 'impact', 'request', 'response', 'steps_to_reproduce', 'severity_justification',
+                   'endpoints', 'endpoints_to_add', 'references', 'active', 'mitigated', 'mitigated_by', 'verified', 'false_p', 'duplicate',
                    'out_of_scope', 'risk_accept', 'under_defect_review')
 
     def __init__(self, *args, **kwargs):
@@ -1251,6 +1258,7 @@ def __init__(self, *args, **kwargs):
             self.fields['duplicate'].help_text = "You can mark findings as duplicate only from the view finding page."
 
         self.fields['sla_start_date'].disabled = True
+        self.fields['sla_expiration_date'].disabled = True
 
         if self.can_edit_mitigated_data:
             if hasattr(self, 'instance'):
@@ -2436,6 +2444,22 @@ def clean(self):
 
 
 class SLAConfigForm(forms.ModelForm):
+    def __init__(self, *args, **kwargs):
+        super(SLAConfigForm, self).__init__(*args, **kwargs)
+
+        # if this sla config has findings being asynchronously updated, disable the days by severity fields
+        if self.instance.async_updating:
+            msg = 'Finding SLA expiration dates are currently being recalculated. ' + \
+                  'This field cannot be changed until the calculation is complete.'
+            self.fields['critical'].disabled = True
+            self.fields['critical'].widget.attrs['message'] = msg
+            self.fields['high'].disabled = True
+            self.fields['high'].widget.attrs['message'] = msg
+            self.fields['medium'].disabled = True
+            self.fields['medium'].widget.attrs['message'] = msg
+            self.fields['low'].disabled = True
+            self.fields['low'].widget.attrs['message'] = msg
+
     class Meta:
         model = SLA_Configuration
         fields = ['name', 'description', 'critical', 'high', 'medium', 'low']