Merge branch 'release-1.5.2'

.circleci/config.yml

@@ -1,7 +1,7 @@
 #
 # Copyright (C) 2019-2020 Woods Hole Oceanographic Institution
 #
-# This file is part of the Roundabout Database project ("RDB" or 
+# This file is part of the Roundabout Database project ("RDB" or
 # "ooicgsn-roundabout").
 #
 # ooicgsn-roundabout is free software: you can redistribute it and/or modify

.envs.example/.local/.django

@@ -19,6 +19,9 @@ DJANGO_SU_NAME=admin
 [email protected]
 DJANGO_SU_PASSWORD=admin
 
+# Site Domain
+RDB_SITE_URL=https://yourdomain.com
+
 # Settings for RDB serial number generation functions
 # ------------------------------------------------------------------------------
 # Default pattern - "1, 2, 3, ... etc."

.envs.example/.production/.django

@@ -44,6 +44,9 @@ DJANGO_SU_NAME=admin
 [email protected]
 DJANGO_SU_PASSWORD=admin
 
+# Site Domain
+RDB_SITE_URL=https://yourdomain.com
+
 # Settings for RDB serial number generation functions
 # ------------------------------------------------------------------------------
 # Default pattern - "1, 2, 3, ... etc."

compose/local/tests/Dockerfile

@@ -1,5 +1,6 @@
 FROM ubuntu:18.04
 RUN apt-get update && apt-get install -y --no-install-recommends apt-utils
+RUN apt-get update
 RUN apt-get install curl -y && apt-get install npm -y
 # Tools for Circleci debugging
 RUN apt-get install iputils-ping -y && apt-get install net-tools -y

config/settings/base.py

@@ -1,7 +1,7 @@
 """
 # Copyright (C) 2019-2020 Woods Hole Oceanographic Institution
 #
-# This file is part of the Roundabout Database project ("RDB" or 
+# This file is part of the Roundabout Database project ("RDB" or
 # "ooicgsn-roundabout").
 #
 # ooicgsn-roundabout is free software: you can redistribute it and/or modify
@@ -98,6 +98,10 @@
     'import_export', # simple model import/export using admin interface
     'django_tables2', # interactive tables views
     'django_tables2_column_shifter', # show/hide tables2 columns
+    'django_filters', #filters for API searching
+    'dynamic_rest', # dynamic functionality for API
+    'rest_framework.authtoken',
+    'rest_flex_fields',
 ]
 LOCAL_APPS = [
     'roundabout.users.apps.UsersAppConfig',
@@ -114,6 +118,7 @@
     'roundabout.cruises',
     'roundabout.calibrations',
     'roundabout.configs_constants',
+    'roundabout.field_instances',
     'roundabout.search',
     'roundabout.exports',
 ]
@@ -301,12 +306,20 @@
     # When you enable API versioning, the request.version attribute will contain a string
     # that corresponds to the version requested in the incoming client request.
     'DEFAULT_VERSIONING_CLASS': 'rest_framework.versioning.URLPathVersioning',
+    'DEFAULT_FILTER_BACKENDS': ['django_filters.rest_framework.DjangoFilterBackend'],
     'DEFAULT_AUTHENTICATION_CLASSES': (
-        'rest_framework_simplejwt.authentication.JWTAuthentication',
+        'rest_framework.authentication.SessionAuthentication',
+        'rest_framework.authentication.TokenAuthentication',
     ),
-    #'DEFAULT_PERMISSION_CLASSES': [
-    #    'rest_framework.permissions.IsAuthenticated',
-    #]
+    'DEFAULT_RENDERER_CLASSES': (
+        'rest_framework.renderers.JSONRenderer',
+        'roundabout.core.api.renderers.BrowsableAPIRendererWithoutForms',
+    ),
+    'DEFAULT_PAGINATION_CLASS': 'drf_link_header_pagination.LinkHeaderPagination',
+    'PAGE_SIZE': 30,
+    'DEFAULT_PERMISSION_CLASSES': [
+        'rest_framework.permissions.IsAuthenticated',
+    ]
 }
 
 # Summernote CONFIGURATION

config/urls.py

@@ -26,10 +26,6 @@
 from django.views.generic import TemplateView
 from django.views import defaults as default_views
 
-from rest_framework_simplejwt.views import (
-    TokenObtainPairView,
-    TokenRefreshView,
-)
 
 urlpatterns = [
     path("", TemplateView.as_view(template_name="pages/home.html"), name="home"),
@@ -60,14 +56,9 @@
     path('export/', include('roundabout.exports.urls', namespace='export')),
     path('calibrations/', include('roundabout.calibrations.urls', namespace='calibrations')),
     path('configs_constants/', include('roundabout.configs_constants.urls', namespace='configs_constants')),
+    path('field_instances/', include('roundabout.field_instances.urls', namespace='field_instances')),
     # API urls
-    path('api/v1/', include('roundabout.inventory.api.urls')),
-    path('api/v1/', include('roundabout.locations.api.urls')),
-    path('api/v1/', include('roundabout.parts.api.urls')),
-    path('api/v1/', include('roundabout.assemblies.api.urls')),
-    # API JWT token paths
-    path('api/token/', TokenObtainPairView.as_view(), name='token_obtain_pair'),
-    path('api/refresh/', TokenRefreshView.as_view(), name='token_refresh'),
+    path('api/v1/', include('roundabout.core.api.urls', namespace='api_v1')),
     #Summernote WYSIWYG
     path('summernote/', include('django_summernote.urls')),
 ] + static(

requirements/base.txt

@@ -17,6 +17,10 @@ django-redis==4.10.0  # https://github.com/niwinz/django-redis
 djangorestframework==3.10.3 # https://github.com/encode/django-rest-framework
 djangorestframework_simplejwt==4.3.0
 coreapi==2.3.3  # https://github.com/core-api/python-client
+django-filter==2.2.0
+dynamic-rest==1.9.6
+djangorestframework-link-header-pagination==0.1.1
+drf-flex-fields==0.8.6
 
 # Your custom requirements go here
 django-mptt==0.9.1

roundabout/admintools/admin.py

@@ -1,7 +1,7 @@
 """
 # Copyright (C) 2019-2020 Woods Hole Oceanographic Institution
 #
-# This file is part of the Roundabout Database project ("RDB" or 
+# This file is part of the Roundabout Database project ("RDB" or
 # "ooicgsn-roundabout").
 #
 # ooicgsn-roundabout is free software: you can redistribute it and/or modify

roundabout/admintools/forms.py

@@ -56,80 +56,88 @@ class ImportCalibrationForm(forms.Form):
 
     def clean_cal_csv(self):
         cal_files = self.files.getlist('cal_csv')
-        for cal_csv in cal_files:
-            cal_csv.seek(0)
-            ext = cal_csv.name[-3:]
+        csv_files = []
+        ext_files = []
+        for file in cal_files:
+            ext = file.name[-3:]
             if ext == 'ext':
-                continue
+                ext_files.append(file)
             if ext == 'csv':
-                reader = csv.DictReader(io.StringIO(cal_csv.read().decode('utf-8')))
-                headers = reader.fieldnames
-                try:
-                    inv_serial = cal_csv.name.split('__')[0]
-                    inventory_item = Inventory.objects.get(serial_number=inv_serial)
-                except:
-                    raise ValidationError(
-                        _('%(value)s: Unable to find Inventory item with this Serial Number'),
-                        params={'value': inv_serial},
-                    )
-                try:
-                    cal_date_string = cal_csv.name.split('__')[1][:8]
-                    cal_date_date = datetime.datetime.strptime(cal_date_string, "%Y%m%d").date()
-                except:
-                    raise ValidationError(
-                        _('%(value)s: Unable to parse Calibration Date from Filename'),
-                        params={'value': cal_date_string},
-                    )
-                for idx, row in enumerate(reader):
-                    row_data = row.items()
-                    for key, value in row_data:
-                        if key == 'name':
-                            calibration_name = value.strip()
-                            try:
-                                cal_name_item = CoefficientName.objects.get(
-                                    calibration_name = calibration_name,
-                                    coeff_name_event =  inventory_item.part.coefficient_name_events.first()
-                                )
-                            except:
-                                raise ValidationError(
-                                    _('Row %(row)s, %(value)s: Unable to find Calibration item with this Name'),
-                                    params={'value': calibration_name, 'row': idx},
-                                )
-                        elif key == 'value':
-                            valset_keys = {'cal_dec_places': inventory_item.part.cal_dec_places}
-                            mock_valset_instance = SimpleNamespace(**valset_keys)
-                            try:
-                                raw_valset = str(value)
-                                if '[' in raw_valset:
-                                    raw_valset = raw_valset[1:-1]
-                                if 'SheetRef' in raw_valset:
-                                    for file in cal_files:
-                                        file.seek(0)
-                                        file_extension = file.name[-3:]
-                                        if file_extension == 'ext':
-                                            cal_ext_split = file.name.split('__')
-                                            inv_ext_serial = cal_ext_split[0]
-                                            cal_ext_date = cal_ext_split[1]
-                                            cal_ext_name = cal_ext_split[2][:-4]
-                                            if (inv_ext_serial == inv_serial) and (cal_ext_date == cal_date_string) and (cal_ext_name == calibration_name):
-                                                reader = io.StringIO(file.read().decode('utf-8'))
-                                                contents = reader.getvalue()
-                                                raw_valset = contents
-                            except:
-                                raise ValidationError(
-                                    _('Row %(row)s: Unable to parse Calibration Coefficient value(s)'),
-                                    params={'row': idx},
-                                )
-                            validate_coeff_vals(mock_valset_instance, cal_name_item.value_set_type, raw_valset)
-                        elif key == 'notes':
+                csv_files.append(file)
+        for cal_csv in csv_files:
+            cal_csv_filename = cal_csv.name[:-4]
+            cal_csv.seek(0)
+            reader = csv.DictReader(io.StringIO(cal_csv.read().decode('utf-8')))
+            headers = reader.fieldnames
+            try:
+                inv_serial = cal_csv.name.split('__')[0]
+                inventory_item = Inventory.objects.get(serial_number=inv_serial)
+            except:
+                raise ValidationError(
+                    _('File: %(filename)s, %(value)s: Unable to find Inventory item with this Serial Number'),
+                    params={'value': inv_serial, 'filename': cal_csv.name},
+                )
+            try:
+                cal_date_string = cal_csv.name.split('__')[1][:8]
+                cal_date_date = datetime.datetime.strptime(cal_date_string, "%Y%m%d").date()
+            except:
+                raise ValidationError(
+                    _('File: %(filename)s, %(value)s: Unable to parse Calibration Date from Filename'),
+                    params={'value': cal_date_string, 'filename': cal_csv.name},
+                )
+            for idx, row in enumerate(reader):
+                row_data = row.items()
+                for key, value in row_data:
+                    if key == 'name':
+                        calibration_name = value.strip()
+                        try:
+                            cal_name_item = CoefficientName.objects.get(
+                                calibration_name = calibration_name,
+                                coeff_name_event =  inventory_item.part.coefficient_name_events.first()
+                            )
+                        except:
+                            raise ValidationError(
+                                _('File: %(filename)s, Calibration Name: %(value)s, Row %(row)s: Unable to find Calibration item with this Name'),
+                                params={'value': calibration_name, 'row': idx, 'filename': cal_csv.name},
+                            )
+                    elif key == 'value':
+                        valset_keys = {'cal_dec_places': inventory_item.part.cal_dec_places}
+                        mock_valset_instance = SimpleNamespace(**valset_keys)
+                        try:
+                            raw_valset = str(value)
+                        except:
+                            raise ValidationError(
+                                _('File: %(filename)s, Calibration Name: %(value)s, Row %(row)s, %(value)s: Unable to parse Calibration Coefficient value(s)'),
+                                params={'value': calibration_name,'row': idx, 'filename': cal_csv.name},
+                            )
+                        if '[' in raw_valset:
+                            raw_valset = raw_valset[1:-1]
+                        if 'SheetRef' in raw_valset:
+                            ext_finder_filename = "__".join((cal_csv_filename,calibration_name))
                             try:
-                                notes = value.strip()
+                                ref_file = [file for file in ext_files if ext_finder_filename in file.name][0]
+                                assert len(ref_file) > 0
                             except:
                                 raise ValidationError(
-                                    _('Row %(row)s: Unable to parse Calibration Coefficient note(s)'),
-                                    params={'row': idx},
+                                    _('File: %(filename)s, Calibration Name: %(value)s, Row %(row)s: No associated .ext file selected'),
+                                    params={'value': calibration_name, 'row': idx, 'filename': cal_csv.name},
                                 )
-        return cal_csv
+                            ref_file.seek(0)
+                            reader = io.StringIO(ref_file.read().decode('utf-8'))
+                            contents = reader.getvalue()
+                            raw_valset = contents
+                            validate_coeff_vals(mock_valset_instance, cal_name_item.value_set_type, raw_valset, filename = ref_file.name, cal_name = calibration_name)
+                        else:
+                            validate_coeff_vals(mock_valset_instance, cal_name_item.value_set_type, raw_valset, filename = cal_csv.name, cal_name = calibration_name)
+                    elif key == 'notes':
+                        try:
+                            notes = value.strip()
+                        except:
+                            raise ValidationError(
+                                _('File: %(filename)s, Calibration Name: %(value)s, Row %(row)s: Unable to parse Calibration Coefficient note(s)'),
+                                params={'value': calibration_name, 'row': idx, 'filename': cal_csv.name},
+                            )
+        return cal_files
 
 
 class PrinterForm(forms.ModelForm):

roundabout/admintools/models.py

@@ -1,7 +1,7 @@
 """
 # Copyright (C) 2019-2020 Woods Hole Oceanographic Institution
 #
-# This file is part of the Roundabout Database project ("RDB" or 
+# This file is part of the Roundabout Database project ("RDB" or
 # "ooicgsn-roundabout").
 #
 # ooicgsn-roundabout is free software: you can redistribute it and/or modify
@@ -20,14 +20,16 @@
 """
 
 from django.db import models
+from django.utils import timezone
 from mptt.models import MPTTModel, TreeForeignKey
 from django.contrib.postgres.fields import JSONField
 
 from roundabout.assemblies.models import AssemblyType
 from roundabout.parts.models import Part
+from roundabout.users.models import User
+from roundabout.cruises.models import Cruise
 
 # AdminTool models
-
 class Printer(models.Model):
     PRINTER_TYPES = (
         ('Brady', 'Brady'),

roundabout/admintools/urls.py

@@ -1,7 +1,7 @@
 """
 # Copyright (C) 2019-2020 Woods Hole Oceanographic Institution
 #
-# This file is part of the Roundabout Database project ("RDB" or 
+# This file is part of the Roundabout Database project ("RDB" or
 # "ooicgsn-roundabout").
 #
 # ooicgsn-roundabout is free software: you can redistribute it and/or modify