From 32186e0de1328e8213edb265c3e1f98b06a6c019 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 29 Nov 2020 16:33:33 +0100
Subject: [PATCH 01/78] added a menu for bulk edits.

---
 .../document-list.component.html              | 26 +++++++++++++++++++
 1 file changed, 26 insertions(+)

diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html
index cc682b8e3..d142fbb04 100644
--- a/src-ui/src/app/components/document-list/document-list.component.html
+++ b/src-ui/src/app/components/document-list/document-list.component.html
@@ -1,5 +1,31 @@
 <app-page-header [title]="getTitle()">
 
+  <div ngbDropdown class="d-inline-block mr-2">
+    <button class="btn btn-sm btn-outline-primary" id="dropdownBasic1" ngbDropdownToggle>
+      <svg class="toolbaricon" fill="currentColor">
+        <use xlink:href="assets/bootstrap-icons.svg#text-indent-left" />
+      </svg>
+      Bulk edit
+    </button>
+    <div ngbDropdownMenu aria-labelledby="dropdownBasic1">
+      <button ngbDropdownItem>Select page</button>
+      <button ngbDropdownItem>Select all</button>
+      <button ngbDropdownItem>Select none</button>
+      <div class="dropdown-divider"></div>
+      <button ngbDropdownItem>Re-create archived document</button>
+      <div class="dropdown-divider"></div>
+      <button ngbDropdownItem>Set correspondent</button>
+      <button ngbDropdownItem>Remove correspondent</button>
+      <button ngbDropdownItem>Set document type</button>
+      <button ngbDropdownItem>Remove document type</button>
+      <button ngbDropdownItem>Add tag</button>
+      <button ngbDropdownItem>Remove tag</button>
+      <div class="dropdown-divider"></div>
+      <button ngbDropdownItem>Delete</button>
+
+    </div>
+  </div>
+
   <div class="btn-group btn-group-toggle" ngbRadioGroup [(ngModel)]="displayMode"
     (ngModelChange)="saveDisplayMode()">
     <label ngbButtonLabel class="btn-outline-primary btn-sm">

From fd4c9a1758b550218ba7fd62f762cc7313c69923 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 29 Nov 2020 23:00:52 +0100
Subject: [PATCH 02/78] not sure if this works

---
 src/documents/forms.py | 12 ++++++++++++
 src/documents/views.py | 11 +++++++++++
 2 files changed, 23 insertions(+)

diff --git a/src/documents/forms.py b/src/documents/forms.py
index 63dd307b2..0c73c3810 100644
--- a/src/documents/forms.py
+++ b/src/documents/forms.py
@@ -1,4 +1,5 @@
 import os
+import re
 import tempfile
 from datetime import datetime
 from time import mktime
@@ -11,6 +12,17 @@
 
 from documents.parsers import is_mime_type_supported
 
+class BuldEditForm(forms.Form):
+
+    def clean_ids(self):
+        ids = self.cleaned_data.get("ids")
+        if not re.match(r"[0-9,]+", ids):
+            raise forms.ValidationError("id list invalid")
+        id_list = [int(id) for id in ids.split(",")]
+
+
+
+
 
 class UploadForm(forms.Form):
 
diff --git a/src/documents/views.py b/src/documents/views.py
index 84f4a3999..ee45af267 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -93,6 +93,10 @@ class DocumentTypeViewSet(ModelViewSet):
     ordering_fields = ("name", "matching_algorithm", "match", "document_count")
 
 
+class BulkEditForm(object):
+    pass
+
+
 class DocumentViewSet(RetrieveModelMixin,
                       UpdateModelMixin,
                       DestroyModelMixin,
@@ -149,6 +153,13 @@ def post_document(self, request, pk=None):
         else:
             return HttpResponseBadRequest(str(form.errors))
 
+    @action(methods=['post'], detail=False)
+    def bulk_edit(self, request, pk=None):
+        form = BulkEditForm(data=request.POST)
+        if not form.is_valid():
+            return HttpResponseBadRequest("")
+        return Response({'asd': request.POST['content']})
+
     @action(methods=['get'], detail=True)
     def metadata(self, request, pk=None):
         try:

From 35124023f0a207a6b127210b8661688325aa541e Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Mon, 30 Nov 2020 13:58:40 +0100
Subject: [PATCH 03/78] basic support for bulk editing.

---
 .../src/app/services/rest/document.service.ts |  8 ++++
 src/documents/bulk_edit.py                    | 39 +++++++++++++++++++
 src/documents/forms.py                        | 12 ------
 src/documents/views.py                        |  9 +++--
 4 files changed, 52 insertions(+), 16 deletions(-)
 create mode 100644 src/documents/bulk_edit.py

diff --git a/src-ui/src/app/services/rest/document.service.ts b/src-ui/src/app/services/rest/document.service.ts
index cdea89914..07e69c87a 100644
--- a/src-ui/src/app/services/rest/document.service.ts
+++ b/src-ui/src/app/services/rest/document.service.ts
@@ -66,4 +66,12 @@ export class DocumentService extends AbstractPaperlessService<PaperlessDocument>
     return this.http.post(this.getResourceUrl(null, 'post_document'), formData)
   }
 
+  bulk_edit(ids: number[], method: string, args: any[]) {
+    return this.http.post(this.getResourceUrl(null, 'bulk_edit'), {
+      'ids': ids,
+      'method': method,
+      'args': args
+    })
+  }
+
 }
diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py
new file mode 100644
index 000000000..ef5d3f509
--- /dev/null
+++ b/src/documents/bulk_edit.py
@@ -0,0 +1,39 @@
+from documents.models import Document
+
+
+methods_supported = [
+    "set_correspondent"
+]
+
+
+def validate_data(data):
+    if 'ids' not in data or not isinstance(data['ids'], list):
+        raise ValueError()
+    ids = data['ids']
+    if not all([isinstance(i, int) for i in ids]):
+        raise ValueError()
+    count = Document.objects.filter(pk__in=ids).count()
+    if not count == len(ids):
+        raise Document.DoesNotExist()
+
+    if 'method' not in data or not isinstance(data['method'], str):
+        raise ValueError()
+    method = data['method']
+    if method not in methods_supported:
+        raise ValueError()
+
+    if 'args' not in data or not isinstance(data['args'], list):
+        raise ValueError()
+    parameters = data['args']
+
+    return ids, method, parameters
+
+
+def perform_bulk_edit(data):
+    ids, method, args = validate_data(data)
+
+    getattr(__file__, method)(ids, args)
+
+
+def set_correspondent(ids, args):
+    print("WOW")
diff --git a/src/documents/forms.py b/src/documents/forms.py
index 0c73c3810..63dd307b2 100644
--- a/src/documents/forms.py
+++ b/src/documents/forms.py
@@ -1,5 +1,4 @@
 import os
-import re
 import tempfile
 from datetime import datetime
 from time import mktime
@@ -12,17 +11,6 @@
 
 from documents.parsers import is_mime_type_supported
 
-class BuldEditForm(forms.Form):
-
-    def clean_ids(self):
-        ids = self.cleaned_data.get("ids")
-        if not re.match(r"[0-9,]+", ids):
-            raise forms.ValidationError("id list invalid")
-        id_list = [int(id) for id in ids.split(",")]
-
-
-
-
 
 class UploadForm(forms.Form):
 
diff --git a/src/documents/views.py b/src/documents/views.py
index ee45af267..95448ad62 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -23,6 +23,7 @@
 import documents.index as index
 from paperless.db import GnuPG
 from paperless.views import StandardPagination
+from .bulk_edit import perform_bulk_edit
 from .filters import (
     CorrespondentFilterSet,
     DocumentFilterSet,
@@ -155,10 +156,10 @@ def post_document(self, request, pk=None):
 
     @action(methods=['post'], detail=False)
     def bulk_edit(self, request, pk=None):
-        form = BulkEditForm(data=request.POST)
-        if not form.is_valid():
-            return HttpResponseBadRequest("")
-        return Response({'asd': request.POST['content']})
+        try:
+            return Response(perform_bulk_edit(request.data))
+        except Exception as e:
+            return HttpResponseBadRequest(str(e))
 
     @action(methods=['get'], detail=True)
     def metadata(self, request, pk=None):

From 5369e0be037b0ea373d655b6f7ecd246efec792d Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 6 Dec 2020 14:39:53 +0100
Subject: [PATCH 04/78] more bulk edit

---
 src/documents/bulk_edit.py   | 16 +++++++++++++---
 src/documents/serialisers.py | 28 ++++++++++++++++++++++++++++
 src/documents/views.py       | 32 ++++++++++++++++++++++++--------
 3 files changed, 65 insertions(+), 11 deletions(-)

diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py
index ef5d3f509..f80c55c58 100644
--- a/src/documents/bulk_edit.py
+++ b/src/documents/bulk_edit.py
@@ -1,5 +1,4 @@
-from documents.models import Document
-
+from documents.models import Document, Correspondent
 
 methods_supported = [
     "set_correspondent"
@@ -36,4 +35,15 @@ def perform_bulk_edit(data):
 
 
 def set_correspondent(ids, args):
-    print("WOW")
+    if not len(args) == 1:
+        raise ValueError()
+
+    if not args[0]:
+        correspondent = None
+    else:
+        if not isinstance(args[0], int):
+            raise ValueError()
+
+        correspondent = Correspondent.objects.get(args[0])
+
+    Document.objects.filter(id__in=ids).update(correspondent=correspondent)
diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py
index c988b2137..a8da79cdd 100644
--- a/src/documents/serialisers.py
+++ b/src/documents/serialisers.py
@@ -113,6 +113,34 @@ class Meta:
         )
 
 
+class BulkEditSerializer(serializers.Serializer):
+
+    documents = serializers.PrimaryKeyRelatedField(
+        many=True,
+        label="Documents",
+        write_only=True,
+        queryset=Document.objects.all()
+    )
+
+    method = serializers.ChoiceField(
+        choices=[
+            "set_correspondent",
+            "set_document_type",
+            "add_tag",
+            "remove_tag",
+            "delete"
+        ],
+        label="Method",
+        write_only=True,
+    )
+
+    parameters = serializers.DictField(allow_empty=True)
+
+    def validate(self, attrs):
+
+        return attrs
+
+
 class PostDocumentSerializer(serializers.Serializer):
 
     document = serializers.FileField(
diff --git a/src/documents/views.py b/src/documents/views.py
index 219cc61b7..88ceb2efd 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -46,7 +46,8 @@
     LogSerializer,
     TagSerializer,
     DocumentTypeSerializer,
-    PostDocumentSerializer
+    PostDocumentSerializer,
+    BulkEditSerializer
 )
 
 
@@ -165,13 +166,6 @@ def file_response(self, pk, request, disposition):
             disposition, filename)
         return response
 
-    @action(methods=['post'], detail=False)
-    def bulk_edit(self, request, pk=None):
-        try:
-            return Response(perform_bulk_edit(request.data))
-        except Exception as e:
-            return HttpResponseBadRequest(str(e))
-
     @action(methods=['get'], detail=True)
     def metadata(self, request, pk=None):
         try:
@@ -225,6 +219,28 @@ class LogViewSet(ReadOnlyModelViewSet):
     ordering_fields = ("created",)
 
 
+class BulkEditView(APIView):
+
+    permission_classes = (IsAuthenticated,)
+    serializer_class = BulkEditSerializer
+    parser_classes = (parsers.JSONParser,)
+
+    def get_serializer_context(self):
+        return {
+            'request': self.request,
+            'format': self.format_kwarg,
+            'view': self
+        }
+
+    def get_serializer(self, *args, **kwargs):
+        kwargs['context'] = self.get_serializer_context()
+        return self.serializer_class(*args, **kwargs)
+
+    def post(self, request, *args, **kwargs):
+        serializer = self.get_serializer(data=request.data)
+        serializer.is_valid(raise_exception=True)
+
+
 class PostDocumentView(APIView):
 
     permission_classes = (IsAuthenticated,)

From fbb3a069cd20118a75b4cb5b9fe057d474d1fb83 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 14:27:54 +0100
Subject: [PATCH 05/78] add bulk editing methods

---
 src/documents/bulk_edit.py | 90 +++++++++++++++++++++++---------------
 src/documents/tasks.py     |  7 +++
 2 files changed, 62 insertions(+), 35 deletions(-)

diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py
index f80c55c58..1349f9d54 100644
--- a/src/documents/bulk_edit.py
+++ b/src/documents/bulk_edit.py
@@ -1,49 +1,69 @@
-from documents.models import Document, Correspondent
+from django.db.models import Q
+from django_q.tasks import async_task
 
-methods_supported = [
-    "set_correspondent"
-]
+from documents.models import Document, Correspondent, DocumentType
 
 
-def validate_data(data):
-    if 'ids' not in data or not isinstance(data['ids'], list):
-        raise ValueError()
-    ids = data['ids']
-    if not all([isinstance(i, int) for i in ids]):
-        raise ValueError()
-    count = Document.objects.filter(pk__in=ids).count()
-    if not count == len(ids):
-        raise Document.DoesNotExist()
+def set_correspondent(doc_ids, correspondent):
+    if correspondent:
+        correspondent = Correspondent.objects.get(id=correspondent)
 
-    if 'method' not in data or not isinstance(data['method'], str):
-        raise ValueError()
-    method = data['method']
-    if method not in methods_supported:
-        raise ValueError()
+    qs = Document.objects.filter(Q(id__in=doc_ids) & ~Q(correspondent=correspondent))
+    affected_docs = [doc.id for doc in qs]
+    qs.update(correspondent=correspondent)
 
-    if 'args' not in data or not isinstance(data['args'], list):
-        raise ValueError()
-    parameters = data['args']
+    async_task("documents.tasks.bulk_rename_files", affected_docs)
 
-    return ids, method, parameters
+    return "OK"
 
 
-def perform_bulk_edit(data):
-    ids, method, args = validate_data(data)
+def set_document_type(doc_ids, document_type):
+    if document_type:
+        document_type = DocumentType.objects.get(id=document_type)
 
-    getattr(__file__, method)(ids, args)
+    qs = Document.objects.filter(Q(id__in=doc_ids) & ~Q(document_type=document_type))
+    affected_docs = [doc.id for doc in qs]
+    qs.update(document_type=document_type)
 
+    async_task("documents.tasks.bulk_rename_files", affected_docs)
 
-def set_correspondent(ids, args):
-    if not len(args) == 1:
-        raise ValueError()
+    return "OK"
 
-    if not args[0]:
-        correspondent = None
-    else:
-        if not isinstance(args[0], int):
-            raise ValueError()
 
-        correspondent = Correspondent.objects.get(args[0])
+def add_tag(doc_ids, tag):
 
-    Document.objects.filter(id__in=ids).update(correspondent=correspondent)
+    qs = Document.objects.filter(Q(id__in=doc_ids) & ~Q(tags__id=tag))
+    affected_docs = [doc.id for doc in qs]
+
+    DocumentTagRelationship = Document.tags.through
+
+    DocumentTagRelationship.objects.bulk_create([
+        DocumentTagRelationship(document_id=doc, tag_id=tag) for doc in affected_docs
+    ])
+
+    async_task("documents.tasks.bulk_rename_files", affected_docs)
+
+    return "OK"
+
+
+def remove_tag(doc_ids, tag):
+
+    qs = Document.objects.filter(Q(id__in=doc_ids) & Q(tags__id=tag))
+    affected_docs = [doc.id for doc in qs]
+
+    DocumentTagRelationship = Document.tags.through
+
+    DocumentTagRelationship.objects.filter(
+        Q(document_id__in=affected_docs) &
+        Q(tag_id=tag)
+    ).delete()
+
+    async_task("documents.tasks.bulk_rename_files", affected_docs)
+
+    return "OK"
+
+
+def delete(doc_ids):
+    Document.objects.filter(id__in=doc_ids).delete()
+
+    return "OK"
diff --git a/src/documents/tasks.py b/src/documents/tasks.py
index 8c9b00dd6..af4c91448 100644
--- a/src/documents/tasks.py
+++ b/src/documents/tasks.py
@@ -2,6 +2,7 @@
 
 import tqdm
 from django.conf import settings
+from django.db.models.signals import post_save
 from whoosh.writing import AsyncWriter
 
 from documents import index, sanity_checker
@@ -87,3 +88,9 @@ def sanity_check():
         raise SanityFailedError(messages)
     else:
         return "No issues detected."
+
+
+def bulk_rename_files(ids):
+    qs = Document.objects.filter(id__in=ids)
+    for doc in qs:
+        post_save.send(Document, instance=doc, created=False)

From 66d6d29c239670575c0dc0c100234764f8eeead9 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 14:29:43 +0100
Subject: [PATCH 06/78] add support to the documents api to only serve selected
 fields

---
 src/documents/serialisers.py | 24 +++++++++++++++++++++++-
 src/documents/views.py       | 11 +++++++++++
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py
index faded5125..69cbb4092 100644
--- a/src/documents/serialisers.py
+++ b/src/documents/serialisers.py
@@ -7,6 +7,28 @@
 from .parsers import is_mime_type_supported
 
 
+# https://www.django-rest-framework.org/api-guide/serializers/#example
+class DynamicFieldsModelSerializer(serializers.ModelSerializer):
+    """
+    A ModelSerializer that takes an additional `fields` argument that
+    controls which fields should be displayed.
+    """
+
+    def __init__(self, *args, **kwargs):
+        # Don't pass the 'fields' arg up to the superclass
+        fields = kwargs.pop('fields', None)
+
+        # Instantiate the superclass normally
+        super(DynamicFieldsModelSerializer, self).__init__(*args, **kwargs)
+
+        if fields is not None:
+            # Drop any fields that are not specified in the `fields` argument.
+            allowed = set(fields)
+            existing = set(self.fields)
+            for field_name in existing - allowed:
+                self.fields.pop(field_name)
+
+
 class CorrespondentSerializer(serializers.ModelSerializer):
 
     document_count = serializers.IntegerField(read_only=True)
@@ -90,7 +112,7 @@ def get_queryset(self):
         return DocumentType.objects.all()
 
 
-class DocumentSerializer(serializers.ModelSerializer):
+class DocumentSerializer(DynamicFieldsModelSerializer):
 
     correspondent = CorrespondentField(allow_null=True)
     tags = TagsField(many=True)
diff --git a/src/documents/views.py b/src/documents/views.py
index b1d93b77b..10cb30eb3 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -131,6 +131,17 @@ class DocumentViewSet(RetrieveModelMixin,
         "added",
         "archive_serial_number")
 
+    def get_serializer(self, *args, **kwargs):
+        fields_param = self.request.query_params.get('fields', None)
+        if fields_param:
+            fields = fields_param.split(",")
+        else:
+            fields = None
+        serializer_class = self.get_serializer_class()
+        kwargs.setdefault('context', self.get_serializer_context())
+        kwargs.setdefault('fields', fields)
+        return serializer_class(*args, **kwargs)
+
     def update(self, request, *args, **kwargs):
         response = super(DocumentViewSet, self).update(
             request, *args, **kwargs)

From 4b0027797a6c70879bc9ce2850c645ca136931c7 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 14:30:18 +0100
Subject: [PATCH 07/78] bulk edit view

---
 src/documents/serialisers.py | 22 ++++++++++++++++++----
 src/documents/views.py       | 11 +++++++++++
 src/paperless/urls.py        |  7 ++++++-
 3 files changed, 35 insertions(+), 5 deletions(-)

diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py
index 69cbb4092..5418ec0fb 100644
--- a/src/documents/serialisers.py
+++ b/src/documents/serialisers.py
@@ -3,6 +3,7 @@
 from rest_framework import serializers
 from rest_framework.fields import SerializerMethodField
 
+from . import bulk_edit
 from .models import Correspondent, Tag, Document, Log, DocumentType
 from .parsers import is_mime_type_supported
 
@@ -164,11 +165,10 @@ class Meta:
 
 class BulkEditSerializer(serializers.Serializer):
 
-    documents = serializers.PrimaryKeyRelatedField(
-        many=True,
+    documents = serializers.ListField(
+        child=serializers.IntegerField(),
         label="Documents",
-        write_only=True,
-        queryset=Document.objects.all()
+        write_only=True
     )
 
     method = serializers.ChoiceField(
@@ -185,6 +185,20 @@ class BulkEditSerializer(serializers.Serializer):
 
     parameters = serializers.DictField(allow_empty=True)
 
+    def validate_method(self, method):
+        if method == "set_correspondent":
+            return bulk_edit.set_correspondent
+        elif method == "set_document_type":
+            return bulk_edit.set_document_type
+        elif method == "add_tag":
+            return bulk_edit.add_tag
+        elif method == "remove_tag":
+            return bulk_edit.remove_tag
+        elif method == "delete":
+            return bulk_edit.delete
+        else:
+            raise serializers.ValidationError("Unsupported method.")
+
     def validate(self, attrs):
 
         return attrs
diff --git a/src/documents/views.py b/src/documents/views.py
index 10cb30eb3..4ce78348e 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -278,6 +278,17 @@ def post(self, request, *args, **kwargs):
         serializer = self.get_serializer(data=request.data)
         serializer.is_valid(raise_exception=True)
 
+        method = serializer.validated_data.get("method")
+        parameters = serializer.validated_data.get("parameters")
+        documents = serializer.validated_data.get("documents")
+
+        try:
+            # TODO: parameter validation
+            result = method(documents, **parameters)
+            return Response({"result": result})
+        except Exception as e:
+            return HttpResponseBadRequest(str(e))
+
 
 class PostDocumentView(APIView):
 
diff --git a/src/paperless/urls.py b/src/paperless/urls.py
index 9b390b139..dc416f05f 100755
--- a/src/paperless/urls.py
+++ b/src/paperless/urls.py
@@ -17,7 +17,8 @@
     IndexView,
     SearchAutoCompleteView,
     StatisticsView,
-    PostDocumentView
+    PostDocumentView,
+    BulkEditView
 )
 from paperless.views import FaviconView
 
@@ -50,6 +51,10 @@
         re_path(r"^documents/post_document/", PostDocumentView.as_view(),
                 name="post_document"),
 
+
+        re_path(r"^documents/bulk_edit/", BulkEditView.as_view(),
+                name="bulk_edit"),
+
         path('token/', views.obtain_auth_token)
 
     ] + api_router.urls)),

From 63a58ccc38e2da9ed5a02fb3e650c9fb61ba1c2b Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 14:30:59 +0100
Subject: [PATCH 08/78] a simple dialog that selects tags/correspondents/types

---
 src-ui/src/app/app.module.ts                  |  4 ++-
 .../select-dialog.component.html              | 15 ++++++++
 .../select-dialog.component.scss              |  0
 .../select-dialog.component.spec.ts           | 25 ++++++++++++++
 .../select-dialog/select-dialog.component.ts  | 34 +++++++++++++++++++
 5 files changed, 77 insertions(+), 1 deletion(-)
 create mode 100644 src-ui/src/app/components/common/select-dialog/select-dialog.component.html
 create mode 100644 src-ui/src/app/components/common/select-dialog/select-dialog.component.scss
 create mode 100644 src-ui/src/app/components/common/select-dialog/select-dialog.component.spec.ts
 create mode 100644 src-ui/src/app/components/common/select-dialog/select-dialog.component.ts

diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts
index ad12c9c47..af66993c6 100644
--- a/src-ui/src/app/app.module.ts
+++ b/src-ui/src/app/app.module.ts
@@ -48,6 +48,7 @@ import { WidgetFrameComponent } from './components/dashboard/widgets/widget-fram
 import { WelcomeWidgetComponent } from './components/dashboard/widgets/welcome-widget/welcome-widget.component';
 import { YesNoPipe } from './pipes/yes-no.pipe';
 import { FileSizePipe } from './pipes/file-size.pipe';
+import { SelectDialogComponent } from './components/common/select-dialog/select-dialog.component';
 
 @NgModule({
   declarations: [
@@ -88,7 +89,8 @@ import { FileSizePipe } from './pipes/file-size.pipe';
     WidgetFrameComponent,
     WelcomeWidgetComponent,
     YesNoPipe,
-    FileSizePipe
+    FileSizePipe,
+    SelectDialogComponent
   ],
   imports: [
     BrowserModule,
diff --git a/src-ui/src/app/components/common/select-dialog/select-dialog.component.html b/src-ui/src/app/components/common/select-dialog/select-dialog.component.html
new file mode 100644
index 000000000..8bde38d61
--- /dev/null
+++ b/src-ui/src/app/components/common/select-dialog/select-dialog.component.html
@@ -0,0 +1,15 @@
+<div class="modal-header">
+  <h4 class="modal-title" id="modal-basic-title">{{title}}</h4>
+  <button type="button" class="close" aria-label="Close" (click)="cancelClicked()">
+    <span aria-hidden="true">&times;</span>
+  </button>
+</div>
+<div class="modal-body">
+
+  <app-input-select [items]="objects" [title]="message" [(ngModel)]="selected"></app-input-select>
+
+</div>
+<div class="modal-footer">
+  <button type="button" class="btn btn-outline-dark" (click)="cancelClicked()">Cancel</button>
+  <button type="button" class="btn btn-primary" (click)="selectClicked.emit(selected)">Select</button>
+</div>
\ No newline at end of file
diff --git a/src-ui/src/app/components/common/select-dialog/select-dialog.component.scss b/src-ui/src/app/components/common/select-dialog/select-dialog.component.scss
new file mode 100644
index 000000000..e69de29bb
diff --git a/src-ui/src/app/components/common/select-dialog/select-dialog.component.spec.ts b/src-ui/src/app/components/common/select-dialog/select-dialog.component.spec.ts
new file mode 100644
index 000000000..3810bcbea
--- /dev/null
+++ b/src-ui/src/app/components/common/select-dialog/select-dialog.component.spec.ts
@@ -0,0 +1,25 @@
+import { ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { SelectDialogComponent } from './select-dialog.component';
+
+describe('SelectDialogComponent', () => {
+  let component: SelectDialogComponent;
+  let fixture: ComponentFixture<SelectDialogComponent>;
+
+  beforeEach(async () => {
+    await TestBed.configureTestingModule({
+      declarations: [ SelectDialogComponent ]
+    })
+    .compileComponents();
+  });
+
+  beforeEach(() => {
+    fixture = TestBed.createComponent(SelectDialogComponent);
+    component = fixture.componentInstance;
+    fixture.detectChanges();
+  });
+
+  it('should create', () => {
+    expect(component).toBeTruthy();
+  });
+});
diff --git a/src-ui/src/app/components/common/select-dialog/select-dialog.component.ts b/src-ui/src/app/components/common/select-dialog/select-dialog.component.ts
new file mode 100644
index 000000000..76b23491c
--- /dev/null
+++ b/src-ui/src/app/components/common/select-dialog/select-dialog.component.ts
@@ -0,0 +1,34 @@
+import { Component, EventEmitter, Input, OnInit, Output } from '@angular/core';
+import { NgbActiveModal } from '@ng-bootstrap/ng-bootstrap';
+import { ObjectWithId } from 'src/app/data/object-with-id';
+
+@Component({
+  selector: 'app-select-dialog',
+  templateUrl: './select-dialog.component.html',
+  styleUrls: ['./select-dialog.component.scss']
+})
+
+export class SelectDialogComponent implements OnInit {
+  constructor(public activeModal: NgbActiveModal) { }
+
+  @Output()
+  public selectClicked = new EventEmitter()
+
+  @Input()
+  title = "Select"
+
+  @Input()
+  message = "Please select an object"
+
+  @Input()
+  objects: ObjectWithId[] = []
+
+  selected: number
+
+  ngOnInit(): void {
+  }
+
+  cancelClicked() {
+    this.activeModal.close()
+  }
+}

From 2c702eb568dff1d4d0a3e5cff0b7f07e96abc2a6 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 14:46:38 +0100
Subject: [PATCH 09/78] fixed some issues with the data access service, support
 for requesting all IDs of filtered documents (required for selection)

---
 .../app/services/document-list-view.service.ts   |  2 +-
 .../services/rest/abstract-paperless-service.ts  |  6 +++---
 src-ui/src/app/services/rest/document.service.ts | 16 +++++++++++-----
 3 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/src-ui/src/app/services/document-list-view.service.ts b/src-ui/src/app/services/document-list-view.service.ts
index 811ac3c4b..149096591 100644
--- a/src-ui/src/app/services/document-list-view.service.ts
+++ b/src-ui/src/app/services/document-list-view.service.ts
@@ -90,7 +90,7 @@ export class DocumentListViewService {
 
   reload(onFinish?) {
     this.isReloading = true
-    this.documentService.list(
+    this.documentService.listFiltered(
       this.currentPage,
       this.currentPageSize,
       this.view.sortField,
diff --git a/src-ui/src/app/services/rest/abstract-paperless-service.ts b/src-ui/src/app/services/rest/abstract-paperless-service.ts
index 3feed320e..d05b4aced 100644
--- a/src-ui/src/app/services/rest/abstract-paperless-service.ts
+++ b/src-ui/src/app/services/rest/abstract-paperless-service.ts
@@ -54,9 +54,9 @@ export abstract class AbstractPaperlessService<T extends ObjectWithId> {
 
   private _listAll: Observable<Results<T>>
 
-  listAll(ordering?: string, extraParams?): Observable<Results<T>> {
+  listAll(sortField?: string, sortDirection?: string, extraParams?): Observable<Results<T>> {
     if (!this._listAll) {
-      this._listAll = this.list(1, 100000, ordering, extraParams).pipe(
+      this._listAll = this.list(1, 100000, sortField, sortDirection, extraParams).pipe(
         publishReplay(1),
         refCount()
       )
@@ -94,4 +94,4 @@ export abstract class AbstractPaperlessService<T extends ObjectWithId> {
     this._listAll = null
     return this.http.put<T>(this.getResourceUrl(o.id), o)
   }
-}
\ No newline at end of file
+}
diff --git a/src-ui/src/app/services/rest/document.service.ts b/src-ui/src/app/services/rest/document.service.ts
index 4979ba8be..99ec1f3ee 100644
--- a/src-ui/src/app/services/rest/document.service.ts
+++ b/src-ui/src/app/services/rest/document.service.ts
@@ -64,8 +64,8 @@ export class DocumentService extends AbstractPaperlessService<PaperlessDocument>
     return doc
   }
 
-  list(page?: number, pageSize?: number, sortField?: string, sortDirection?: string, filterRules?: FilterRule[]): Observable<Results<PaperlessDocument>> {
-    return super.list(page, pageSize, sortField, sortDirection, this.filterRulesToQueryParams(filterRules)).pipe(
+  listFiltered(page?: number, pageSize?: number, sortField?: string, sortDirection?: string, filterRules?: FilterRule[], extraParams = {}): Observable<Results<PaperlessDocument>> {
+    return this.list(page, pageSize, sortField, sortDirection, Object.assign(extraParams, this.filterRulesToQueryParams(filterRules))).pipe(
       map(results => {
         results.results.forEach(doc => this.addObservablesToDocument(doc))
         return results
@@ -73,6 +73,12 @@ export class DocumentService extends AbstractPaperlessService<PaperlessDocument>
     )
   }
 
+  listAllFilteredIds(filterRules?: FilterRule[]): Observable<number[]> {
+    return this.listFiltered(1, 100000, null, null, filterRules, {"fields": "id"}).pipe(
+      map(response => response.results.map(doc => doc.id))
+    )
+  }
+
   getPreviewUrl(id: number, original: boolean = false): string {
     let url = this.getResourceUrl(id, 'preview')
     if (original) {
@@ -101,11 +107,11 @@ export class DocumentService extends AbstractPaperlessService<PaperlessDocument>
     return this.http.get<PaperlessDocumentMetadata>(this.getResourceUrl(id, 'metadata'))
   }
 
-  bulk_edit(ids: number[], method: string, args: any[]) {
+  bulkEdit(ids: number[], method: string, args: any) {
     return this.http.post(this.getResourceUrl(null, 'bulk_edit'), {
-      'ids': ids,
+      'documents': ids,
       'method': method,
-      'args': args
+      'parameters': args
     })
   }
 

From a8f27f79ddf517b9bcc24a280c7b0077e17f2a9f Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 14:47:33 +0100
Subject: [PATCH 10/78] delete dialog: delay enable delete button

---
 .../delete-dialog/delete-dialog.component.html    |  2 +-
 .../delete-dialog/delete-dialog.component.ts      | 15 +++++++++++++++
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.html b/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.html
index 2de507549..52287fc69 100644
--- a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.html
+++ b/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.html
@@ -10,5 +10,5 @@ <h4 class="modal-title" id="modal-basic-title">{{title}}</h4>
     </div>
     <div class="modal-footer">
       <button type="button" class="btn btn-outline-dark" (click)="cancelClicked()">Cancel</button>
-      <button type="button" class="btn btn-danger" (click)="deleteClicked.emit()">Delete</button>
+      <button type="button" class="btn btn-danger" (click)="deleteClicked.emit()" [disabled]="!deleteButtonEnabled">Delete<span *ngIf="!deleteButtonEnabled"> ({{seconds}})</span></button>
     </div>
\ No newline at end of file
diff --git a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts b/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts
index 20114c78c..38ec93bae 100644
--- a/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts
+++ b/src-ui/src/app/components/common/delete-dialog/delete-dialog.component.ts
@@ -22,6 +22,21 @@ export class DeleteDialogComponent implements OnInit {
   @Input()
   message2
 
+  deleteButtonEnabled = true
+  seconds = 0
+
+  delayConfirm(seconds: number) {
+    this.deleteButtonEnabled = false
+    this.seconds = seconds
+    setTimeout(() => {
+      if (this.seconds <= 1) {
+        this.deleteButtonEnabled = true
+      } else {
+        this.delayConfirm(seconds - 1)
+      }
+    }, 1000)
+  }
+
   ngOnInit(): void {
   }
 

From 56dfc71bb9f5c45b58eb338b9deeee8e6b413c4e Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 14:48:33 +0100
Subject: [PATCH 11/78] document list service: selection model

---
 .../services/document-list-view.service.ts    | 44 +++++++++++++++++++
 1 file changed, 44 insertions(+)

diff --git a/src-ui/src/app/services/document-list-view.service.ts b/src-ui/src/app/services/document-list-view.service.ts
index 149096591..b3fe351ac 100644
--- a/src-ui/src/app/services/document-list-view.service.ts
+++ b/src-ui/src/app/services/document-list-view.service.ts
@@ -118,6 +118,7 @@ export class DocumentListViewService {
     //want changes in the filter editor to propagate into here right away.
     this.view.filterRules = cloneFilterRules(filterRules)
     this.reload()
+    this.reduceSelectionToFilter()
     this.saveDocumentListView()
   }
 
@@ -192,6 +193,49 @@ export class DocumentListViewService {
     }
   }
 
+  selected = new Set<number>()
+
+  selectNone() {
+    this.selected.clear()
+  }
+
+  private reduceSelectionToFilter() {
+    if (this.selected.size > 0) {
+      this.documentService.listAllFilteredIds(this.filterRules).subscribe(ids => {
+        let subset = new Set<number>()
+        for (let id of ids) {
+          if (this.selected.has(id)) {
+            subset.add(id)
+          }
+        }
+        this.selected = subset
+      })
+    }
+  }
+
+  selectAll() {
+    this.documentService.listAllFilteredIds(this.filterRules).subscribe(ids => ids.forEach(id => this.selected.add(id)))
+  }
+
+  selectPage() {
+    this.selected.clear()
+    this.documents.forEach(doc => {
+      this.selected.add(doc.id)
+    })
+  }
+
+  isSelected(d: PaperlessDocument) {
+    return this.selected.has(d.id)
+  }
+
+  setSelected(d: PaperlessDocument, value: boolean) {
+    if (value) {
+      this.selected.add(d.id)
+    } else if (!value) {
+      this.selected.delete(d.id)
+    }
+  }
+
   constructor(private documentService: DocumentService) { 
     let documentListViewConfigJson = sessionStorage.getItem(DOCUMENT_LIST_SERVICE.CURRENT_VIEW_CONFIG)
     if (documentListViewConfigJson) {

From d1f285113d2035aebc21a4f91d1e18889b7b78ad Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 14:49:22 +0100
Subject: [PATCH 12/78] bulk edit menu and methods

---
 .../document-list.component.html              |  34 +++---
 .../document-list/document-list.component.ts  | 112 +++++++++++++++++-
 2 files changed, 129 insertions(+), 17 deletions(-)

diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html
index 58c32e9d1..24def7d64 100644
--- a/src-ui/src/app/components/document-list/document-list.component.html
+++ b/src-ui/src/app/components/document-list/document-list.component.html
@@ -7,22 +7,19 @@
       </svg>
       Bulk edit
     </button>
-    <div ngbDropdownMenu aria-labelledby="dropdownBasic1">
-      <button ngbDropdownItem>Select page</button>
-      <button ngbDropdownItem>Select all</button>
-      <button ngbDropdownItem>Select none</button>
+    <div ngbDropdownMenu aria-labelledby="dropdownBasic1" class="shadow">
+      <button ngbDropdownItem (click)="list.selectPage()">Select page</button>
+      <button ngbDropdownItem (click)="list.selectAll()">Select all</button>
+      <button ngbDropdownItem (click)="list.selectNone()">Select none</button>
       <div class="dropdown-divider"></div>
-      <button ngbDropdownItem>Re-create archived document</button>
+      <button ngbDropdownItem [disabled]="list.selected.size == 0" (click)="bulkSetCorrespondent()">Set correspondent</button>
+      <button ngbDropdownItem [disabled]="list.selected.size == 0" (click)="bulkRemoveCorrespondent()">Remove correspondent</button>
+      <button ngbDropdownItem [disabled]="list.selected.size == 0" (click)="bulkSetDocumentType()">Set document type</button>
+      <button ngbDropdownItem [disabled]="list.selected.size == 0" (click)="bulkRemoveDocumentType()">Remove document type</button>
+      <button ngbDropdownItem [disabled]="list.selected.size == 0" (click)="bulkAddTag()">Add tag</button>
+      <button ngbDropdownItem [disabled]="list.selected.size == 0" (click)="bulkRemoveTag()">Remove tag</button>
       <div class="dropdown-divider"></div>
-      <button ngbDropdownItem>Set correspondent</button>
-      <button ngbDropdownItem>Remove correspondent</button>
-      <button ngbDropdownItem>Set document type</button>
-      <button ngbDropdownItem>Remove document type</button>
-      <button ngbDropdownItem>Add tag</button>
-      <button ngbDropdownItem>Remove tag</button>
-      <div class="dropdown-divider"></div>
-      <button ngbDropdownItem>Delete</button>
-
+      <button ngbDropdownItem [disabled]="list.selected.size == 0" (click)="bulkDelete()">Delete</button>
     </div>
   </div>
 
@@ -101,7 +98,7 @@ <h5 class="card-title">Filter</h5>
 </div>
 
 <div class="d-flex justify-content-between align-items-center">
-  <p>{{list.collectionSize || 0}} document(s) <span *ngIf="isFiltered">(filtered)</span></p>
+  <p><span *ngIf="list.selected.size > 0">Selected {{list.selected.size}} of </span>{{list.collectionSize || 0}} document(s) <span *ngIf="isFiltered">(filtered)</span></p>
   <ngb-pagination [pageSize]="list.currentPageSize" [collectionSize]="list.collectionSize" [(page)]="list.currentPage" [maxSize]="5"
   [rotate]="true" (pageChange)="list.reload()" aria-label="Default pagination"></ngb-pagination>
 </div>
@@ -113,6 +110,7 @@ <h5 class="card-title">Filter</h5>
 
 <table class="table table-sm border shadow-sm" *ngIf="displayMode == 'details'">
   <thead>
+    <th></th>
     <th class="d-none d-lg-table-cell">ASN</th>
     <th class="d-none d-md-table-cell">Correspondent</th>
     <th>Title</th>
@@ -122,6 +120,12 @@ <h5 class="card-title">Filter</h5>
   </thead>
   <tbody>
     <tr *ngFor="let d of list.documents">
+      <td>
+        <div class="custom-control custom-checkbox">
+          <input type="checkbox" class="custom-control-input" id="docCheck{{d.id}}" [checked]="list.isSelected(d)" (change)="list.setSelected(d, $event.target.checked)">
+          <label class="custom-control-label" for="docCheck{{d.id}}"></label>
+        </div>
+      </td>
       <td class="d-none d-lg-table-cell">
         {{d.archive_serial_number}}
       </td>
diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts
index 09e73dd96..4d5597220 100644
--- a/src-ui/src/app/components/document-list/document-list.component.ts
+++ b/src-ui/src/app/components/document-list/document-list.component.ts
@@ -2,14 +2,21 @@ import { Component, OnInit } from '@angular/core';
 import { Title } from '@angular/platform-browser';
 import { ActivatedRoute } from '@angular/router';
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap';
+import { Observable } from 'rxjs';
+import { map } from 'rxjs/operators';
 import { cloneFilterRules, FilterRule } from 'src/app/data/filter-rule';
 import { FILTER_CORRESPONDENT, FILTER_DOCUMENT_TYPE, FILTER_HAS_TAG, FILTER_RULE_TYPES } from 'src/app/data/filter-rule-type';
 import { SavedViewConfig } from 'src/app/data/saved-view-config';
 import { DocumentListViewService } from 'src/app/services/document-list-view.service';
-import { DOCUMENT_SORT_FIELDS } from 'src/app/services/rest/document.service';
+import { CorrespondentService } from 'src/app/services/rest/correspondent.service';
+import { DocumentTypeService } from 'src/app/services/rest/document-type.service';
+import { DocumentService, DOCUMENT_SORT_FIELDS } from 'src/app/services/rest/document.service';
+import { TagService } from 'src/app/services/rest/tag.service';
 import { SavedViewConfigService } from 'src/app/services/saved-view-config.service';
 import { Toast, ToastService } from 'src/app/services/toast.service';
 import { environment } from 'src/environments/environment';
+import { DeleteDialogComponent } from '../common/delete-dialog/delete-dialog.component';
+import { SelectDialogComponent } from '../common/select-dialog/select-dialog.component';
 import { SaveViewConfigDialogComponent } from './save-view-config-dialog/save-view-config-dialog.component';
 
 @Component({
@@ -25,7 +32,11 @@ export class DocumentListComponent implements OnInit {
     public route: ActivatedRoute,
     private toastService: ToastService,
     public modalService: NgbModal,
-    private titleService: Title) { }
+    private titleService: Title,
+    private correspondentService: CorrespondentService,
+    private documentTypeService: DocumentTypeService,
+    private tagService: TagService,
+    private documentService: DocumentService) { }
 
   displayMode = 'smallCards' // largeCards, smallCards, details
 
@@ -142,4 +153,101 @@ export class DocumentListComponent implements OnInit {
     this.applyFilterRules()
   }
 
+  private executeBulkOperation(method: string, args): Observable<any> {
+    return this.documentService.bulkEdit(Array.from(this.list.selected), method, args).pipe(
+      map(r => {
+
+        this.list.reload()
+        this.list.selectNone()
+
+        return r
+      })
+    )
+  }
+
+  bulkSetCorrespondent() {
+    let modal = this.modalService.open(SelectDialogComponent, {backdrop: 'static'})
+    modal.componentInstance.title = "Select correspondent"
+    modal.componentInstance.message = `Select the correspondent you wish to assign to ${this.list.selected.size} selected document(s):`
+    this.correspondentService.listAll().subscribe(response => {
+      modal.componentInstance.objects = response.results
+    })
+    modal.componentInstance.selectClicked.subscribe(selectedId => {
+      this.executeBulkOperation('set_correspondent', {"correspondent": selectedId}).subscribe(
+        response => {
+          modal.close()
+        }
+      )
+    })
+  }
+
+  bulkRemoveCorrespondent() {
+    this.executeBulkOperation('set_correspondent', {"correspondent": null}).subscribe(r => {})
+  }
+
+  bulkSetDocumentType() {
+    let modal = this.modalService.open(SelectDialogComponent, {backdrop: 'static'})
+    modal.componentInstance.title = "Select document type"
+    modal.componentInstance.message = `Select the document type you wish to assign to ${this.list.selected.size} selected document(s):`
+    this.documentTypeService.listAll().subscribe(response => {
+      modal.componentInstance.objects = response.results
+    })
+    modal.componentInstance.selectClicked.subscribe(selectedId => {
+      this.executeBulkOperation('set_document_type', {"document_type": selectedId}).subscribe(
+        response => {
+          modal.close()
+        }
+      )
+    })
+  }
+
+  bulkRemoveDocumentType() {
+    this.executeBulkOperation('set_document_type', {"document_type": null}).subscribe(r => {})
+  }
+
+  bulkAddTag() {
+    let modal = this.modalService.open(SelectDialogComponent, {backdrop: 'static'})
+    modal.componentInstance.title = "Select tag"
+    modal.componentInstance.message = `Select the tag you wish to assign to ${this.list.selected.size} selected document(s):`
+    this.tagService.listAll().subscribe(response => {
+      modal.componentInstance.objects = response.results
+    })
+    modal.componentInstance.selectClicked.subscribe(selectedId => {
+      this.executeBulkOperation('add_tag', {"tag": selectedId}).subscribe(
+        response => {
+          modal.close()
+        }
+      )
+    })
+  }
+
+  bulkRemoveTag() {
+    let modal = this.modalService.open(SelectDialogComponent, {backdrop: 'static'})
+    modal.componentInstance.title = "Select tag"
+    modal.componentInstance.message = `Select the tag you wish to remove from ${this.list.selected.size} selected document(s):`
+    this.tagService.listAll().subscribe(response => {
+      modal.componentInstance.objects = response.results
+    })
+    modal.componentInstance.selectClicked.subscribe(selectedId => {
+      this.executeBulkOperation('remove_tag', {"tag": selectedId}).subscribe(
+        response => {
+          modal.close()
+        }
+      )
+    })
+  }
+
+  bulkDelete() {
+    let modal = this.modalService.open(DeleteDialogComponent, {backdrop: 'static'})
+    modal.componentInstance.delayConfirm(5)
+    modal.componentInstance.message = `This operation will permanently delete all ${this.list.selected.size} selected document(s).`
+    modal.componentInstance.message2 = `This operation cannot be undone.`
+    modal.componentInstance.deleteClicked.subscribe(() => {
+      this.executeBulkOperation("delete", {}).subscribe(
+        response => {
+          modal.close()
+        }
+      )
+    })    
+  }
 }

From 66240188c750d215870e9eda6ee0a4fda1cd064d Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 14:51:20 +0100
Subject: [PATCH 13/78] import fix

---
 src/documents/views.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/documents/views.py b/src/documents/views.py
index 4ce78348e..5e173d703 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -31,7 +31,6 @@
 import documents.index as index
 from paperless.db import GnuPG
 from paperless.views import StandardPagination
-from .bulk_edit import perform_bulk_edit
 from .filters import (
     CorrespondentFilterSet,
     DocumentFilterSet,

From d1d09ac6acf8f8d539483c6af464188cdde322b8 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 17:35:21 +0100
Subject: [PATCH 14/78] checboxes for small cards. does not work yet.

---
 .../document-card-small.component.html                | 11 ++++++++++-
 .../document-card-small.component.scss                |  8 ++++++++
 .../document-card-small.component.ts                  |  2 ++
 3 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html
index da469ebc4..8993674ba 100644
--- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html
+++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html
@@ -1,7 +1,16 @@
-<div class="col p-2 h-100" style="width: 16rem;">
+<div class="col p-2 h-100 document-card" style="width: 16rem;">
   <div class="card h-100 shadow-sm">
     <div class="border-bottom">
       <img class="card-img doc-img" [src]="getThumbUrl()">
+
+      <div style="top: 0; left: 0" class="position-absolute border-right border-bottom bg-light p-1" [class.document-card-check]="!selected">
+        <div class="custom-control custom-checkbox">
+          <input type="checkbox" class="custom-control-input" id="smallCardCheck{{document.id}}" [(ngModel)]="selected">
+          <label class="custom-control-label" for="smallCardCheck{{document.id}}">L</label>
+        </div>
+      </div>
+
+
       <div style="top: 0; right: 0; font-size: large" class="text-right position-absolute mr-1">
         <div *ngFor="let t of getTagsLimited$() | async">
           <app-tag [tag]="t" (click)="clickTag.emit(t.id)" [clickable]="true" linkTitle="Filter by tag"></app-tag>
diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss
index 0068667d0..ba7190615 100644
--- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss
+++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss
@@ -2,4 +2,12 @@
   object-fit: cover;
   object-position: top;
   height: 200px;
+}
+
+.document-card-check {
+  display: none
+}
+
+.document-card:hover .document-card-check {
+  display: block;
 }
\ No newline at end of file
diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts
index d60552d4f..037c02cf0 100644
--- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts
+++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts
@@ -13,6 +13,8 @@ export class DocumentCardSmallComponent implements OnInit {
 
   constructor(private documentService: DocumentService) { }
 
+  selected = false
+
   @Input()
   document: PaperlessDocument
 

From 80b47fa287aaaf637feb31073849996cd54fca0a Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 23:33:59 +0100
Subject: [PATCH 15/78] codestyle

---
 src/documents/bulk_edit.py | 17 ++++++++++-------
 src/documents/tasks.py     |  4 ++--
 2 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/src/documents/bulk_edit.py b/src/documents/bulk_edit.py
index 1349f9d54..aa5b8ea3f 100644
--- a/src/documents/bulk_edit.py
+++ b/src/documents/bulk_edit.py
@@ -8,11 +8,12 @@ def set_correspondent(doc_ids, correspondent):
     if correspondent:
         correspondent = Correspondent.objects.get(id=correspondent)
 
-    qs = Document.objects.filter(Q(id__in=doc_ids) & ~Q(correspondent=correspondent))
+    qs = Document.objects.filter(
+        Q(id__in=doc_ids) & ~Q(correspondent=correspondent))
     affected_docs = [doc.id for doc in qs]
     qs.update(correspondent=correspondent)
 
-    async_task("documents.tasks.bulk_rename_files", affected_docs)
+    async_task("documents.tasks.bulk_rename_files", document_ids=affected_docs)
 
     return "OK"
 
@@ -21,11 +22,12 @@ def set_document_type(doc_ids, document_type):
     if document_type:
         document_type = DocumentType.objects.get(id=document_type)
 
-    qs = Document.objects.filter(Q(id__in=doc_ids) & ~Q(document_type=document_type))
+    qs = Document.objects.filter(
+        Q(id__in=doc_ids) & ~Q(document_type=document_type))
     affected_docs = [doc.id for doc in qs]
     qs.update(document_type=document_type)
 
-    async_task("documents.tasks.bulk_rename_files", affected_docs)
+    async_task("documents.tasks.bulk_rename_files", document_ids=affected_docs)
 
     return "OK"
 
@@ -38,10 +40,11 @@ def add_tag(doc_ids, tag):
     DocumentTagRelationship = Document.tags.through
 
     DocumentTagRelationship.objects.bulk_create([
-        DocumentTagRelationship(document_id=doc, tag_id=tag) for doc in affected_docs
+        DocumentTagRelationship(
+            document_id=doc, tag_id=tag) for doc in affected_docs
     ])
 
-    async_task("documents.tasks.bulk_rename_files", affected_docs)
+    async_task("documents.tasks.bulk_rename_files", document_ids=affected_docs)
 
     return "OK"
 
@@ -58,7 +61,7 @@ def remove_tag(doc_ids, tag):
         Q(tag_id=tag)
     ).delete()
 
-    async_task("documents.tasks.bulk_rename_files", affected_docs)
+    async_task("documents.tasks.bulk_rename_files", document_ids=affected_docs)
 
     return "OK"
 
diff --git a/src/documents/tasks.py b/src/documents/tasks.py
index af4c91448..fafe6e10f 100644
--- a/src/documents/tasks.py
+++ b/src/documents/tasks.py
@@ -90,7 +90,7 @@ def sanity_check():
         return "No issues detected."
 
 
-def bulk_rename_files(ids):
-    qs = Document.objects.filter(id__in=ids)
+def bulk_rename_files(document_ids):
+    qs = Document.objects.filter(id__in=document_ids)
     for doc in qs:
         post_save.send(Document, instance=doc, created=False)

From f5df9108945cdcfe0c095bcb64903b9269adad2f Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 23:34:24 +0100
Subject: [PATCH 16/78] document list validation.

---
 src/documents/serialisers.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/src/documents/serialisers.py b/src/documents/serialisers.py
index 5418ec0fb..92fc35719 100644
--- a/src/documents/serialisers.py
+++ b/src/documents/serialisers.py
@@ -185,6 +185,13 @@ class BulkEditSerializer(serializers.Serializer):
 
     parameters = serializers.DictField(allow_empty=True)
 
+    def validate_documents(self, documents):
+        count = Document.objects.filter(id__in=documents).count()
+        if not count == len(documents):
+            raise serializers.ValidationError(
+                "Some documents don't exist or were specified twice.")
+        return documents
+
     def validate_method(self, method):
         if method == "set_correspondent":
             return bulk_edit.set_correspondent

From a85792e327ffb5604f40c69d72cfce47cfa2b623 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 11 Dec 2020 23:34:34 +0100
Subject: [PATCH 17/78] tests.

---
 src/documents/tests/test_api.py | 116 +++++++++++++++++++++++++++++++-
 1 file changed, 115 insertions(+), 1 deletion(-)

diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py
index ab1716366..bd0d9a421 100644
--- a/src/documents/tests/test_api.py
+++ b/src/documents/tests/test_api.py
@@ -1,14 +1,16 @@
+import json
 import os
 import shutil
 import tempfile
 from unittest import mock
 
 from django.contrib.auth.models import User
+from django.test import client
 from pathvalidate import ValidationError
 from rest_framework.test import APITestCase
 from whoosh.writing import AsyncWriter
 
-from documents import index
+from documents import index, bulk_edit
 from documents.models import Document, Correspondent, DocumentType, Tag
 from documents.tests.utils import DirectoriesMixin
 
@@ -515,3 +517,115 @@ def test_get_metadata_no_archive(self):
         self.assertFalse(meta['has_archive_version'])
         self.assertGreater(len(meta['original_metadata']), 0)
         self.assertIsNone(meta['archive_metadata'])
+
+
+class TestBulkEdit(DirectoriesMixin, APITestCase):
+
+    def setUp(self):
+        super(TestBulkEdit, self).setUp()
+
+        user = User.objects.create_superuser(username="temp_admin")
+        self.client.force_login(user=user)
+
+        patcher = mock.patch('documents.bulk_edit.async_task')
+        self.async_task = patcher.start()
+        self.addCleanup(patcher.stop)
+        self.c1 = Correspondent.objects.create(name="c1")
+        self.c2 = Correspondent.objects.create(name="c2")
+        self.dt1 = DocumentType.objects.create(name="dt1")
+        self.dt2 = DocumentType.objects.create(name="dt2")
+        self.t1 = Tag.objects.create(name="t1")
+        self.t2 = Tag.objects.create(name="t2")
+        self.doc1 = Document.objects.create(checksum="A", title="A")
+        self.doc2 = Document.objects.create(checksum="B", title="B", correspondent=self.c1, document_type=self.dt1)
+        self.doc3 = Document.objects.create(checksum="C", title="C", correspondent=self.c2, document_type=self.dt2)
+        self.doc4 = Document.objects.create(checksum="D", title="D")
+        self.doc5 = Document.objects.create(checksum="E", title="E")
+        self.doc2.tags.add(self.t1)
+        self.doc3.tags.add(self.t2)
+        self.doc4.tags.add(self.t1, self.t2)
+
+    def test_set_correspondent(self):
+        self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 1)
+        bulk_edit.set_correspondent([self.doc1.id, self.doc2.id, self.doc3.id], self.c2.id)
+        self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 3)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc1.id, self.doc2.id])
+
+    def test_unset_correspondent(self):
+        self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 1)
+        bulk_edit.set_correspondent([self.doc1.id, self.doc2.id, self.doc3.id], None)
+        self.assertEqual(Document.objects.filter(correspondent=self.c2).count(), 0)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc2.id, self.doc3.id])
+
+    def test_set_document_type(self):
+        self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 1)
+        bulk_edit.set_document_type([self.doc1.id, self.doc2.id, self.doc3.id], self.dt2.id)
+        self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 3)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc1.id, self.doc2.id])
+
+    def test_unset_document_type(self):
+        self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 1)
+        bulk_edit.set_document_type([self.doc1.id, self.doc2.id, self.doc3.id], None)
+        self.assertEqual(Document.objects.filter(document_type=self.dt2).count(), 0)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc2.id, self.doc3.id])
+
+    def test_add_tag(self):
+        self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 2)
+        bulk_edit.add_tag([self.doc1.id, self.doc2.id, self.doc3.id, self.doc4.id], self.t1.id)
+        self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 4)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc1.id, self.doc3.id])
+
+
+    def test_remove_tag(self):
+        self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 2)
+        bulk_edit.remove_tag([self.doc1.id, self.doc3.id, self.doc4.id], self.t1.id)
+        self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 1)
+        self.async_task.assert_called_once()
+        args, kwargs = self.async_task.call_args
+        self.assertCountEqual(kwargs['document_ids'], [self.doc4.id])
+
+    def test_delete(self):
+        self.assertEqual(Document.objects.count(), 5)
+        bulk_edit.delete([self.doc1.id, self.doc2.id])
+        self.assertEqual(Document.objects.count(), 3)
+        self.assertCountEqual([doc.id for doc in Document.objects.all()], [self.doc3.id, self.doc4.id, self.doc5.id])
+
+    def test_api(self):
+        self.assertEqual(Document.objects.count(), 5)
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "delete",
+            "parameters": {}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(Document.objects.count(), 4)
+
+    def test_api_invalid_doc(self):
+        self.assertEqual(Document.objects.count(), 5)
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [-235],
+            "method": "delete",
+            "parameters": {}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(Document.objects.count(), 5)
+
+    def test_api_invalid_method(self):
+        self.assertEqual(Document.objects.count(), 5)
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc2.id],
+            "method": "exterminate",
+            "parameters": {}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(Document.objects.count(), 5)

From 7906d8fef15ec985d066e5022120c55448592d36 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 13 Dec 2020 14:10:55 +0100
Subject: [PATCH 18/78] selection for small cards

---
 .../document-card-small.component.html            | 10 +++++-----
 .../document-card-small.component.scss            | 11 +++++++++++
 .../document-card-small.component.ts              | 15 ++++++++++++++-
 .../document-list/document-list.component.html    |  2 +-
 src-ui/src/theme.scss                             |  1 +
 5 files changed, 32 insertions(+), 7 deletions(-)

diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html
index 6909a24fb..b78fedfe3 100644
--- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html
+++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html
@@ -1,12 +1,12 @@
 <div class="col p-2 h-100 document-card" style="width: 16rem;">
-  <div class="card h-100 shadow-sm">
-    <div class="border-bottom">
-      <img class="card-img doc-img" [src]="getThumbUrl()">
+  <div class="card h-100 shadow-sm" [class.card-selected]="selected">
+    <div class="border-bottom" [class.doc-img-background-selected]="selected">
+      <img class="card-img doc-img" [src]="getThumbUrl()" (click)="selected = !selected">
 
       <div style="top: 0; left: 0" class="position-absolute border-right border-bottom bg-light p-1" [class.document-card-check]="!selected">
         <div class="custom-control custom-checkbox">
-          <input type="checkbox" class="custom-control-input" id="smallCardCheck{{document.id}}" [(ngModel)]="selected">
-          <label class="custom-control-label" for="smallCardCheck{{document.id}}">L</label>
+          <input type="checkbox" class="custom-control-input" id="smallCardCheck{{document.id}}" [checked]="selected" (change)="selected = $event.target.checked">
+          <label class="custom-control-label" for="smallCardCheck{{document.id}}"></label>
         </div>
       </div>
 
diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss
index ba7190615..36db2203c 100644
--- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss
+++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.scss
@@ -1,7 +1,10 @@
+@import "/src/theme";
+
 .doc-img {
   object-fit: cover;
   object-position: top;
   height: 200px;
+  mix-blend-mode: multiply;
 }
 
 .document-card-check {
@@ -10,4 +13,12 @@
 
 .document-card:hover .document-card-check {
   display: block;
+}
+
+.card-selected {
+  border-color: $primary;
+}
+
+.doc-img-background-selected {
+  background-color: $primaryFaded;
 }
\ No newline at end of file
diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts
index 037c02cf0..5d664697b 100644
--- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts
+++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.ts
@@ -13,7 +13,20 @@ export class DocumentCardSmallComponent implements OnInit {
 
   constructor(private documentService: DocumentService) { }
 
-  selected = false
+  _selected = false
+
+  get selected() {
+    return this._selected
+  }
+
+  @Input()
+  set selected(value: boolean) {
+    this._selected = value
+    this.selectedChange.emit(value)
+  }
+
+  @Output()
+  selectedChange = new EventEmitter<boolean>()
 
   @Input()
   document: PaperlessDocument
diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html
index a87a89bbf..0c3674421 100644
--- a/src-ui/src/app/components/document-list/document-list.component.html
+++ b/src-ui/src/app/components/document-list/document-list.component.html
@@ -155,5 +155,5 @@ <h5 class="card-title">Filter</h5>
 
 
 <div class=" m-n2 row" *ngIf="displayMode == 'smallCards'">
-  <app-document-card-small [document]="d" *ngFor="let d of list.documents" (clickTag)="filterByTag($event)" (clickCorrespondent)="filterByCorrespondent($event)"></app-document-card-small>    
+  <app-document-card-small [selected]="list.isSelected(d)" (selectedChange)="list.setSelected(d, $event)"  [document]="d" *ngFor="let d of list.documents" (clickTag)="filterByTag($event)" (clickCorrespondent)="filterByCorrespondent($event)"></app-document-card-small>    
 </div>
diff --git a/src-ui/src/theme.scss b/src-ui/src/theme.scss
index 88f3ae30f..df2aea003 100644
--- a/src-ui/src/theme.scss
+++ b/src-ui/src/theme.scss
@@ -1,5 +1,6 @@
 $paperless-green: #17541f;
 $primary: #17541f;
+$primaryFaded: #d1ddd2;
 
 $theme-colors: (
   "primary": $primary

From b5a85caa72422763c29dbf6baf10af8b19e0b564 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 13 Dec 2020 15:20:24 +0100
Subject: [PATCH 19/78] confirm dialogs for remove operations

---
 .../document-list/document-list.component.ts   | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts
index 36c70a00e..ce4ebec73 100644
--- a/src-ui/src/app/components/document-list/document-list.component.ts
+++ b/src-ui/src/app/components/document-list/document-list.component.ts
@@ -182,7 +182,14 @@ export class DocumentListComponent implements OnInit {
   }
 
   bulkRemoveCorrespondent() {
-    this.executeBulkOperation('set_correspondent', {"correspondent": null}).subscribe(r => {})
+    let modal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'})
+    modal.componentInstance.title = "Remove correspondent"
+    modal.componentInstance.message = `This operation will remove the correspondent from all ${this.list.selected.size} selected document(s).`
+    modal.componentInstance.confirmClicked.subscribe(() => {
+      this.executeBulkOperation('set_correspondent', {"correspondent": null}).subscribe(r => {
+        modal.close()
+      })
+    })
   }
 
   bulkSetDocumentType() {
@@ -202,7 +209,14 @@ export class DocumentListComponent implements OnInit {
   }
 
   bulkRemoveDocumentType() {
-    this.executeBulkOperation('set_document_type', {"document_type": null}).subscribe(r => {})
+    let modal = this.modalService.open(ConfirmDialogComponent, {backdrop: 'static'})
+    modal.componentInstance.title = "Remove document type"
+    modal.componentInstance.message = `This operation will remove the document type from all ${this.list.selected.size} selected document(s).`
+    modal.componentInstance.confirmClicked.subscribe(() => {
+      this.executeBulkOperation('set_document_type', {"document_type": null}).subscribe(r => {
+        modal.close()
+      })
+    })
   }
 
   bulkAddTag() {

From 2dc3019083a5ef7de57df74b2dc3cad8df49eb99 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 13 Dec 2020 15:28:20 +0100
Subject: [PATCH 20/78] table selection highlighting

---
 .../components/document-list/document-list.component.html    | 2 +-
 .../components/document-list/document-list.component.scss    | 5 +++++
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html
index 0c3674421..396e7e12d 100644
--- a/src-ui/src/app/components/document-list/document-list.component.html
+++ b/src-ui/src/app/components/document-list/document-list.component.html
@@ -119,7 +119,7 @@ <h5 class="card-title">Filter</h5>
     <th class="d-none d-xl-table-cell">Added</th>
   </thead>
   <tbody>
-    <tr *ngFor="let d of list.documents">
+    <tr *ngFor="let d of list.documents" [ngClass]="list.isSelected(d) ? 'table-row-selected' : ''">
       <td>
         <div class="custom-control custom-checkbox">
           <input type="checkbox" class="custom-control-input" id="docCheck{{d.id}}" [checked]="list.isSelected(d)" (change)="list.setSelected(d, $event.target.checked)">
diff --git a/src-ui/src/app/components/document-list/document-list.component.scss b/src-ui/src/app/components/document-list/document-list.component.scss
index e69de29bb..b9553930b 100644
--- a/src-ui/src/app/components/document-list/document-list.component.scss
+++ b/src-ui/src/app/components/document-list/document-list.component.scss
@@ -0,0 +1,5 @@
+@import "/src/theme";
+
+.table-row-selected {
+  background-color: $primaryFaded;
+}
\ No newline at end of file

From 677cfb7a1e20032b0755a722759e8c88e70c43f2 Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Tue, 15 Dec 2020 14:31:18 -0800
Subject: [PATCH 21/78] Use bootstrap row-cols-* classes to keep card list view
 full width

---
 .../document-card-small/document-card-small.component.html  | 6 +++---
 .../components/document-list/document-list.component.html   | 2 +-
 2 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html
index 2647e702c..da0829349 100644
--- a/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html
+++ b/src-ui/src/app/components/document-list/document-card-small/document-card-small.component.html
@@ -1,4 +1,4 @@
-<div class="col p-2 h-100" style="width: 16rem;">
+<div class="col p-2 h-100">
   <div class="card h-100 shadow-sm">
     <div class="border-bottom">
       <img class="card-img doc-img" [src]="getThumbUrl()">
@@ -22,7 +22,7 @@
     </div>
     <div class="card-footer">
 
-      <div class="d-flex justify-content-between align-items-center ml-n2">
+      <div class="d-flex justify-content-between align-items-center mx-n2">
         <div class="btn-group">
           <a routerLink="/documents/{{document.id}}" class="btn btn-sm btn-outline-secondary" title="Edit">
             <svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-pencil" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
@@ -42,7 +42,7 @@
             </svg>
           </a>
         </div>
-        <small class="text-muted">{{document.created | date}}</small>
+        <small class="text-muted pl-1">{{document.created | date}}</small>
       </div>
 
     </div>
diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html
index 1b3596098..31b00f482 100644
--- a/src-ui/src/app/components/document-list/document-list.component.html
+++ b/src-ui/src/app/components/document-list/document-list.component.html
@@ -116,6 +116,6 @@
 </table>
 
 
-<div class=" m-n2 row" *ngIf="displayMode == 'smallCards'">
+<div class="m-n2 row row row-cols-2 row-cols-md-3 row-cols-lg-4 row-cols-xl-5" *ngIf="displayMode == 'smallCards'">
   <app-document-card-small [document]="d" *ngFor="let d of list.documents" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)"></app-document-card-small>
 </div>

From fbca412d309725f7dd1b1fc03c94b08b88da8218 Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Wed, 16 Dec 2020 16:18:41 -0800
Subject: [PATCH 22/78] Add more card columns on very large screens

---
 .../document-list.component.html              |  2 +-
 .../document-list.component.scss              | 21 +++++++++++++++++++
 2 files changed, 22 insertions(+), 1 deletion(-)

diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html
index 31b00f482..0b98a5633 100644
--- a/src-ui/src/app/components/document-list/document-list.component.html
+++ b/src-ui/src/app/components/document-list/document-list.component.html
@@ -116,6 +116,6 @@
 </table>
 
 
-<div class="m-n2 row row row-cols-2 row-cols-md-3 row-cols-lg-4 row-cols-xl-5" *ngIf="displayMode == 'smallCards'">
+<div class="m-n2 row row-cols-paperless-cards" *ngIf="displayMode == 'smallCards'">
   <app-document-card-small [document]="d" *ngFor="let d of list.documents" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)"></app-document-card-small>
 </div>
diff --git a/src-ui/src/app/components/document-list/document-list.component.scss b/src-ui/src/app/components/document-list/document-list.component.scss
index e69de29bb..08b88e0d0 100644
--- a/src-ui/src/app/components/document-list/document-list.component.scss
+++ b/src-ui/src/app/components/document-list/document-list.component.scss
@@ -0,0 +1,21 @@
+$paperless-card-breakpoints: (
+  0: 2, // xs
+  768px: 3, //md
+  992px: 4, //lg
+  1200px: 5, //xl
+  1400px: 6, // xxl
+  1600px: 7,
+  1800px: 8,
+  2000px: 9
+);
+
+.row-cols-paperless-cards {
+  @each $width, $n_cols in $paperless-card-breakpoints {
+    @media(min-width: $width) {
+      > * {
+        flex: 0 0 auto;
+        width: 100% / $n_cols;
+      }
+    }
+  }
+}

From 164418880a93301b61e577363d4211b8142dcf0e Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Thu, 17 Dec 2020 21:36:21 +0100
Subject: [PATCH 23/78] more like this searching

---
 .../document-detail.component.html            |  6 +++
 .../document-detail.component.ts              |  4 ++
 .../document-card-large.component.html        | 11 +++++-
 .../document-card-large.component.ts          | 13 +++++++
 .../result-highlight.component.scss           |  2 +-
 .../components/search/search.component.html   | 10 ++++-
 .../app/components/search/search.component.ts | 26 +++++++++++--
 .../src/app/services/rest/search.service.ts   | 10 ++++-
 src/documents/index.py                        | 37 ++++++++++++++-----
 src/documents/views.py                        | 23 +++++++-----
 10 files changed, 114 insertions(+), 28 deletions(-)

diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html
index f4a64c2cc..f7e1ff855 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.html
+++ b/src-ui/src/app/components/document-detail/document-detail.component.html
@@ -24,6 +24,12 @@
 
     </div>
 
+    <button type="button" class="btn btn-sm btn-outline-primary mr-2" (click)="moreLike()">
+        <svg class="buttonicon" fill="currentColor">
+            <use xlink:href="assets/bootstrap-icons.svg#three-dots" />
+        </svg>
+        <span class="d-none d-lg-inline"> More like this</span>
+    </button>
 
     <button type="button" class="btn btn-sm btn-outline-primary" (click)="close()">
         <svg class="buttonicon" fill="currentColor">
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts
index b4005b920..90594bd0a 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.ts
+++ b/src-ui/src/app/components/document-detail/document-detail.component.ts
@@ -168,6 +168,10 @@ export class DocumentDetailComponent implements OnInit {
 
   }
 
+  moreLike() {
+    this.router.navigate(["search"], {queryParams: {more_like:this.document.id}})
+  }
+
   hasNext() {
     return this.documentListViewService.hasNext(this.documentId)
   }
diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
index c2645db5e..32abaaef1 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
@@ -25,6 +25,12 @@ <h5 class="card-title" *ngIf="document.archive_serial_number">#{{document.archiv
 
         <div class="d-flex justify-content-between align-items-center">
           <div class="btn-group">
+            <a routerLink="/search" [queryParams]="{'more_like': document.id}" class="btn btn-sm btn-outline-secondary">
+              <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-three-dots" viewBox="0 0 16 16">
+                <path fill-rule="evenodd" d="M3 9.5a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3z"/>
+              </svg>
+              More like this
+            </a>
             <a routerLink="/documents/{{document.id}}" class="btn btn-sm btn-outline-secondary">
               <svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-pencil" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
                 <path fill-rule="evenodd" d="M12.146.146a.5.5 0 0 1 .708 0l3 3a.5.5 0 0 1 0 .708l-10 10a.5.5 0 0 1-.168.11l-5 2a.5.5 0 0 1-.65-.65l2-5a.5.5 0 0 1 .11-.168l10-10zM11.207 2.5L13.5 4.793 14.793 3.5 12.5 1.207 11.207 2.5zm1.586 3L10.5 3.207 4 9.707V10h.5a.5.5 0 0 1 .5.5v.5h.5a.5.5 0 0 1 .5.5v.5h.293l6.5-6.5zm-9.761 5.175l-.106.106-1.528 3.821 3.821-1.528.106-.106A.5.5 0 0 1 5 12.5V12h-.5a.5.5 0 0 1-.5-.5V11h-.5a.5.5 0 0 1-.468-.325z"/>
@@ -45,10 +51,13 @@ <h5 class="card-title" *ngIf="document.archive_serial_number">#{{document.archiv
               </svg>
               Download
             </a>
+            <ngb-progressbar [type]="searchScoreClass" [value]="searchScore" style="width: 100px; height: 5px; margin: 10px;" [max]="1"></ngb-progressbar>
+            
           </div>
+          
           <small class="text-muted">Created: {{document.created | date}}</small>
         </div>
-
+        
       </div>
     </div>
   </div>
diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
index 2e056cc70..44f9cb906 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
@@ -24,6 +24,19 @@ export class DocumentCardLargeComponent implements OnInit {
   @Output()
   clickCorrespondent = new EventEmitter<number>()
 
+  @Input()
+  searchScore: number
+
+  get searchScoreClass() {
+    if (this.searchScore > 0.7) {
+      return "success"
+    } else if (this.searchScore > 0.3) {
+      return "warning"
+    } else {
+      return "danger"
+    }
+  }
+
   ngOnInit(): void {
   }
 
diff --git a/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss b/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss
index 645fb0426..e04dd13b2 100644
--- a/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss
+++ b/src-ui/src/app/components/search/result-highlight/result-highlight.component.scss
@@ -1,4 +1,4 @@
 .match {
     color: black;
-    background-color: orange;
+    background-color: rgb(255, 211, 66);
 }
\ No newline at end of file
diff --git a/src-ui/src/app/components/search/search.component.html b/src-ui/src/app/components/search/search.component.html
index 55fcee900..609bea9e5 100644
--- a/src-ui/src/app/components/search/search.component.html
+++ b/src-ui/src/app/components/search/search.component.html
@@ -3,7 +3,12 @@
 
 <div *ngIf="errorMessage" class="alert alert-danger">Invalid search query: {{errorMessage}}</div>
 
-<p>
+<p *ngIf="more_like">
+    Showing documents similar to
+    <a routerLink="/documents/{{more_like}}">{{more_like_doc?.original_file_name}}</a>
+</p>
+
+<p *ngIf="query">
     Search string: <i>{{query}}</i>
     <ng-container *ngIf="correctedQuery">
         - Did you mean "<a [routerLink]="" (click)="searchCorrectedQuery()">{{correctedQuery}}</a>"?
@@ -15,7 +20,8 @@
     <p>{{resultCount}} result(s)</p>
     <app-document-card-large *ngFor="let result of results"
         [document]="result.document"
-        [details]="result.highlights">
+        [details]="result.highlights"
+        [searchScore]="result.score / maxScore">
 
 </app-document-card-large>
 </div>
diff --git a/src-ui/src/app/components/search/search.component.ts b/src-ui/src/app/components/search/search.component.ts
index de8b4652f..b2b10d632 100644
--- a/src-ui/src/app/components/search/search.component.ts
+++ b/src-ui/src/app/components/search/search.component.ts
@@ -1,6 +1,9 @@
 import { Component, OnInit } from '@angular/core';
 import { ActivatedRoute, Router } from '@angular/router';
+import { PaperlessDocument } from 'src/app/data/paperless-document';
+import { PaperlessDocumentType } from 'src/app/data/paperless-document-type';
 import { SearchHit } from 'src/app/data/search-result';
+import { DocumentService } from 'src/app/services/rest/document.service';
 import { SearchService } from 'src/app/services/rest/search.service';
 
 @Component({
@@ -14,6 +17,10 @@ export class SearchComponent implements OnInit {
 
   query: string = ""
 
+  more_like: number
+
+  more_like_doc: PaperlessDocument
+
   searching = false
 
   currentPage = 1
@@ -26,11 +33,23 @@ export class SearchComponent implements OnInit {
 
   errorMessage: string
 
-  constructor(private searchService: SearchService, private route: ActivatedRoute, private router: Router) { }
+  get maxScore() {
+    return this.results?.length > 0 ? this.results[0].score : 100
+  }
+
+  constructor(private searchService: SearchService, private route: ActivatedRoute, private router: Router, private documentService: DocumentService) { }
 
   ngOnInit(): void {
     this.route.queryParamMap.subscribe(paramMap => {
       this.query = paramMap.get('query')
+      this.more_like = paramMap.has('more_like') ? +paramMap.get('more_like') : null
+      if (this.more_like) {
+        this.documentService.get(this.more_like).subscribe(r => {
+          this.more_like_doc = r
+        })
+      } else {
+        this.more_like_doc = null
+      }
       this.searching = true
       this.currentPage = 1
       this.loadPage()
@@ -39,13 +58,14 @@ export class SearchComponent implements OnInit {
   }
 
   searchCorrectedQuery() {
-    this.router.navigate(["search"], {queryParams: {query: this.correctedQuery}})
+    this.router.navigate(["search"], {queryParams: {query: this.correctedQuery, more_like: this.more_like}})
   }
 
   loadPage(append: boolean = false) {
     this.errorMessage = null
     this.correctedQuery = null
-    this.searchService.search(this.query, this.currentPage).subscribe(result => {
+
+    this.searchService.search(this.query, this.currentPage, this.more_like).subscribe(result => {
       if (append) {
         this.results.push(...result.results)
       } else {
diff --git a/src-ui/src/app/services/rest/search.service.ts b/src-ui/src/app/services/rest/search.service.ts
index b19a55769..3799f3dc7 100644
--- a/src-ui/src/app/services/rest/search.service.ts
+++ b/src-ui/src/app/services/rest/search.service.ts
@@ -15,11 +15,17 @@ export class SearchService {
   
   constructor(private http: HttpClient, private documentService: DocumentService) { }
 
-  search(query: string, page?: number): Observable<SearchResult> {
-    let httpParams = new HttpParams().set('query', query)
+  search(query: string, page?: number, more_like?: number): Observable<SearchResult> {
+    let httpParams = new HttpParams()
+    if (query) {
+      httpParams = httpParams.set('query', query)
+    }
     if (page) {
       httpParams = httpParams.set('page', page.toString())
     }
+    if (more_like) {
+      httpParams = httpParams.set('more_like', more_like.toString())
+    }
     return this.http.get<SearchResult>(`${environment.apiBaseUrl}search/`, {params: httpParams}).pipe(
       map(result => {
         result.results.forEach(hit => this.documentService.addObservablesToDocument(hit.document))
diff --git a/src/documents/index.py b/src/documents/index.py
index 53bf34542..7d022182f 100644
--- a/src/documents/index.py
+++ b/src/documents/index.py
@@ -3,7 +3,7 @@
 from contextlib import contextmanager
 
 from django.conf import settings
-from whoosh import highlight
+from whoosh import highlight, classify, query
 from whoosh.fields import Schema, TEXT, NUMERIC, KEYWORD, DATETIME
 from whoosh.highlight import Formatter, get_text
 from whoosh.index import create_in, exists_in, open_dir
@@ -120,22 +120,39 @@ def remove_document_from_index(document):
 
 
 @contextmanager
-def query_page(ix, querystring, page):
+def query_page(ix, page, querystring, more_like_doc_id, more_like_doc_content):
     searcher = ix.searcher()
     try:
-        qp = MultifieldParser(
-            ["content", "title", "correspondent", "tag", "type"],
-            ix.schema)
-        qp.add_plugin(DateParserPlugin())
+        if querystring:
+            qp = MultifieldParser(
+                ["content", "title", "correspondent", "tag", "type"],
+                ix.schema)
+            qp.add_plugin(DateParserPlugin())
+            str_q = qp.parse(querystring)
+            corrected = searcher.correct_query(str_q, querystring)
+        else:
+            str_q = None
+            corrected = None
+
+        if more_like_doc_id:
+            docnum = searcher.document_number(id=more_like_doc_id)
+            kts = searcher.key_terms_from_text('content', more_like_doc_content, numterms=20,
+                                           model=classify.Bo1Model, normalize=False)
+            more_like_q = query.Or([query.Term('content', word, boost=weight)
+                          for word, weight in kts])
+            result_page = searcher.search_page(more_like_q, page, filter=str_q, mask={docnum})
+        elif str_q:
+            result_page = searcher.search_page(str_q, page)
+        else:
+            raise ValueError(
+                "Either querystring or more_like_doc_id is required."
+            )
 
-        q = qp.parse(querystring)
-        result_page = searcher.search_page(q, page)
         result_page.results.fragmenter = highlight.ContextFragmenter(
             surround=50)
         result_page.results.formatter = JsonFormatter()
 
-        corrected = searcher.correct_query(q, querystring)
-        if corrected.query != q:
+        if corrected and corrected.query != str_q:
             corrected_query = corrected.string
         else:
             corrected_query = None
diff --git a/src/documents/views.py b/src/documents/views.py
index bf31c749b..bd9a748e8 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -335,14 +335,19 @@ def add_infos_to_hit(self, r):
                 }
 
     def get(self, request, format=None):
-        if 'query' not in request.query_params:
-            return Response({
-                'count': 0,
-                'page': 0,
-                'page_count': 0,
-                'results': []})
-
-        query = request.query_params['query']
+
+        if 'query' in request.query_params:
+            query = request.query_params['query']
+        else:
+            query = None
+
+        if 'more_like' in request.query_params:
+            more_like_id = request.query_params['more_like']
+            more_like_content = Document.objects.get(id=more_like_id).content
+        else:
+            more_like_id = None
+            more_like_content = None
+
         try:
             page = int(request.query_params.get('page', 1))
         except (ValueError, TypeError):
@@ -352,7 +357,7 @@ def get(self, request, format=None):
             page = 1
 
         try:
-            with index.query_page(self.ix, query, page) as (result_page,
+            with index.query_page(self.ix, page, query, more_like_id, more_like_content) as (result_page,
                                                             corrected_query):
                 return Response(
                     {'count': len(result_page),

From 48796e6961b0f61366e3ff77f2b78a371153768e Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Thu, 17 Dec 2020 21:46:56 +0100
Subject: [PATCH 24/78] fixes #149

---
 src-ui/src/app/interceptors/csrf.interceptor.ts | 7 +++++--
 src/documents/templates/index.html              | 1 +
 src/documents/views.py                          | 5 +++++
 3 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/src-ui/src/app/interceptors/csrf.interceptor.ts b/src-ui/src/app/interceptors/csrf.interceptor.ts
index 32f3e99dc..2ef03dc56 100644
--- a/src-ui/src/app/interceptors/csrf.interceptor.ts
+++ b/src-ui/src/app/interceptors/csrf.interceptor.ts
@@ -7,16 +7,19 @@ import {
 } from '@angular/common/http';
 import { Observable } from 'rxjs';
 import { CookieService } from 'ngx-cookie-service';
+import { Meta } from '@angular/platform-browser';
 
 @Injectable()
 export class CsrfInterceptor implements HttpInterceptor {
 
-  constructor(private cookieService: CookieService) {
+  constructor(private cookieService: CookieService, private meta: Meta) {
 
   }
 
   intercept(request: HttpRequest<unknown>, next: HttpHandler): Observable<HttpEvent<unknown>> {
-    let csrfToken = this.cookieService.get('csrftoken')
+
+    let prefix = this.meta.getTag('name=cookie_prefix').content
+    let csrfToken = this.cookieService.get(`${prefix?prefix:''}csrftoken`)
     if (csrfToken) {
      request = request.clone({
         setHeaders: {
diff --git a/src/documents/templates/index.html b/src/documents/templates/index.html
index 728f3a0e7..06dbb678e 100644
--- a/src/documents/templates/index.html
+++ b/src/documents/templates/index.html
@@ -8,6 +8,7 @@
   <title>PaperlessUi</title>
   <base href="/">
   <meta name="viewport" content="width=device-width, initial-scale=1">
+	<meta name="cookie_prefix" content="{{cookie_prefix}}">
   <link rel="icon" type="image/x-icon" href="favicon.ico">
 <link rel="stylesheet" href="{% static 'frontend/styles.css' %}"></head>
 <body>
diff --git a/src/documents/views.py b/src/documents/views.py
index bf31c749b..f90e9f7bc 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -54,6 +54,11 @@
 class IndexView(TemplateView):
     template_name = "index.html"
 
+    def get_context_data(self, **kwargs):
+        context = super().get_context_data(**kwargs)
+        context['cookie_prefix'] = settings.COOKIE_PREFIX
+        return context
+
 
 class CorrespondentViewSet(ModelViewSet):
     model = Correspondent

From 35dcc54dc875f5015ac003f7701f4f4c04aa8350 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Thu, 17 Dec 2020 21:54:05 +0100
Subject: [PATCH 25/78] fixes cookie_prefix for development setups

---
 src-ui/src/app/interceptors/csrf.interceptor.ts | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/src-ui/src/app/interceptors/csrf.interceptor.ts b/src-ui/src/app/interceptors/csrf.interceptor.ts
index 2ef03dc56..2c654aa36 100644
--- a/src-ui/src/app/interceptors/csrf.interceptor.ts
+++ b/src-ui/src/app/interceptors/csrf.interceptor.ts
@@ -17,8 +17,10 @@ export class CsrfInterceptor implements HttpInterceptor {
   }
 
   intercept(request: HttpRequest<unknown>, next: HttpHandler): Observable<HttpEvent<unknown>> {
-
-    let prefix = this.meta.getTag('name=cookie_prefix').content
+    let prefix = ""
+    if (this.meta.getTag('name=cookie_prefix')) {
+      prefix = this.meta.getTag('name=cookie_prefix').content
+    }    
     let csrfToken = this.cookieService.get(`${prefix?prefix:''}csrftoken`)
     if (csrfToken) {
      request = request.clone({

From 2c3eaadbce9fac1a465c0e4c866f9df3ae4216b2 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Thu, 17 Dec 2020 23:24:28 +0100
Subject: [PATCH 26/78] test cases

---
 src/documents/tests/test_api.py | 19 +++++++++++++++++++
 src/documents/views.py          |  8 ++++++++
 2 files changed, 27 insertions(+)

diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py
index 49dddee87..ba1ab45ca 100644
--- a/src/documents/tests/test_api.py
+++ b/src/documents/tests/test_api.py
@@ -351,6 +351,25 @@ def test_search_spelling_correction(self):
 
         self.assertEqual(correction, None)
 
+    def test_search_more_like(self):
+        d1=Document.objects.create(title="invoice", content="the thing i bought at a shop and paid with bank account", checksum="A", pk=1)
+        d2=Document.objects.create(title="bank statement 1", content="things i paid for in august", pk=2, checksum="B")
+        d3=Document.objects.create(title="bank statement 3", content="things i paid for in september", pk=3, checksum="C")
+        with AsyncWriter(index.open_index()) as writer:
+            index.update_document(writer, d1)
+            index.update_document(writer, d2)
+            index.update_document(writer, d3)
+
+        response = self.client.get(f"/api/search/?more_like={d2.id}")
+
+        self.assertEqual(response.status_code, 200)
+
+        results = response.data['results']
+
+        self.assertEqual(len(results), 2)
+        self.assertEqual(results[0]['id'], d3.id)
+        self.assertEqual(results[1]['id'], d1.id)
+
     def test_statistics(self):
 
         doc1 = Document.objects.create(title="none1", checksum="A")
diff --git a/src/documents/views.py b/src/documents/views.py
index bd9a748e8..59fbfb213 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -348,6 +348,14 @@ def get(self, request, format=None):
             more_like_id = None
             more_like_content = None
 
+        if not query and not more_like_id:
+            return Response({
+                'count': 0,
+                'page': 0,
+                'page_count': 0,
+                'corrected_query': None,
+                'results': []})
+
         try:
             page = int(request.query_params.get('page', 1))
         except (ValueError, TypeError):

From 659cd3e9d5362d20b44c0f9112aec2e0a5fd7dfe Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Thu, 17 Dec 2020 23:41:46 +0100
Subject: [PATCH 27/78] hide search controls on document list

---
 .../document-card-large/document-card-large.component.html  | 4 ++--
 .../document-card-large/document-card-large.component.scss  | 6 ++++++
 .../document-card-large/document-card-large.component.ts    | 3 +++
 src-ui/src/app/components/search/search.component.html      | 3 ++-
 4 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
index 32abaaef1..58c0f6241 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
@@ -25,7 +25,7 @@ <h5 class="card-title" *ngIf="document.archive_serial_number">#{{document.archiv
 
         <div class="d-flex justify-content-between align-items-center">
           <div class="btn-group">
-            <a routerLink="/search" [queryParams]="{'more_like': document.id}" class="btn btn-sm btn-outline-secondary">
+            <a routerLink="/search" [queryParams]="{'more_like': document.id}" class="btn btn-sm btn-outline-secondary" *ngIf="moreLikeThis">
               <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-three-dots" viewBox="0 0 16 16">
                 <path fill-rule="evenodd" d="M3 9.5a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3zm5 0a1.5 1.5 0 1 1 0-3 1.5 1.5 0 0 1 0 3z"/>
               </svg>
@@ -51,7 +51,7 @@ <h5 class="card-title" *ngIf="document.archive_serial_number">#{{document.archiv
               </svg>
               Download
             </a>
-            <ngb-progressbar [type]="searchScoreClass" [value]="searchScore" style="width: 100px; height: 5px; margin: 10px;" [max]="1"></ngb-progressbar>
+            <ngb-progressbar *ngIf="searchScore" [type]="searchScoreClass" [value]="searchScore" class="search-score-bar" [max]="1"></ngb-progressbar>
             
           </div>
           
diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss
index 11fb10562..438d2c768 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss
@@ -9,4 +9,10 @@
   height: 100%;
   position: absolute;
 
+}
+
+.search-score-bar {
+  width: 100px;
+  height: 5px;
+  margin: 10px;
 }
\ No newline at end of file
diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
index 44f9cb906..bcc1b1f3c 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
@@ -12,6 +12,9 @@ export class DocumentCardLargeComponent implements OnInit {
 
   constructor(private documentService: DocumentService, private sanitizer: DomSanitizer) { }
 
+  @Input()
+  moreLikeThis: boolean = false
+
   @Input()
   document: PaperlessDocument
 
diff --git a/src-ui/src/app/components/search/search.component.html b/src-ui/src/app/components/search/search.component.html
index 609bea9e5..de6f0133f 100644
--- a/src-ui/src/app/components/search/search.component.html
+++ b/src-ui/src/app/components/search/search.component.html
@@ -21,7 +21,8 @@
     <app-document-card-large *ngFor="let result of results"
         [document]="result.document"
         [details]="result.highlights"
-        [searchScore]="result.score / maxScore">
+        [searchScore]="result.score / maxScore"
+        [moreLikeThis]="true">
 
 </app-document-card-large>
 </div>

From 93be4e98d5ad79e6bf602e866937c8d1192798f7 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Thu, 17 Dec 2020 23:41:55 +0100
Subject: [PATCH 28/78] scroll to top when searching again

---
 src-ui/src/app/components/search/search.component.ts | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src-ui/src/app/components/search/search.component.ts b/src-ui/src/app/components/search/search.component.ts
index b2b10d632..4570ac3fa 100644
--- a/src-ui/src/app/components/search/search.component.ts
+++ b/src-ui/src/app/components/search/search.component.ts
@@ -41,6 +41,7 @@ export class SearchComponent implements OnInit {
 
   ngOnInit(): void {
     this.route.queryParamMap.subscribe(paramMap => {
+      window.scrollTo(0, 0)
       this.query = paramMap.get('query')
       this.more_like = paramMap.has('more_like') ? +paramMap.get('more_like') : null
       if (this.more_like) {

From ca2cb694d0dbd70c550c8dc9e69a578e9fe6fd4e Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 18 Dec 2020 00:10:16 +0100
Subject: [PATCH 29/78] code style

---
 src/documents/index.py | 13 ++++++++-----
 src/documents/views.py |  3 +--
 2 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/src/documents/index.py b/src/documents/index.py
index 7d022182f..fdf7d7041 100644
--- a/src/documents/index.py
+++ b/src/documents/index.py
@@ -136,11 +136,14 @@ def query_page(ix, page, querystring, more_like_doc_id, more_like_doc_content):
 
         if more_like_doc_id:
             docnum = searcher.document_number(id=more_like_doc_id)
-            kts = searcher.key_terms_from_text('content', more_like_doc_content, numterms=20,
-                                           model=classify.Bo1Model, normalize=False)
-            more_like_q = query.Or([query.Term('content', word, boost=weight)
-                          for word, weight in kts])
-            result_page = searcher.search_page(more_like_q, page, filter=str_q, mask={docnum})
+            kts = searcher.key_terms_from_text(
+                'content', more_like_doc_content, numterms=20,
+                model=classify.Bo1Model, normalize=False)
+            more_like_q = query.Or(
+                [query.Term('content', word, boost=weight)
+                 for word, weight in kts])
+            result_page = searcher.search_page(
+                more_like_q, page, filter=str_q, mask={docnum})
         elif str_q:
             result_page = searcher.search_page(str_q, page)
         else:
diff --git a/src/documents/views.py b/src/documents/views.py
index 1f5489999..54d0de3f6 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -370,8 +370,7 @@ def get(self, request, format=None):
             page = 1
 
         try:
-            with index.query_page(self.ix, page, query, more_like_id, more_like_content) as (result_page,
-                                                            corrected_query):
+            with index.query_page(self.ix, page, query, more_like_id, more_like_content) as (result_page, corrected_query):  # NOQA: E501
                 return Response(
                     {'count': len(result_page),
                      'page': result_page.pagenum,

From 9b244d02655c7837323633b71826a89473b6e19a Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Thu, 17 Dec 2020 23:09:27 -0800
Subject: [PATCH 30/78] Use ng-select for document detail screen

---
 src-ui/package-lock.json                      |  8 +++++++
 src-ui/package.json                           |  1 +
 src-ui/src/app/app.module.ts                  |  4 +++-
 .../common/input/select/select.component.html | 18 +++++++++------
 .../common/input/select/select.component.scss |  1 +
 src-ui/src/styles.scss                        | 23 ++++++++++++++++++-
 6 files changed, 46 insertions(+), 9 deletions(-)

diff --git a/src-ui/package-lock.json b/src-ui/package-lock.json
index 5eca0b3c0..10215a32d 100644
--- a/src-ui/package-lock.json
+++ b/src-ui/package-lock.json
@@ -2056,6 +2056,14 @@
         "tslib": "^2.0.0"
       }
     },
+    "@ng-select/ng-select": {
+      "version": "5.0.9",
+      "resolved": "https://registry.npmjs.org/@ng-select/ng-select/-/ng-select-5.0.9.tgz",
+      "integrity": "sha512-YZeSAiS8/Nx/eHZJPmOOYL8YmcvSq+dr1P8WIrsKmRA7mueorBpPc5xlUj+nLQbpLtsiQvdWDQspf/ykOvD/lA==",
+      "requires": {
+        "tslib": "^2.0.0"
+      }
+    },
     "@ngtools/webpack": {
       "version": "10.2.0",
       "resolved": "https://registry.npmjs.org/@ngtools/webpack/-/webpack-10.2.0.tgz",
diff --git a/src-ui/package.json b/src-ui/package.json
index 6293f2672..14d828483 100644
--- a/src-ui/package.json
+++ b/src-ui/package.json
@@ -21,6 +21,7 @@
     "@angular/platform-browser-dynamic": "~10.1.5",
     "@angular/router": "~10.1.5",
     "@ng-bootstrap/ng-bootstrap": "^8.0.0",
+    "@ng-select/ng-select": "^5.0.9",
     "bootstrap": "^4.5.0",
     "ng-bootstrap": "^1.6.3",
     "ng2-pdf-viewer": "^6.3.2",
diff --git a/src-ui/src/app/app.module.ts b/src-ui/src/app/app.module.ts
index 3c00cd0b7..d9c3800d6 100644
--- a/src-ui/src/app/app.module.ts
+++ b/src-ui/src/app/app.module.ts
@@ -54,6 +54,7 @@ import { FileSizePipe } from './pipes/file-size.pipe';
 import { FilterPipe } from './pipes/filter.pipe';
 import { DocumentTitlePipe } from './pipes/document-title.pipe';
 import { MetadataCollapseComponent } from './components/document-detail/metadata-collapse/metadata-collapse.component';
+import { NgSelectModule } from '@ng-select/ng-select';
 
 @NgModule({
   declarations: [
@@ -110,7 +111,8 @@ import { MetadataCollapseComponent } from './components/document-detail/metadata
     ReactiveFormsModule,
     NgxFileDropModule,
     InfiniteScrollModule,
-    PdfViewerModule
+    PdfViewerModule,
+    NgSelectModule
   ],
   providers: [
     DatePipe,
diff --git a/src-ui/src/app/components/common/input/select/select.component.html b/src-ui/src/app/components/common/input/select/select.component.html
index 717aa7964..655adbe74 100644
--- a/src-ui/src/app/components/common/input/select/select.component.html
+++ b/src-ui/src/app/components/common/input/select/select.component.html
@@ -1,11 +1,15 @@
-<div class="form-group">
+<div class="form-group paperless-input-select">
   <label [for]="inputId">{{title}}</label>
   <div [class.input-group]="showPlusButton()">
-    <select class="form-control" [id]="inputId" [(ngModel)]="value" (change)="onChange(value)" (blur)="onTouched()"
-      [disabled]="disabled" [style.color]="textColor" [style.background]="backgroundColor">
-      <option *ngIf="allowNull" [ngValue]="null" class="form-control">---</option>
-      <option *ngFor="let i of items" [ngValue]="i.id" class="form-control">{{i.name}}</option>
-    </select>
+    <ng-select name="correspondent" [(ngModel)]="value"
+      [disabled]="disabled"
+      [style.color]="textColor"
+      [style.background]="backgroundColor"
+      (change)="onChange(value)"
+      (blur)="onTouched()">
+      <ng-option *ngFor="let i of items" [value]="i.id">{{i.name}}</ng-option>
+    </ng-select>
+
     <div *ngIf="showPlusButton()" class="input-group-append">
       <button class="btn btn-outline-secondary" type="button" (click)="createNew.emit()">
         <svg class="buttonicon" fill="currentColor">
@@ -15,4 +19,4 @@
     </div>
   </div>
   <small *ngIf="hint" class="form-text text-muted">{{hint}}</small>
-</div>
\ No newline at end of file
+</div>
diff --git a/src-ui/src/app/components/common/input/select/select.component.scss b/src-ui/src/app/components/common/input/select/select.component.scss
index e69de29bb..8faec3bc0 100644
--- a/src-ui/src/app/components/common/input/select/select.component.scss
+++ b/src-ui/src/app/components/common/input/select/select.component.scss
@@ -0,0 +1 @@
+// styles for ng-select child are in styles.scss
diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss
index b0b66b7f9..2eeb40d41 100644
--- a/src-ui/src/styles.scss
+++ b/src-ui/src/styles.scss
@@ -2,6 +2,7 @@
 
 @import "node_modules/bootstrap/scss/bootstrap";
 
+@import "~@ng-select/ng-select/themes/default.theme.css";
 
 .toolbaricon {
   width: 1.2em;
@@ -65,4 +66,24 @@ body {
   display: block;
   background-size: 1rem;
   float: right;
-}
\ No newline at end of file
+}
+
+.paperless-input-select {
+  .ng-select {
+    position: relative;
+    flex: 1 1 auto;
+    margin-bottom: 0;
+    height: calc(1.5em + 0.75rem + 5px);
+    line-height: 1.5;
+
+    .ng-select-container {
+      height: 100%;
+      border-top-right-radius: 0;
+      border-bottom-right-radius: 0;
+
+      .ng-value-container .ng-input {
+        top: 8px;
+      }
+    }
+  }
+}

From e10a2391c44dcb48db992b0d477a8f150a0bb481 Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Fri, 18 Dec 2020 00:53:01 -0800
Subject: [PATCH 31/78] Use ng-select for document detail screen tags

---
 .../common/input/tags/tags.component.html     | 36 ++++++++++---------
 .../common/input/tags/tags.component.scss     | 12 ++-----
 .../common/input/tags/tags.component.ts       | 23 ++++++------
 .../document-detail.component.html            |  4 +--
 src-ui/src/styles.scss                        | 15 ++++++--
 5 files changed, 48 insertions(+), 42 deletions(-)

diff --git a/src-ui/src/app/components/common/input/tags/tags.component.html b/src-ui/src/app/components/common/input/tags/tags.component.html
index 8029dd860..89e391813 100644
--- a/src-ui/src/app/components/common/input/tags/tags.component.html
+++ b/src-ui/src/app/components/common/input/tags/tags.component.html
@@ -1,30 +1,34 @@
-<div class="form-group">
-  <label for="exampleFormControlTextarea1">Tags</label>
+<div class="form-group paperless-input-select paperless-input-tags">
+  <label for="tags">Tags</label>
 
   <div class="input-group">
-    <div class="form-control tags-form-control" id="tags">
-      <app-tag class="mr-2" *ngFor="let id of displayValue" [tag]="getTag(id)" (click)="removeTag(id)"></app-tag>
-    </div>
+    <ng-select name="tags" [items]="tags" bindLabel="name" bindValue="id" [(ngModel)]="displayValue"
+      [multiple]="true"
+      [closeOnSelect]="false"
+      [disabled]="disabled"
+      (change)="ngSelectChange()">
 
-    <div class="input-group-append" ngbDropdown placement="top-right">
-      <button class="btn btn-outline-secondary" type="button" ngbDropdownToggle></button>
-      <div ngbDropdownMenu class="scrollable-menu shadow">
-        <button type="button" *ngFor="let tag of tags" ngbDropdownItem (click)="addTag(tag.id)">
-          <app-tag [tag]="tag"></app-tag>
-        </button>
-      </div>
-    </div>
+      <ng-template ng-label-tmp let-item="item">
+          <app-tag style="background-color: none;" [tag]="getTag(item.id)" (click)="removeTag(item.id)"></app-tag>
+      </ng-template>
+      <ng-template ng-option-tmp let-item="item" let-index="index" let-search="searchTerm">
+        <div class="selected-icon d-inline-block mr-1">
+          <svg *ngIf="displayValue.includes(item.id)" width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-check" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+            <path fill-rule="evenodd" d="M10.97 4.97a.75.75 0 0 1 1.071 1.05l-3.992 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.236.236 0 0 1 .02-.022z"/>
+          </svg>
+        </div>
+        <app-tag class="mr-2" [tag]="getTag(item.id)"></app-tag>
+      </ng-template>
+    </ng-select>
 
     <div class="input-group-append">
-
       <button class="btn btn-outline-secondary" type="button" (click)="createTag()">
         <svg class="buttonicon" fill="currentColor">
           <use xlink:href="assets/bootstrap-icons.svg#plus" />
         </svg>
       </button>
     </div>
-
   </div>
   <small class="form-text text-muted" *ngIf="hint">{{hint}}</small>
 
-</div>
\ No newline at end of file
+</div>
diff --git a/src-ui/src/app/components/common/input/tags/tags.component.scss b/src-ui/src/app/components/common/input/tags/tags.component.scss
index f2635b7f2..41fc6acc4 100644
--- a/src-ui/src/app/components/common/input/tags/tags.component.scss
+++ b/src-ui/src/app/components/common/input/tags/tags.component.scss
@@ -1,10 +1,4 @@
-.tags-form-control {
-  height: auto;
+.selected-icon {
+  min-width: 1em;
+  min-height: 1em;
 }
-
-
-.scrollable-menu {
-  height: auto;
-  max-height: 300px;
-  overflow-x: hidden;
-}
\ No newline at end of file
diff --git a/src-ui/src/app/components/common/input/tags/tags.component.ts b/src-ui/src/app/components/common/input/tags/tags.component.ts
index cca99cc55..5501ac5a6 100644
--- a/src-ui/src/app/components/common/input/tags/tags.component.ts
+++ b/src-ui/src/app/components/common/input/tags/tags.component.ts
@@ -21,7 +21,7 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
 
 
   onChange = (newValue: number[]) => {};
-  
+
   onTouched = () => {};
 
   writeValue(newValue: number[]): void {
@@ -66,29 +66,28 @@ export class TagsComponent implements OnInit, ControlValueAccessor {
   removeTag(id) {
     let index = this.displayValue.indexOf(id)
     if (index > -1) {
-      this.displayValue.splice(index, 1)
+      let oldValue = this.displayValue
+      oldValue.splice(index, 1)
+      this.displayValue = [...oldValue]
       this.onChange(this.displayValue)
     }
   }
 
-  addTag(id) {
-    let index = this.displayValue.indexOf(id)
-    if (index == -1) {
-      this.displayValue.push(id)
-      this.onChange(this.displayValue)
-    }
-  }
-
-
   createTag() {
     var modal = this.modalService.open(TagEditDialogComponent, {backdrop: 'static'})
     modal.componentInstance.dialogMode = 'create'
     modal.componentInstance.success.subscribe(newTag => {
       this.tagService.listAll().subscribe(tags => {
         this.tags = tags.results
-        this.addTag(newTag.id)
+        this.displayValue = [...this.displayValue, newTag.id]
+        this.onChange(this.displayValue)
       })
     })
   }
 
+  ngSelectChange() {
+    this.value = this.displayValue
+    this.onChange(this.displayValue)
+  }
+
 }
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html
index f4a64c2cc..a3bc7e1e6 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.html
+++ b/src-ui/src/app/components/document-detail/document-detail.component.html
@@ -52,9 +52,9 @@
                         </div>
                         <app-input-date-time titleDate="Date created" formControlName="created"></app-input-date-time>
                         <app-input-select [items]="correspondents" title="Correspondent" formControlName="correspondent"
-                            allowNull="true" (createNew)="createCorrespondent()"></app-input-select>
+                            (createNew)="createCorrespondent()"></app-input-select>
                         <app-input-select [items]="documentTypes" title="Document type" formControlName="document_type"
-                            allowNull="true" (createNew)="createDocumentType()"></app-input-select>
+                            (createNew)="createDocumentType()"></app-input-select>
                         <app-input-tags formControlName="tags" title="Tags"></app-input-tags>
 
                     </ng-template>
diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss
index 2eeb40d41..0dc662e31 100644
--- a/src-ui/src/styles.scss
+++ b/src-ui/src/styles.scss
@@ -1,7 +1,5 @@
 @import "theme";
-
 @import "node_modules/bootstrap/scss/bootstrap";
-
 @import "~@ng-select/ng-select/themes/default.theme.css";
 
 .toolbaricon {
@@ -21,7 +19,7 @@
 }
 
 body {
-  font-size: .875rem;
+  font-size: 0.875rem;
 }
 
 .form-control-dark {
@@ -85,5 +83,16 @@ body {
         top: 8px;
       }
     }
+
+    .ng-dropdown-panel .ng-dropdown-panel-items .ng-option.ng-option-selected,
+    .ng-dropdown-panel .ng-dropdown-panel-items .ng-option.ng-option-selected.ng-option-marked {
+      background: none;
+    }
+  }
+}
+
+.paperless-input-tags {
+  .ng-select.ng-select-multiple .ng-select-container .ng-value-container .ng-value {
+    background-color: transparent;
   }
 }

From 55c4c690ef8eec3ba494cd6c3a74dc9b5cd810ec Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Fri, 18 Dec 2020 01:13:30 -0800
Subject: [PATCH 32/78] Fix wrapping with multiple tags, embiggen tags, pretty
 icons

---
 .../common/input/tags/tags.component.html     | 21 ++++++++++++-------
 .../common/input/tags/tags.component.scss     |  8 +++++++
 src-ui/src/styles.scss                        |  2 +-
 3 files changed, 23 insertions(+), 8 deletions(-)

diff --git a/src-ui/src/app/components/common/input/tags/tags.component.html b/src-ui/src/app/components/common/input/tags/tags.component.html
index 89e391813..8a5dbc4f2 100644
--- a/src-ui/src/app/components/common/input/tags/tags.component.html
+++ b/src-ui/src/app/components/common/input/tags/tags.component.html
@@ -1,7 +1,7 @@
 <div class="form-group paperless-input-select paperless-input-tags">
   <label for="tags">Tags</label>
 
-  <div class="input-group">
+  <div class="input-group flex-nowrap">
     <ng-select name="tags" [items]="tags" bindLabel="name" bindValue="id" [(ngModel)]="displayValue"
       [multiple]="true"
       [closeOnSelect]="false"
@@ -9,15 +9,22 @@
       (change)="ngSelectChange()">
 
       <ng-template ng-label-tmp let-item="item">
-          <app-tag style="background-color: none;" [tag]="getTag(item.id)" (click)="removeTag(item.id)"></app-tag>
+        <span class="tag-wrap tag-wrap-delete" (click)="removeTag(item.id)">
+          <svg width="1.2em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+            <use xlink:href="assets/bootstrap-icons.svg#x"/>
+          </svg>
+          <app-tag style="background-color: none;" [tag]="getTag(item.id)"></app-tag>
+        </span>
       </ng-template>
       <ng-template ng-option-tmp let-item="item" let-index="index" let-search="searchTerm">
-        <div class="selected-icon d-inline-block mr-1">
-          <svg *ngIf="displayValue.includes(item.id)" width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-check" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
-            <path fill-rule="evenodd" d="M10.97 4.97a.75.75 0 0 1 1.071 1.05l-3.992 4.99a.75.75 0 0 1-1.08.02L4.324 8.384a.75.75 0 1 1 1.06-1.06l2.094 2.093 3.473-4.425a.236.236 0 0 1 .02-.022z"/>
-          </svg>
+        <div class="tag-wrap">
+          <div class="selected-icon d-inline-block mr-1">
+            <svg *ngIf="displayValue.includes(item.id)" width="1em" height="1em" viewBox="0 0 16 16" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+              <use xlink:href="assets/bootstrap-icons.svg#check"/>
+            </svg>
+          </div>
+          <app-tag class="mr-2" [tag]="getTag(item.id)"></app-tag>
         </div>
-        <app-tag class="mr-2" [tag]="getTag(item.id)"></app-tag>
       </ng-template>
     </ng-select>
 
diff --git a/src-ui/src/app/components/common/input/tags/tags.component.scss b/src-ui/src/app/components/common/input/tags/tags.component.scss
index 41fc6acc4..2eaaa4f6d 100644
--- a/src-ui/src/app/components/common/input/tags/tags.component.scss
+++ b/src-ui/src/app/components/common/input/tags/tags.component.scss
@@ -2,3 +2,11 @@
   min-width: 1em;
   min-height: 1em;
 }
+
+.tag-wrap {
+  font-size: 1rem;
+}
+
+.tag-wrap-delete {
+  cursor: pointer;
+}
diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss
index 0dc662e31..ffb296271 100644
--- a/src-ui/src/styles.scss
+++ b/src-ui/src/styles.scss
@@ -71,7 +71,7 @@ body {
     position: relative;
     flex: 1 1 auto;
     margin-bottom: 0;
-    height: calc(1.5em + 0.75rem + 5px);
+    min-height: calc(1.5em + 0.75rem + 5px);
     line-height: 1.5;
 
     .ng-select-container {

From c05de3d57f6148d4abee8f8775d6668f6984ed2d Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Fri, 18 Dec 2020 01:18:11 -0800
Subject: [PATCH 33/78] Tiny padding fixes

---
 src-ui/src/styles.scss | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src-ui/src/styles.scss b/src-ui/src/styles.scss
index ffb296271..6e09db630 100644
--- a/src-ui/src/styles.scss
+++ b/src-ui/src/styles.scss
@@ -80,7 +80,7 @@ body {
       border-bottom-right-radius: 0;
 
       .ng-value-container .ng-input {
-        top: 8px;
+        top: 10px;
       }
     }
 
@@ -95,4 +95,8 @@ body {
   .ng-select.ng-select-multiple .ng-select-container .ng-value-container .ng-value {
     background-color: transparent;
   }
+
+  .ng-select.ng-select-multiple .ng-select-container .ng-value-container {
+    padding-top: 1px;
+  }
 }

From 273c474e3fe00f68898c1dc74256a2f4de7174e9 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 18 Dec 2020 14:09:12 +0100
Subject: [PATCH 34/78] layout changes

---
 .../document-card-large/document-card-large.component.html | 7 +++++--
 .../document-card-large/document-card-large.component.scss | 2 +-
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
index 58c0f6241..5bf0c9af2 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
@@ -23,7 +23,7 @@ <h5 class="card-title" *ngIf="document.archive_serial_number">#{{document.archiv
         </p>
 
 
-        <div class="d-flex justify-content-between align-items-center">
+        <div class="d-flex align-items-center">
           <div class="btn-group">
             <a routerLink="/search" [queryParams]="{'more_like': document.id}" class="btn btn-sm btn-outline-secondary" *ngIf="moreLikeThis">
               <svg xmlns="http://www.w3.org/2000/svg" width="16" height="16" fill="currentColor" class="bi bi-three-dots" viewBox="0 0 16 16">
@@ -51,9 +51,12 @@ <h5 class="card-title" *ngIf="document.archive_serial_number">#{{document.archiv
               </svg>
               Download
             </a>
-            <ngb-progressbar *ngIf="searchScore" [type]="searchScoreClass" [value]="searchScore" class="search-score-bar" [max]="1"></ngb-progressbar>
             
           </div>
+
+          <small class="text-muted ml-auto">Score:</small>
+
+          <ngb-progressbar *ngIf="searchScore" [type]="searchScoreClass" [value]="searchScore" class="search-score-bar mx-2" [max]="1"></ngb-progressbar>
           
           <small class="text-muted">Created: {{document.created | date}}</small>
         </div>
diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss
index 438d2c768..a20a56672 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss
@@ -14,5 +14,5 @@
 .search-score-bar {
   width: 100px;
   height: 5px;
-  margin: 10px;
+  margin-top: 2px;
 }
\ No newline at end of file

From 789abb3bbb3c14b85952e98485cf98c289deb9c0 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 18 Dec 2020 16:42:33 +0100
Subject: [PATCH 35/78] changed up the highlight fragment formatter

---
 docs/api.rst                                  | 15 ++++------
 .../result-highlight.component.html           |  2 +-
 src/documents/index.py                        | 29 +++++++++++--------
 3 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/docs/api.rst b/docs/api.rst
index d352758fa..cff72a970 100644
--- a/docs/api.rst
+++ b/docs/api.rst
@@ -221,21 +221,16 @@ Each fragment contains a list of strings, and some of them are marked as a highl
 
     [
         [
-            {"text": "This is a sample text with a "},
-            {"text": "highlighted", "term": 0},
-            {"text": " word."}
+            {"text": "This is a sample text with a ", "highlight": false},
+            {"text": "highlighted", "highlight": true},
+            {"text": " word.", "highlight": false}
         ],
         [
-            {"text": "Another", "term": 1},
-            {"text": " fragment with a highlight."}
+            {"text": "Another", "highlight": true},
+            {"text": " fragment with a highlight.", "highlight": false}
         ]
     ]
 
-
-
-When ``term`` is present within a string, the word within ``text`` should be highlighted.
-The term index groups multiple matches together and words with the same index
-should get identical highlighting.
 A client may use this example to produce the following output:
 
 ... This is a sample text with a **highlighted** word. ... **Another** fragment with a highlight. ...
diff --git a/src-ui/src/app/components/search/result-highlight/result-highlight.component.html b/src-ui/src/app/components/search/result-highlight/result-highlight.component.html
index 1842f5cea..5dc5baa94 100644
--- a/src-ui/src/app/components/search/result-highlight/result-highlight.component.html
+++ b/src-ui/src/app/components/search/result-highlight/result-highlight.component.html
@@ -1,3 +1,3 @@
 ... <span *ngFor="let fragment of highlights">
-    <span *ngFor="let token of fragment" [ngClass]="token.term != null ? 'match term'+ token.term : ''">{{token.text}}</span> ... 
+    <span *ngFor="let token of fragment" [class.match]="token.highlight">{{token.text}}</span> ... 
 </span>
\ No newline at end of file
diff --git a/src/documents/index.py b/src/documents/index.py
index fdf7d7041..308ee932e 100644
--- a/src/documents/index.py
+++ b/src/documents/index.py
@@ -20,32 +20,37 @@ def __init__(self):
         self.seen = {}
 
     def format_token(self, text, token, replace=False):
-        seen = self.seen
         ttext = self._text(get_text(text, token, replace))
-        if ttext in seen:
-            termnum = seen[ttext]
-        else:
-            termnum = len(seen)
-            seen[ttext] = termnum
-
-        return {'text': ttext, 'term': termnum}
+        return {'text': ttext, 'highlight': 'true'}
 
     def format_fragment(self, fragment, replace=False):
         output = []
         index = fragment.startchar
         text = fragment.text
-
+        amend_token = None
         for t in fragment.matches:
             if t.startchar is None:
                 continue
             if t.startchar < index:
                 continue
             if t.startchar > index:
-                output.append({'text': text[index:t.startchar]})
-            output.append(self.format_token(text, t, replace))
+                text_inbetween = text[index:t.startchar]
+                if amend_token and t.startchar - index < 10:
+                    amend_token['text'] += text_inbetween
+                else:
+                    output.append({'text': text_inbetween,
+                                   'highlight': False})
+                    amend_token = None
+            token = self.format_token(text, t, replace)
+            if amend_token:
+                amend_token['text'] += token['text']
+            else:
+                output.append(token)
+                amend_token = token
             index = t.endchar
         if index < fragment.endchar:
-            output.append({'text': text[index:fragment.endchar]})
+            output.append({'text': text[index:fragment.endchar],
+                           'highlight': False})
         return output
 
     def format(self, fragments, replace=False):

From dfb88ebf8328ef9ed6b71c9da4b9ddac49768ea5 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Fri, 18 Dec 2020 20:17:17 +0100
Subject: [PATCH 36/78] removed the date hack. fixes #144

also refer to #148
---
 .../filter-dropdown-date.component.html       |  51 +++----
 .../filter-dropdown-date.component.ts         | 135 ++++++++----------
 .../filter-editor/filter-editor.component.ts  |  31 ++--
 3 files changed, 104 insertions(+), 113 deletions(-)

diff --git a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html
index 6f6a42fe2..aca6e836c 100644
--- a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html
+++ b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.html
@@ -4,38 +4,39 @@
   </button>
   <div class="dropdown-menu date-filter shadow pt-0" ngbDropdownMenu attr.aria-labelledby="dropdown{{title}}">
     <div class="list-group list-group-flush">
-        <button class="list-group-item small list-goup list-group-item-action d-flex p-2 pl-3" (click)="clear()">Clear</button>
-        <button *ngFor="let range of [7, 30, 'month', 'year']" class="list-group-item small list-goup list-group-item-action d-flex p-2 pl-3" role="menuitem" (click)="setDateQuickFilter(range)">
-          <ng-container *ngIf="isStringRange(range)">This </ng-container>
-          {{ range }}
-          <ng-container *ngIf="!isStringRange(range)"> days</ng-container>
+        <button *ngFor="let qf of quickFilters" class="list-group-item small list-goup list-group-item-action d-flex p-2 pl-3" role="menuitem" (click)="setDateQuickFilter(qf.id)">
+          {{qf.name}}
         </button>
         <div class="list-group-item d-flex flex-column align-items-start" role="menuitem">
-          <div>Before</div>
+
+          <div class="mb-2 d-flex flex-row w-100 justify-content-between small">
+            <div>After</div>
+            <a *ngIf="dateAfter" class="btn btn-link p-0 m-0" (click)="clearAfter()">
+              <svg width="0.8em" height="0.8em" viewBox="0 0 16 16" class="bi bi-x" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+                <path fill-rule="evenodd" d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z" />
+              </svg>
+              <small>Clear</small>
+            </a>
+          </div>
+
           <div class="input-group input-group-sm">
-            <input class="form-control" type="text" placeholder="yyyy-mm-dd" name="before" [(ngModel)]="_dateBefore" [maxDate]="this._maxDate" ngbDatepicker (dateSelect)="onBeforeSelected($event)" #dpBefore="ngbDatepicker">
-            <div class="input-group-append">
-              <button class="btn btn-outline-secondary btn-sm" (click)="dpBefore.toggle()" type="button">
-                <svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-calendar-date" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
-                  <path fill-rule="evenodd" d="M3.5 0a.5.5 0 0 1 .5.5V1h8V.5a.5.5 0 0 1 1 0V1h1a2 2 0 0 1 2 2v11a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2V3a2 2 0 0 1 2-2h1V.5a.5.5 0 0 1 .5-.5zM1 4v10a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1V4H1z"/>
-                  <path d="M6.445 11.688V6.354h-.633A12.6 12.6 0 0 0 4.5 7.16v.695c.375-.257.969-.62 1.258-.777h.012v4.61h.675zm1.188-1.305c.047.64.594 1.406 1.703 1.406 1.258 0 2-1.066 2-2.871 0-1.934-.781-2.668-1.953-2.668-.926 0-1.797.672-1.797 1.809 0 1.16.824 1.77 1.676 1.77.746 0 1.23-.376 1.383-.79h.027c-.004 1.316-.461 2.164-1.305 2.164-.664 0-1.008-.45-1.05-.82h-.684zm2.953-2.317c0 .696-.559 1.18-1.184 1.18-.601 0-1.144-.383-1.144-1.2 0-.823.582-1.21 1.168-1.21.633 0 1.16.398 1.16 1.23z"/>
-                </svg>
-              </button>
-            </div>
+            <input type="date" class="form-control" id="date_after" [(ngModel)]="dateAfter" (change)="onChangeDebounce()">
           </div>
         </div>
         <div class="list-group-item d-flex flex-column align-items-start" role="menuitem">
-          <div>After</div>
+
+          <div class="mb-2 d-flex flex-row w-100 justify-content-between small">
+            <div>Before</div>
+            <a *ngIf="dateBefore" class="btn btn-link p-0 m-0" (click)="clearBefore()">
+              <svg width="0.8em" height="0.8em" viewBox="0 0 16 16" class="bi bi-x" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
+                <path fill-rule="evenodd" d="M4.646 4.646a.5.5 0 0 1 .708 0L8 7.293l2.646-2.647a.5.5 0 0 1 .708.708L8.707 8l2.647 2.646a.5.5 0 0 1-.708.708L8 8.707l-2.646 2.647a.5.5 0 0 1-.708-.708L7.293 8 4.646 5.354a.5.5 0 0 1 0-.708z" />
+              </svg>
+              <small>Clear</small>
+            </a>
+          </div>
+
           <div class="input-group input-group-sm">
-            <input class="form-control form-control-sm" type="text" placeholder="yyyy-mm-dd" name="after" [(ngModel)]="_dateAfter" [maxDate]="this._maxDate" ngbDatepicker (dateSelect)="onAfterSelected($event)" #dpAfter="ngbDatepicker">
-            <div class="input-group-append">
-              <button class="btn btn-outline-secondary btn-sm" (click)="dpAfter.toggle()" type="button">
-                <svg width="1em" height="1em" viewBox="0 0 16 16" class="bi bi-calendar-date" fill="currentColor" xmlns="http://www.w3.org/2000/svg">
-                  <path fill-rule="evenodd" d="M3.5 0a.5.5 0 0 1 .5.5V1h8V.5a.5.5 0 0 1 1 0V1h1a2 2 0 0 1 2 2v11a2 2 0 0 1-2 2H2a2 2 0 0 1-2-2V3a2 2 0 0 1 2-2h1V.5a.5.5 0 0 1 .5-.5zM1 4v10a1 1 0 0 0 1 1h12a1 1 0 0 0 1-1V4H1z"/>
-                  <path d="M6.445 11.688V6.354h-.633A12.6 12.6 0 0 0 4.5 7.16v.695c.375-.257.969-.62 1.258-.777h.012v4.61h.675zm1.188-1.305c.047.64.594 1.406 1.703 1.406 1.258 0 2-1.066 2-2.871 0-1.934-.781-2.668-1.953-2.668-.926 0-1.797.672-1.797 1.809 0 1.16.824 1.77 1.676 1.77.746 0 1.23-.376 1.383-.79h.027c-.004 1.316-.461 2.164-1.305 2.164-.664 0-1.008-.45-1.05-.82h-.684zm2.953-2.317c0 .696-.559 1.18-1.184 1.18-.601 0-1.144-.383-1.144-1.2 0-.823.582-1.21 1.168-1.21.633 0 1.16.398 1.16 1.23z"/>
-                </svg>
-              </button>
-            </div>
+            <input type="date" class="form-control" id="date_before" [(ngModel)]="dateBefore" (change)="onChangeDebounce()">
           </div>
         </div>
     </div>
diff --git a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts
index 91402d084..e570862cd 100644
--- a/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts
+++ b/src-ui/src/app/components/filter-editor/filter-dropdown-date/filter-dropdown-date.component.ts
@@ -1,24 +1,37 @@
-import { Component, EventEmitter, Input, Output, ElementRef, ViewChild, SimpleChange } from '@angular/core';
-import { NgbDate, NgbDateStruct, NgbDatepicker } from '@ng-bootstrap/ng-bootstrap';
+import { formatDate } from '@angular/common';
+import { Component, EventEmitter, Input, Output, OnInit, OnDestroy } from '@angular/core';
+import { Subject, Subscription } from 'rxjs';
+import { debounceTime, distinctUntilChanged } from 'rxjs/operators';
 
 export interface DateSelection {
-  before?: NgbDateStruct
-  after?: NgbDateStruct
+  before?: string
+  after?: string
 }
 
+const FILTER_LAST_7_DAYS = 0
+const FILTER_LAST_MONTH = 1
+const FILTER_LAST_3_MONTHS = 2
+const FILTER_LAST_YEAR = 3
 
 @Component({
   selector: 'app-filter-dropdown-date',
   templateUrl: './filter-dropdown-date.component.html',
   styleUrls: ['./filter-dropdown-date.component.scss']
 })
-export class FilterDropdownDateComponent {
+export class FilterDropdownDateComponent implements OnInit, OnDestroy {
+
+  quickFilters = [
+    {id: FILTER_LAST_7_DAYS, name: "Last 7 days"}, 
+    {id: FILTER_LAST_MONTH, name: "Last month"},
+    {id: FILTER_LAST_3_MONTHS, name: "Last 3 months"},
+    {id: FILTER_LAST_YEAR, name: "Last year"}
+  ]
 
   @Input()
-  dateBefore: NgbDateStruct
+  dateBefore: string
 
   @Input()
-  dateAfter: NgbDateStruct
+  dateAfter: string
 
   @Input()
   title: string
@@ -26,87 +39,65 @@ export class FilterDropdownDateComponent {
   @Output()
   datesSet = new EventEmitter<DateSelection>()
 
-  @ViewChild('dpAfter') dpAfter: NgbDatepicker
-  @ViewChild('dpBefore') dpBefore: NgbDatepicker
-
-  _dateBefore: NgbDateStruct
-  _dateAfter: NgbDateStruct
-
-  get _maxDate(): NgbDate {
-    let date = new Date()
-    return NgbDate.from({year: date.getFullYear(), month: date.getMonth() + 1, day: date.getDate()})
-  }
+  private datesSetDebounce$ = new Subject()
 
-  isStringRange(range: any) {
-    return typeof range == 'string'
+  private sub: Subscription
+  
+  ngOnInit() {
+    this.sub = this.datesSetDebounce$.pipe(
+      debounceTime(400)
+    ).subscribe(() => {
+      this.onChange()
+    })
   }
 
-  ngOnChanges(changes: SimpleChange) {
-    // this is a hacky workaround perhaps because of https://github.com/angular/angular/issues/11097
-    let dateString: string = ''
-    let dateAfterChange: SimpleChange
-    let dateBeforeChange: SimpleChange
-    if (changes) {
-      dateAfterChange = changes['dateAfter']
-      dateBeforeChange = changes['dateBefore']
-    }
-
-    if (this.dpBefore && this.dpAfter) {
-      let dpAfterElRef: ElementRef = this.dpAfter['_elRef']
-      let dpBeforeElRef: ElementRef = this.dpBefore['_elRef']
-
-      if (dateAfterChange && dateAfterChange.currentValue) {
-        let dateAfterDate = dateAfterChange.currentValue as NgbDateStruct
-        dateString = `${dateAfterDate.year}-${dateAfterDate.month.toString().padStart(2,'0')}-${dateAfterDate.day.toString().padStart(2,'0')}`
-        dpAfterElRef.nativeElement.value = dateString
-      } else if (dateBeforeChange && dateBeforeChange.currentValue) {
-        let dateBeforeDate = dateBeforeChange.currentValue as NgbDateStruct
-        dateString = `${dateBeforeDate.year}-${dateBeforeDate.month.toString().padStart(2,'0')}-${dateBeforeDate.day.toString().padStart(2,'0')}`
-        dpBeforeElRef.nativeElement.value = dateString
-      } else {
-        dpAfterElRef.nativeElement.value = dateString
-        dpBeforeElRef.nativeElement.value = dateString
-      }
+  ngOnDestroy() {
+    if (this.sub) {
+      this.sub.unsubscribe()
     }
   }
 
-  setDateQuickFilter(range: any) {
+  setDateQuickFilter(qf: number) {
+    this.dateBefore = null
     let date = new Date()
-    let newDate: NgbDateStruct = { year: date.getFullYear(), month: date.getMonth() + 1, day: date.getDate() }
-    switch (typeof range) {
-      case 'number':
-        date.setDate(date.getDate() - range)
-        newDate.year = date.getFullYear()
-        newDate.month = date.getMonth() + 1
-        newDate.day = date.getDate()
-        break
+    switch (qf) {
+      case FILTER_LAST_7_DAYS:
+        date.setDate(date.getDate() - 7)
+        break;
+
+      case FILTER_LAST_MONTH:
+        date.setMonth(date.getMonth() - 1)
+        break;
 
-      case 'string':
-        newDate.day = 1
-        if (range == 'year') newDate.month = 1
+      case FILTER_LAST_3_MONTHS:
+        date.setMonth(date.getMonth() - 3)
         break
 
-      default:
+      case FILTER_LAST_YEAR:
+        date.setFullYear(date.getFullYear() - 1)
         break
-    }
-    this._dateAfter = newDate
-    this._dateBefore = null
-    this.datesSet.emit({after: newDate, before: null})
+  
+      }
+    this.dateAfter = formatDate(date, 'yyyy-MM-dd', "en-us", "UTC")
+    this.onChange()
   }
 
-  onBeforeSelected(date: NgbDateStruct) {
-    this._dateBefore = date
-    this.datesSet.emit({after: this._dateAfter, before: date})
+  onChange() {
+    this.datesSet.emit({after: this.dateAfter, before: this.dateBefore})
   }
 
-  onAfterSelected(date: NgbDateStruct) {
-    this._dateAfter = date
-    this.datesSet.emit({after: date, before: this._dateBefore})
+  onChangeDebounce() {
+    this.datesSetDebounce$.next({after: this.dateAfter, before: this.dateBefore})
   }
 
-  clear() {
-    this._dateBefore = null
-    this._dateAfter = null
-    this.datesSet.emit({after: null, before: null})
+  clearBefore() {
+    this.dateBefore = null;
+    this.onChange()
   }
+
+  clearAfter() {
+    this.dateAfter = null;
+    this.onChange()
+  }
+
 }
diff --git a/src-ui/src/app/components/filter-editor/filter-editor.component.ts b/src-ui/src/app/components/filter-editor/filter-editor.component.ts
index f762c6138..913c738a5 100644
--- a/src-ui/src/app/components/filter-editor/filter-editor.component.ts
+++ b/src-ui/src/app/components/filter-editor/filter-editor.component.ts
@@ -179,54 +179,53 @@ export class FilterEditorComponent implements OnInit, OnDestroy {
     this.applyFilters()
   }
 
-  get dateCreatedBefore(): NgbDateStruct {
+  get dateCreatedBefore(): string {
     let createdBeforeRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_CREATED_BEFORE)
-    return createdBeforeRule ? this.dateParser.parse(createdBeforeRule.value) : null
+    return createdBeforeRule ? createdBeforeRule.value : null
   }
 
-  get dateCreatedAfter(): NgbDateStruct {
+  get dateCreatedAfter(): string {
     let createdAfterRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_CREATED_AFTER)
-    return createdAfterRule ? this.dateParser.parse(createdAfterRule.value) : null
+    return createdAfterRule ? createdAfterRule.value : null
   }
 
-  get dateAddedBefore(): NgbDateStruct {
+  get dateAddedBefore(): string {
     let addedBeforeRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_ADDED_BEFORE)
-    return addedBeforeRule ? this.dateParser.parse(addedBeforeRule.value) : null
+    return addedBeforeRule ? addedBeforeRule.value : null
   }
 
-  get dateAddedAfter(): NgbDateStruct {
+  get dateAddedAfter(): string {
     let addedAfterRule: FilterRule = this.filterRules.find(fr => fr.rule_type == FILTER_ADDED_AFTER)
-    return addedAfterRule ? this.dateParser.parse(addedAfterRule.value) : null
+    return addedAfterRule ? addedAfterRule.value : null
   }
 
-  setDateCreatedBefore(date?: NgbDateStruct) {
+  setDateCreatedBefore(date?: string) {
     if (date) this.setDateFilter(date, FILTER_CREATED_BEFORE)
     else this.clearDateFilter(FILTER_CREATED_BEFORE)
   }
 
-  setDateCreatedAfter(date?: NgbDateStruct) {
+  setDateCreatedAfter(date?: string) {
     if (date) this.setDateFilter(date, FILTER_CREATED_AFTER)
     else this.clearDateFilter(FILTER_CREATED_AFTER)
   }
 
-  setDateAddedBefore(date?: NgbDateStruct) {
+  setDateAddedBefore(date?: string) {
     if (date) this.setDateFilter(date, FILTER_ADDED_BEFORE)
     else this.clearDateFilter(FILTER_ADDED_BEFORE)
   }
 
-  setDateAddedAfter(date?: NgbDateStruct) {
+  setDateAddedAfter(date?: string) {
     if (date) this.setDateFilter(date, FILTER_ADDED_AFTER)
     else this.clearDateFilter(FILTER_ADDED_AFTER)
   }
 
-  setDateFilter(date: NgbDateStruct, dateRuleTypeID: number) {
+  setDateFilter(date: string, dateRuleTypeID: number) {
     let existingRule = this.filterRules.find(rule => rule.rule_type == dateRuleTypeID)
-    let newValue = this.dateParser.format(date)
 
     if (existingRule) {
-      existingRule.value = newValue
+      existingRule.value = date
     } else {
-      this.filterRules.push({rule_type: dateRuleTypeID, value: newValue})
+      this.filterRules.push({rule_type: dateRuleTypeID, value: date})
     }
   }
 

From e5fe515b69d570b83081d8cc2686635eb2bf8dff Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Fri, 18 Dec 2020 14:31:09 -0800
Subject: [PATCH 37/78] Add page number to toolbar

---
 .../document-detail.component.html              | 12 +++++++++++-
 .../document-detail.component.ts                | 17 ++++++++++++++---
 2 files changed, 25 insertions(+), 4 deletions(-)

diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html
index a3bc7e1e6..0fec2aa44 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.html
+++ b/src-ui/src/app/components/document-detail/document-detail.component.html
@@ -1,4 +1,14 @@
 <app-page-header [(title)]="title">
+    <div class="input-group input-group-sm mr-5">
+      <div class="input-group-prepend">
+        <div class="input-group-text">Page </div>
+      </div>
+      <input class="form-control flex-grow-0 w-auto" type="number" min="1" [max]="previewNumPages" [(ngModel)]="currentPreviewPage" />
+      <div class="input-group-append">
+        <div class="input-group-text">of {{previewNumPages}}</div>
+      </div>
+    </div>
+
     <button type="button" class="btn btn-sm btn-outline-danger mr-2" (click)="delete()">
         <svg class="buttonicon" fill="currentColor">
             <use xlink:href="assets/bootstrap-icons.svg#trash" />
@@ -128,7 +138,7 @@
 
     <div class="col-md-6 col-xl-8 mb-3">
       <div class="pdf-viewer-container" *ngIf="getContentType() == 'application/pdf'">
-        <pdf-viewer [src]="previewUrl" [original-size]="false" [show-borders]="true"></pdf-viewer>
+        <pdf-viewer [src]="previewUrl" [original-size]="false" [show-borders]="true" [show-all]="true" [(page)]="currentPreviewPage" (after-load-complete)="pdfPreviewLoaded($event)"></pdf-viewer>
       </div>
     </div>
 </div>
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts
index b4005b920..a2f80f786 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.ts
+++ b/src-ui/src/app/components/document-detail/document-detail.component.ts
@@ -15,6 +15,7 @@ import { DocumentService } from 'src/app/services/rest/document.service';
 import { ConfirmDialogComponent } from '../common/confirm-dialog/confirm-dialog.component';
 import { CorrespondentEditDialogComponent } from '../manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component';
 import { DocumentTypeEditDialogComponent } from '../manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component';
+import { PDFDocumentProxy } from 'ng2-pdf-viewer';
 
 @Component({
   selector: 'app-document-detail',
@@ -47,8 +48,11 @@ export class DocumentDetailComponent implements OnInit {
     tags: new FormControl([])
   })
 
+  currentPreviewPage: number = 1
+  previewNumPages: number
+
   constructor(
-    private documentsService: DocumentService, 
+    private documentsService: DocumentService,
     private route: ActivatedRoute,
     private correspondentService: CorrespondentService,
     private documentTypeService: DocumentTypeService,
@@ -113,6 +117,8 @@ export class DocumentDetailComponent implements OnInit {
     modal.componentInstance.success.subscribe(newCorrespondent => {
       this.correspondentService.listAll().subscribe(correspondents => {
         this.correspondents = correspondents.results
+        console.log(this.documentForm.get('correspondent'), this.documentForm.get('correspondent').setValue);
+
         this.documentForm.get('correspondent').setValue(newCorrespondent.id)
       })
     })
@@ -126,7 +132,7 @@ export class DocumentDetailComponent implements OnInit {
     }, error => {this.router.navigate(['404'])})
   }
 
-  save() {    
+  save() {
     this.documentsService.update(this.document).subscribe(result => {
       this.close()
     })
@@ -161,7 +167,7 @@ export class DocumentDetailComponent implements OnInit {
     modal.componentInstance.btnCaption = "Delete document"
     modal.componentInstance.confirmClicked.subscribe(() => {
       this.documentsService.delete(this.document).subscribe(() => {
-        modal.close()  
+        modal.close()
         this.close()
       })
     })
@@ -171,4 +177,9 @@ export class DocumentDetailComponent implements OnInit {
   hasNext() {
     return this.documentListViewService.hasNext(this.documentId)
   }
+
+  pdfPreviewLoaded(pdf: PDFDocumentProxy) {
+    this.previewNumPages = pdf.numPages
+  }
+
 }

From f214fe1b3eb5b8b319d7fabeeb847d7a96cddb50 Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Fri, 18 Dec 2020 14:44:17 -0800
Subject: [PATCH 38/78] Log line

---
 .../app/components/document-detail/document-detail.component.ts | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts
index a2f80f786..aa3d4e5b8 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.ts
+++ b/src-ui/src/app/components/document-detail/document-detail.component.ts
@@ -117,8 +117,6 @@ export class DocumentDetailComponent implements OnInit {
     modal.componentInstance.success.subscribe(newCorrespondent => {
       this.correspondentService.listAll().subscribe(correspondents => {
         this.correspondents = correspondents.results
-        console.log(this.documentForm.get('correspondent'), this.documentForm.get('correspondent').setValue);
-
         this.documentForm.get('correspondent').setValue(newCorrespondent.id)
       })
     })

From 2d841e71673559e8853e4f853ffec96e004f8e0a Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Fri, 18 Dec 2020 14:47:06 -0800
Subject: [PATCH 39/78] Refactor

---
 .../components/document-detail/document-detail.component.html | 4 ++--
 .../components/document-detail/document-detail.component.ts   | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html
index 0fec2aa44..e5dde2ad0 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.html
+++ b/src-ui/src/app/components/document-detail/document-detail.component.html
@@ -3,7 +3,7 @@
       <div class="input-group-prepend">
         <div class="input-group-text">Page </div>
       </div>
-      <input class="form-control flex-grow-0 w-auto" type="number" min="1" [max]="previewNumPages" [(ngModel)]="currentPreviewPage" />
+      <input class="form-control flex-grow-0 w-auto" type="number" min="1" [max]="previewNumPages" [(ngModel)]="previewCurrentPage" />
       <div class="input-group-append">
         <div class="input-group-text">of {{previewNumPages}}</div>
       </div>
@@ -138,7 +138,7 @@
 
     <div class="col-md-6 col-xl-8 mb-3">
       <div class="pdf-viewer-container" *ngIf="getContentType() == 'application/pdf'">
-        <pdf-viewer [src]="previewUrl" [original-size]="false" [show-borders]="true" [show-all]="true" [(page)]="currentPreviewPage" (after-load-complete)="pdfPreviewLoaded($event)"></pdf-viewer>
+        <pdf-viewer [src]="previewUrl" [original-size]="false" [show-borders]="true" [show-all]="true" [(page)]="previewCurrentPage" (after-load-complete)="pdfPreviewLoaded($event)"></pdf-viewer>
       </div>
     </div>
 </div>
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts
index aa3d4e5b8..2b839e969 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.ts
+++ b/src-ui/src/app/components/document-detail/document-detail.component.ts
@@ -48,7 +48,7 @@ export class DocumentDetailComponent implements OnInit {
     tags: new FormControl([])
   })
 
-  currentPreviewPage: number = 1
+  previewCurrentPage: number = 1
   previewNumPages: number
 
   constructor(

From e0293db16dde23829e77717a7e7a4e7432b4a65f Mon Sep 17 00:00:00 2001
From: Michael Shamoon <4887959+nikonratm@users.noreply.github.com>
Date: Fri, 18 Dec 2020 15:04:52 -0800
Subject: [PATCH 40/78] Only show page numbers when content type is
 application/pdf

---
 .../components/document-detail/document-detail.component.html   | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html
index e5dde2ad0..d47c07de1 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.html
+++ b/src-ui/src/app/components/document-detail/document-detail.component.html
@@ -1,5 +1,5 @@
 <app-page-header [(title)]="title">
-    <div class="input-group input-group-sm mr-5">
+    <div class="input-group input-group-sm mr-5" *ngIf="getContentType() == 'application/pdf'">
       <div class="input-group-prepend">
         <div class="input-group-text">Page </div>
       </div>

From f184e6b16260f38709c0cc8bb65c4a8b64ab27d2 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sat, 19 Dec 2020 01:15:40 +0100
Subject: [PATCH 41/78] fix some layout issues

---
 .../app/components/document-detail/document-detail.component.ts | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src-ui/src/app/components/document-detail/document-detail.component.ts b/src-ui/src/app/components/document-detail/document-detail.component.ts
index 2b839e969..75a64f548 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.ts
+++ b/src-ui/src/app/components/document-detail/document-detail.component.ts
@@ -49,7 +49,7 @@ export class DocumentDetailComponent implements OnInit {
   })
 
   previewCurrentPage: number = 1
-  previewNumPages: number
+  previewNumPages: number = 1
 
   constructor(
     private documentsService: DocumentService,

From 37237dfcf68221a919aa808625aee368e2915b6b Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sat, 19 Dec 2020 12:46:11 +0100
Subject: [PATCH 42/78] default title

---
 src/documents/templates/index.html | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/documents/templates/index.html b/src/documents/templates/index.html
index 06dbb678e..d086be0fe 100644
--- a/src/documents/templates/index.html
+++ b/src/documents/templates/index.html
@@ -5,7 +5,7 @@
 <html lang="en">
 <head>
   <meta charset="utf-8">
-  <title>PaperlessUi</title>
+  <title>Paperless-ng</title>
   <base href="/">
   <meta name="viewport" content="width=device-width, initial-scale=1">
 	<meta name="cookie_prefix" content="{{cookie_prefix}}">

From 57a5a4147bd168a1ef2e5b7e614f43499b312fee Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sat, 19 Dec 2020 14:48:42 +0100
Subject: [PATCH 43/78] test case

---
 src/documents/tests/test_file_handling.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/src/documents/tests/test_file_handling.py b/src/documents/tests/test_file_handling.py
index 2e60065f1..b24f52aa2 100644
--- a/src/documents/tests/test_file_handling.py
+++ b/src/documents/tests/test_file_handling.py
@@ -14,7 +14,7 @@
 from .utils import DirectoriesMixin
 from ..file_handling import generate_filename, create_source_path_directory, delete_empty_directories, \
     generate_unique_filename
-from ..models import Document, Correspondent, Tag
+from ..models import Document, Correspondent, Tag, DocumentType
 
 
 class TestFileHandling(DirectoriesMixin, TestCase):
@@ -190,6 +190,17 @@ def test_directory_not_empty(self):
         self.assertEqual(os.path.isdir(settings.ORIGINALS_DIR + "/none"), True)
         self.assertTrue(os.path.isfile(important_file))
 
+    @override_settings(PAPERLESS_FILENAME_FORMAT="{document_type} - {title}")
+    def test_document_type(self):
+        dt = DocumentType.objects.create(name="my_doc_type")
+        d = Document.objects.create(title="the_doc", mime_type="application/pdf")
+
+        self.assertEqual(generate_filename(d), "none - the_doc.pdf")
+
+        d.document_type = dt
+
+        self.assertEqual(generate_filename(d), "my_doc_type - the_doc.pdf")
+
     @override_settings(PAPERLESS_FILENAME_FORMAT="{tags[type]}")
     def test_tags_with_underscore(self):
         document = Document()

From 1b1b57eb6a5c77ceb3128c44f75ea17ac1865586 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sat, 19 Dec 2020 15:16:42 +0100
Subject: [PATCH 44/78] more tests

---
 src/paperless/checks.py                      | 16 +++---
 src/paperless/tests/test_checks.py           | 55 ++++++++++++++++++++
 src/paperless_tesseract/checks.py            |  2 +-
 src/paperless_tesseract/tests/test_checks.py | 26 +++++++++
 4 files changed, 90 insertions(+), 9 deletions(-)
 create mode 100644 src/paperless/tests/test_checks.py
 create mode 100644 src/paperless_tesseract/tests/test_checks.py

diff --git a/src/paperless/checks.py b/src/paperless/checks.py
index 819582ffc..3e26ae69f 100644
--- a/src/paperless/checks.py
+++ b/src/paperless/checks.py
@@ -13,18 +13,18 @@
 )
 
 
-def path_check(env_var):
+def path_check(var, directory):
     messages = []
-    directory = os.getenv(env_var)
+    print(directory)
     if directory:
         if not os.path.exists(directory):
             messages.append(Error(
-                exists_message.format(env_var),
+                exists_message.format(var),
                 exists_hint.format(directory)
             ))
         elif not os.access(directory, os.W_OK | os.X_OK):
             messages.append(Error(
-                writeable_message.format(env_var),
+                writeable_message.format(var),
                 writeable_hint.format(directory)
             ))
     return messages
@@ -36,10 +36,10 @@ def paths_check(app_configs, **kwargs):
     Check the various paths for existence, readability and writeability
     """
 
-    check_messages = path_check("PAPERLESS_DATA_DIR") + \
-        path_check("PAPERLESS_MEDIA_ROOT") + \
-        path_check("PAPERLESS_CONSUMPTION_DIR") + \
-        path_check("PAPERLESS_STATICDIR")
+    check_messages = path_check("PAPERLESS_DATA_DIR", settings.DATA_DIR) + \
+        path_check("PAPERLESS_MEDIA_ROOT", settings.MEDIA_ROOT) + \
+        path_check("PAPERLESS_CONSUMPTION_DIR", settings.CONSUMPTION_DIR) + \
+        path_check("PAPERLESS_STATICDIR", settings.STATIC_ROOT)
 
     return check_messages
 
diff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.py
new file mode 100644
index 000000000..61cc05bf0
--- /dev/null
+++ b/src/paperless/tests/test_checks.py
@@ -0,0 +1,55 @@
+import os
+import shutil
+
+from django.test import TestCase, override_settings
+
+from documents.tests.utils import DirectoriesMixin
+from paperless import binaries_check, paths_check
+from paperless.checks import debug_mode_check
+
+
+class TestChecks(DirectoriesMixin, TestCase):
+
+    def test_binaries(self):
+        self.assertEqual(binaries_check(None), [])
+
+    @override_settings(CONVERT_BINARY="uuuhh", OPTIPNG_BINARY="forgot")
+    def test_binaries_fail(self):
+        self.assertEqual(len(binaries_check(None)), 2)
+
+    def test_paths_check(self):
+        self.assertEqual(paths_check(None), [])
+
+    @override_settings(MEDIA_ROOT="uuh",
+                       STATIC_ROOT="somewhere",
+                       DATA_DIR="whatever",
+                       CONSUMPTION_DIR="idontcare")
+    def test_paths_check_dont_exist(self):
+        msgs = paths_check(None)
+        self.assertEqual(len(msgs), 4)
+
+        for msg in msgs:
+            self.assertTrue(msg.msg.endswith("is set but doesn't exist."))
+
+    def test_paths_check_no_access(self):
+        os.chmod(self.dirs.data_dir, 0o000)
+        os.chmod(self.dirs.media_dir, 0o000)
+        os.chmod(self.dirs.consumption_dir, 0o000)
+
+        self.addCleanup(os.chmod, self.dirs.data_dir, 0o777)
+        self.addCleanup(os.chmod, self.dirs.media_dir, 0o777)
+        self.addCleanup(os.chmod, self.dirs.consumption_dir, 0o777)
+
+        msgs = paths_check(None)
+        self.assertEqual(len(msgs), 3)
+
+        for msg in msgs:
+            self.assertTrue(msg.msg.endswith("is not writeable"))
+
+    @override_settings(DEBUG=False)
+    def test_debug_disabled(self):
+        self.assertEqual(debug_mode_check(None), [])
+
+    @override_settings(DEBUG=True)
+    def test_debug_enabled(self):
+        self.assertEqual(len(debug_mode_check(None)), 1)
diff --git a/src/paperless_tesseract/checks.py b/src/paperless_tesseract/checks.py
index 41ea3c9b5..d58b7ac6d 100644
--- a/src/paperless_tesseract/checks.py
+++ b/src/paperless_tesseract/checks.py
@@ -1,7 +1,7 @@
 import subprocess
 
 from django.conf import settings
-from django.core.checks import Error, register
+from django.core.checks import Error, Warning, register
 
 
 def get_tesseract_langs():
diff --git a/src/paperless_tesseract/tests/test_checks.py b/src/paperless_tesseract/tests/test_checks.py
new file mode 100644
index 000000000..c4f15764e
--- /dev/null
+++ b/src/paperless_tesseract/tests/test_checks.py
@@ -0,0 +1,26 @@
+from unittest import mock
+
+from django.core.checks import ERROR
+from django.test import TestCase, override_settings
+
+from paperless_tesseract import check_default_language_available
+
+
+class TestChecks(TestCase):
+
+    def test_default_language(self):
+        msgs = check_default_language_available(None)
+
+    @override_settings(OCR_LANGUAGE="")
+    def test_no_language(self):
+        msgs = check_default_language_available(None)
+        self.assertEqual(len(msgs), 1)
+        self.assertTrue(msgs[0].msg.startswith("No OCR language has been specified with PAPERLESS_OCR_LANGUAGE"))
+
+    @override_settings(OCR_LANGUAGE="ita")
+    @mock.patch("paperless_tesseract.checks.get_tesseract_langs")
+    def test_invalid_language(self, m):
+        m.return_value = ["deu", "eng"]
+        msgs = check_default_language_available(None)
+        self.assertEqual(len(msgs), 1)
+        self.assertEqual(msgs[0].level, ERROR)

From fad3df1e39037af3ea4d7dcaf8220dae9f97ef93 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sat, 19 Dec 2020 16:46:04 +0100
Subject: [PATCH 45/78] removed x-frame-options, since that was only used for
 the <object> pdf display tag.

---
 src/paperless/settings.py | 7 -------
 1 file changed, 7 deletions(-)

diff --git a/src/paperless/settings.py b/src/paperless/settings.py
index 1a6b80a0c..c6f7c9357 100644
--- a/src/paperless/settings.py
+++ b/src/paperless/settings.py
@@ -160,13 +160,6 @@ def __get_boolean(key, default="NO"):
     MIDDLEWARE.insert(_index+1, 'paperless.auth.AutoLoginMiddleware')
 
 
-if DEBUG:
-    X_FRAME_OPTIONS = ''
-    # this should really be 'allow-from uri' but its not supported in any mayor
-    # browser.
-else:
-    X_FRAME_OPTIONS = 'SAMEORIGIN'
-
 # We allow CORS from localhost:8080
 CORS_ALLOWED_ORIGINS = tuple(os.getenv("PAPERLESS_CORS_ALLOWED_HOSTS", "http://localhost:8000").split(","))
 

From bb814da95b1e5efecdfc3cdafd0a5fb772ed2281 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sat, 19 Dec 2020 16:46:09 +0100
Subject: [PATCH 46/78] more test.

---
 src/documents/tests/test_management.py        | 135 ++++++++++++++++++
 .../tests/test_management_archiver.py         |  40 ------
 .../tests/test_management_decrypt.py          |  57 --------
 3 files changed, 135 insertions(+), 97 deletions(-)
 create mode 100644 src/documents/tests/test_management.py
 delete mode 100644 src/documents/tests/test_management_archiver.py
 delete mode 100644 src/documents/tests/test_management_decrypt.py

diff --git a/src/documents/tests/test_management.py b/src/documents/tests/test_management.py
new file mode 100644
index 000000000..58aaf9342
--- /dev/null
+++ b/src/documents/tests/test_management.py
@@ -0,0 +1,135 @@
+import hashlib
+import tempfile
+import filecmp
+import os
+import shutil
+from pathlib import Path
+from unittest import mock
+
+from django.test import TestCase, override_settings
+
+
+from django.core.management import call_command
+
+from documents.file_handling import generate_filename
+from documents.management.commands.document_archiver import handle_document
+from documents.models import Document
+from documents.tests.utils import DirectoriesMixin
+
+
+sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf")
+
+
+class TestArchiver(DirectoriesMixin, TestCase):
+
+    def make_models(self):
+        return Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf")
+
+    def test_archiver(self):
+
+        doc = self.make_models()
+        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf"))
+
+        call_command('document_archiver')
+
+    def test_handle_document(self):
+
+        doc = self.make_models()
+        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf"))
+
+        handle_document(doc.pk)
+
+        doc = Document.objects.get(id=doc.id)
+
+        self.assertIsNotNone(doc.checksum)
+        self.assertTrue(os.path.isfile(doc.archive_path))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(filecmp.cmp(sample_file, doc.source_path))
+
+
+class TestDecryptDocuments(TestCase):
+
+    @override_settings(
+        ORIGINALS_DIR=os.path.join(os.path.dirname(__file__), "samples", "originals"),
+        THUMBNAIL_DIR=os.path.join(os.path.dirname(__file__), "samples", "thumb"),
+        PASSPHRASE="test",
+        PAPERLESS_FILENAME_FORMAT=None
+    )
+    @mock.patch("documents.management.commands.decrypt_documents.input")
+    def test_decrypt(self, m):
+
+        media_dir = tempfile.mkdtemp()
+        originals_dir = os.path.join(media_dir, "documents", "originals")
+        thumb_dir = os.path.join(media_dir, "documents", "thumbnails")
+        os.makedirs(originals_dir, exist_ok=True)
+        os.makedirs(thumb_dir, exist_ok=True)
+
+        override_settings(
+            ORIGINALS_DIR=originals_dir,
+            THUMBNAIL_DIR=thumb_dir,
+            PASSPHRASE="test"
+        ).enable()
+
+        doc = Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg",  mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
+
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg"))
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000002.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg"))
+
+        call_command('decrypt_documents')
+
+        doc.refresh_from_db()
+
+        self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED)
+        self.assertEqual(doc.filename, "0000002.pdf")
+        self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf")))
+        self.assertTrue(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(os.path.join(thumb_dir, f"{doc.id:07}.png")))
+        self.assertTrue(os.path.isfile(doc.thumbnail_path))
+
+        with doc.source_file as f:
+            checksum = hashlib.md5(f.read()).hexdigest()
+            self.assertEqual(checksum, doc.checksum)
+
+
+class TestMakeIndex(TestCase):
+
+    @mock.patch("documents.management.commands.document_index.index_reindex")
+    def test_reindex(self, m):
+        call_command("document_index", "reindex")
+        m.assert_called_once()
+
+    @mock.patch("documents.management.commands.document_index.index_optimize")
+    def test_optimize(self, m):
+        call_command("document_index", "optimize")
+        m.assert_called_once()
+
+
+class TestRenamer(DirectoriesMixin, TestCase):
+
+    def test_rename(self):
+        doc = Document.objects.create(title="test", mime_type="application/pdf")
+        doc.filename = generate_filename(doc)
+        doc.save()
+
+        Path(doc.source_path).touch()
+
+        old_source_path = doc.source_path
+
+        with override_settings(PAPERLESS_FILENAME_FORMAT="{title}"):
+            call_command("document_renamer")
+
+        doc2 = Document.objects.get(id=doc.id)
+
+        self.assertEqual(doc2.filename, "test.pdf")
+        self.assertFalse(os.path.isfile(old_source_path))
+        self.assertFalse(os.path.isfile(doc.source_path))
+        self.assertTrue(os.path.isfile(doc2.source_path))
+
+
+class TestCreateClassifier(TestCase):
+
+    @mock.patch("documents.management.commands.document_create_classifier.train_classifier")
+    def test_create_classifier(self, m):
+        call_command("document_create_classifier")
+
+        m.assert_called_once()
diff --git a/src/documents/tests/test_management_archiver.py b/src/documents/tests/test_management_archiver.py
deleted file mode 100644
index 0828f05ff..000000000
--- a/src/documents/tests/test_management_archiver.py
+++ /dev/null
@@ -1,40 +0,0 @@
-import filecmp
-import os
-import shutil
-
-from django.core.management import call_command
-from django.test import TestCase
-
-from documents.management.commands.document_archiver import handle_document
-from documents.models import Document
-from documents.tests.utils import DirectoriesMixin
-
-
-sample_file = os.path.join(os.path.dirname(__file__), "samples", "simple.pdf")
-
-
-class TestArchiver(DirectoriesMixin, TestCase):
-
-    def make_models(self):
-        return Document.objects.create(checksum="A", title="A", content="first document", mime_type="application/pdf")
-
-    def test_archiver(self):
-
-        doc = self.make_models()
-        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf"))
-
-        call_command('document_archiver')
-
-    def test_handle_document(self):
-
-        doc = self.make_models()
-        shutil.copy(sample_file, os.path.join(self.dirs.originals_dir, f"{doc.id:07}.pdf"))
-
-        handle_document(doc.pk)
-
-        doc = Document.objects.get(id=doc.id)
-
-        self.assertIsNotNone(doc.checksum)
-        self.assertTrue(os.path.isfile(doc.archive_path))
-        self.assertTrue(os.path.isfile(doc.source_path))
-        self.assertTrue(filecmp.cmp(sample_file, doc.source_path))
diff --git a/src/documents/tests/test_management_decrypt.py b/src/documents/tests/test_management_decrypt.py
deleted file mode 100644
index 1d64b1105..000000000
--- a/src/documents/tests/test_management_decrypt.py
+++ /dev/null
@@ -1,57 +0,0 @@
-import hashlib
-import json
-import os
-import shutil
-import tempfile
-from unittest import mock
-
-from django.core.management import call_command
-from django.test import TestCase, override_settings
-
-from documents.management.commands import document_exporter
-from documents.models import Document, Tag, DocumentType, Correspondent
-
-
-class TestDecryptDocuments(TestCase):
-
-    @override_settings(
-        ORIGINALS_DIR=os.path.join(os.path.dirname(__file__), "samples", "originals"),
-        THUMBNAIL_DIR=os.path.join(os.path.dirname(__file__), "samples", "thumb"),
-        PASSPHRASE="test",
-        PAPERLESS_FILENAME_FORMAT=None
-    )
-    @mock.patch("documents.management.commands.decrypt_documents.input")
-    def test_decrypt(self, m):
-
-        media_dir = tempfile.mkdtemp()
-        originals_dir = os.path.join(media_dir, "documents", "originals")
-        thumb_dir = os.path.join(media_dir, "documents", "thumbnails")
-        os.makedirs(originals_dir, exist_ok=True)
-        os.makedirs(thumb_dir, exist_ok=True)
-
-        override_settings(
-            ORIGINALS_DIR=originals_dir,
-            THUMBNAIL_DIR=thumb_dir,
-            PASSPHRASE="test"
-        ).enable()
-
-        doc = Document.objects.create(checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg",  mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
-
-        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), os.path.join(originals_dir, "0000002.pdf.gpg"))
-        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "thumbnails", f"0000002.png.gpg"), os.path.join(thumb_dir, f"{doc.id:07}.png.gpg"))
-
-        call_command('decrypt_documents')
-
-        doc.refresh_from_db()
-
-        self.assertEqual(doc.storage_type, Document.STORAGE_TYPE_UNENCRYPTED)
-        self.assertEqual(doc.filename, "0000002.pdf")
-        self.assertTrue(os.path.isfile(os.path.join(originals_dir, "0000002.pdf")))
-        self.assertTrue(os.path.isfile(doc.source_path))
-        self.assertTrue(os.path.isfile(os.path.join(thumb_dir, f"{doc.id:07}.png")))
-        self.assertTrue(os.path.isfile(doc.thumbnail_path))
-
-        with doc.source_file as f:
-            checksum = hashlib.md5(f.read()).hexdigest()
-            self.assertEqual(checksum, doc.checksum)
-

From e79c45c98d5164de4c99888c1a9cbd7553ab3bcc Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sat, 19 Dec 2020 20:39:56 +0100
Subject: [PATCH 47/78] fix test cases

---
 src/paperless/checks.py            | 7 ++-----
 src/paperless/tests/test_checks.py | 3 +--
 2 files changed, 3 insertions(+), 7 deletions(-)

diff --git a/src/paperless/checks.py b/src/paperless/checks.py
index 3e26ae69f..1e74b30a9 100644
--- a/src/paperless/checks.py
+++ b/src/paperless/checks.py
@@ -36,12 +36,9 @@ def paths_check(app_configs, **kwargs):
     Check the various paths for existence, readability and writeability
     """
 
-    check_messages = path_check("PAPERLESS_DATA_DIR", settings.DATA_DIR) + \
+    return path_check("PAPERLESS_DATA_DIR", settings.DATA_DIR) + \
         path_check("PAPERLESS_MEDIA_ROOT", settings.MEDIA_ROOT) + \
-        path_check("PAPERLESS_CONSUMPTION_DIR", settings.CONSUMPTION_DIR) + \
-        path_check("PAPERLESS_STATICDIR", settings.STATIC_ROOT)
-
-    return check_messages
+        path_check("PAPERLESS_CONSUMPTION_DIR", settings.CONSUMPTION_DIR)
 
 
 @register()
diff --git a/src/paperless/tests/test_checks.py b/src/paperless/tests/test_checks.py
index 61cc05bf0..e1525cab8 100644
--- a/src/paperless/tests/test_checks.py
+++ b/src/paperless/tests/test_checks.py
@@ -21,12 +21,11 @@ def test_paths_check(self):
         self.assertEqual(paths_check(None), [])
 
     @override_settings(MEDIA_ROOT="uuh",
-                       STATIC_ROOT="somewhere",
                        DATA_DIR="whatever",
                        CONSUMPTION_DIR="idontcare")
     def test_paths_check_dont_exist(self):
         msgs = paths_check(None)
-        self.assertEqual(len(msgs), 4)
+        self.assertEqual(len(msgs), 3, str(msgs))
 
         for msg in msgs:
             self.assertTrue(msg.msg.endswith("is set but doesn't exist."))

From 3f94fc2618e5f543c3cbaeed46cd01749a6e8f66 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sat, 19 Dec 2020 21:52:58 +0100
Subject: [PATCH 48/78] fix & test a migration

---
 src/documents/migrations/1003_mime_types.py |   1 +
 src/documents/tests/test_migrations.py      | 129 ++++++++++++++++++++
 2 files changed, 130 insertions(+)
 create mode 100644 src/documents/tests/test_migrations.py

diff --git a/src/documents/migrations/1003_mime_types.py b/src/documents/migrations/1003_mime_types.py
index 78ecced2b..c196f29f4 100644
--- a/src/documents/migrations/1003_mime_types.py
+++ b/src/documents/migrations/1003_mime_types.py
@@ -11,6 +11,7 @@
 STORAGE_TYPE_UNENCRYPTED = "unencrypted"
 STORAGE_TYPE_GPG = "gpg"
 
+
 def source_path(self):
     if self.filename:
         fname = str(self.filename)
diff --git a/src/documents/tests/test_migrations.py b/src/documents/tests/test_migrations.py
new file mode 100644
index 000000000..33ba41444
--- /dev/null
+++ b/src/documents/tests/test_migrations.py
@@ -0,0 +1,129 @@
+import os
+import shutil
+from pathlib import Path
+
+from django.apps import apps
+from django.conf import settings
+from django.db import connection
+from django.db.migrations.executor import MigrationExecutor
+from django.test import TestCase, TransactionTestCase, override_settings
+
+from documents.models import Document
+from documents.parsers import get_default_file_extension
+from documents.tests.utils import DirectoriesMixin
+
+
+class TestMigrations(TransactionTestCase):
+
+    @property
+    def app(self):
+        return apps.get_containing_app_config(type(self).__module__).name
+
+    migrate_from = None
+    migrate_to = None
+
+    def setUp(self):
+        super(TestMigrations, self).setUp()
+
+        assert self.migrate_from and self.migrate_to, \
+            "TestCase '{}' must define migrate_from and migrate_to     properties".format(type(self).__name__)
+        self.migrate_from = [(self.app, self.migrate_from)]
+        self.migrate_to = [(self.app, self.migrate_to)]
+        executor = MigrationExecutor(connection)
+        old_apps = executor.loader.project_state(self.migrate_from).apps
+
+        # Reverse to the original migration
+        executor.migrate(self.migrate_from)
+
+        self.setUpBeforeMigration(old_apps)
+
+        # Run the migration to test
+        executor = MigrationExecutor(connection)
+        executor.loader.build_graph()  # reload.
+        executor.migrate(self.migrate_to)
+
+        self.apps = executor.loader.project_state(self.migrate_to).apps
+
+    def setUpBeforeMigration(self, apps):
+        pass
+
+
+STORAGE_TYPE_UNENCRYPTED = "unencrypted"
+STORAGE_TYPE_GPG = "gpg"
+
+
+def source_path_before(self):
+    if self.filename:
+        fname = str(self.filename)
+    else:
+        fname = "{:07}.{}".format(self.pk, self.file_type)
+        if self.storage_type == STORAGE_TYPE_GPG:
+            fname += ".gpg"
+
+    return os.path.join(
+        settings.ORIGINALS_DIR,
+        fname
+    )
+
+
+def file_type_after(self):
+    return get_default_file_extension(self.mime_type)
+
+
+def source_path_after(doc):
+    if doc.filename:
+        fname = str(doc.filename)
+    else:
+        fname = "{:07}{}".format(doc.pk, file_type_after(doc))
+        if doc.storage_type == STORAGE_TYPE_GPG:
+            fname += ".gpg"  # pragma: no cover
+
+    return os.path.join(
+        settings.ORIGINALS_DIR,
+        fname
+    )
+
+
+@override_settings(PASSPHRASE="test")
+class TestMigrateMimeType(DirectoriesMixin, TestMigrations):
+
+    migrate_from = '1002_auto_20201111_1105'
+    migrate_to = '1003_mime_types'
+
+    def setUpBeforeMigration(self, apps):
+        Document = apps.get_model("documents", "Document")
+        doc = Document.objects.create(title="test", file_type="pdf", filename="file1.pdf")
+        self.doc_id = doc.id
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), source_path_before(doc))
+
+        doc2 = Document.objects.create(checksum="B", file_type="pdf", storage_type=STORAGE_TYPE_GPG)
+        self.doc2_id = doc2.id
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "documents", "originals", "0000002.pdf.gpg"), source_path_before(doc2))
+
+    def testMimeTypesMigrated(self):
+        Document = self.apps.get_model('documents', 'Document')
+
+        doc = Document.objects.get(id=self.doc_id)
+        self.assertEqual(doc.mime_type, "application/pdf")
+
+        doc2 = Document.objects.get(id=self.doc2_id)
+        self.assertEqual(doc2.mime_type, "application/pdf")
+
+
+@override_settings(PASSPHRASE="test")
+class TestMigrateMimeTypeBackwards(DirectoriesMixin, TestMigrations):
+
+    migrate_from = '1003_mime_types'
+    migrate_to = '1002_auto_20201111_1105'
+
+    def setUpBeforeMigration(self, apps):
+        Document = apps.get_model("documents", "Document")
+        doc = Document.objects.create(title="test", mime_type="application/pdf", filename="file1.pdf")
+        self.doc_id = doc.id
+        shutil.copy(os.path.join(os.path.dirname(__file__), "samples", "simple.pdf"), source_path_after(doc))
+
+    def testMimeTypesReverted(self):
+        Document = self.apps.get_model('documents', 'Document')
+
+        doc = Document.objects.get(id=self.doc_id)
+        self.assertEqual(doc.file_type, "pdf")

From 32224f187dd8fa8ec4f6f6ad019c6014a33af0e8 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 20 Dec 2020 00:06:33 +0100
Subject: [PATCH 49/78] test CONSUMER_DELETE_DUPLICATES

---
 src/documents/tests/test_consumer.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py
index b4b19be4c..75d6aa16b 100644
--- a/src/documents/tests/test_consumer.py
+++ b/src/documents/tests/test_consumer.py
@@ -642,3 +642,31 @@ def testClassifyDocument(self, m):
         self.assertEqual(document.document_type, dtype)
         self.assertIn(t1, document.tags.all())
         self.assertNotIn(t2, document.tags.all())
+
+    @override_settings(CONSUMER_DELETE_DUPLICATES=True)
+    def test_delete_duplicate(self):
+        dst = self.get_test_file()
+        self.assertTrue(os.path.isfile(dst))
+        doc = self.consumer.try_consume_file(dst)
+
+        self.assertFalse(os.path.isfile(dst))
+        self.assertIsNotNone(doc)
+
+        dst = self.get_test_file()
+        self.assertTrue(os.path.isfile(dst))
+        self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst)
+        self.assertFalse(os.path.isfile(dst))
+
+    @override_settings(CONSUMER_DELETE_DUPLICATES=False)
+    def test_no_delete_duplicate(self):
+        dst = self.get_test_file()
+        self.assertTrue(os.path.isfile(dst))
+        doc = self.consumer.try_consume_file(dst)
+
+        self.assertFalse(os.path.isfile(dst))
+        self.assertIsNotNone(doc)
+
+        dst = self.get_test_file()
+        self.assertTrue(os.path.isfile(dst))
+        self.assertRaises(ConsumerError, self.consumer.try_consume_file, dst)
+        self.assertTrue(os.path.isfile(dst))

From 7f9a0204b59239088d2e47aec8d797d12d1a581a Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 20 Dec 2020 00:08:05 +0100
Subject: [PATCH 50/78] removed most of the logic that extracts data from
 filename patterns #156

---
 src/documents/consumer.py            |   7 -
 src/documents/models.py              |  66 ---------
 src/documents/tests/test_consumer.py | 212 +--------------------------
 3 files changed, 4 insertions(+), 281 deletions(-)

diff --git a/src/documents/consumer.py b/src/documents/consumer.py
index e4da51f1d..ab4912a36 100755
--- a/src/documents/consumer.py
+++ b/src/documents/consumer.py
@@ -247,7 +247,6 @@ def _store(self, text, date, mime_type):
 
         with open(self.path, "rb") as f:
             document = Document.objects.create(
-                correspondent=file_info.correspondent,
                 title=(self.override_title or file_info.title)[:127],
                 content=text,
                 mime_type=mime_type,
@@ -257,12 +256,6 @@ def _store(self, text, date, mime_type):
                 storage_type=storage_type
             )
 
-        relevant_tags = set(file_info.tags)
-        if relevant_tags:
-            tag_names = ", ".join([t.name for t in relevant_tags])
-            self.log("debug", "Tagging with {}".format(tag_names))
-            document.tags.add(*relevant_tags)
-
         self.apply_overrides(document)
 
         document.save()
diff --git a/src/documents/models.py b/src/documents/models.py
index 3a6d155ed..168dd8c7b 100755
--- a/src/documents/models.py
+++ b/src/documents/models.py
@@ -357,54 +357,12 @@ class SavedViewFilterRule(models.Model):
 # TODO: why is this in the models file?
 class FileInfo:
 
-    # This epic regex *almost* worked for our needs, so I'm keeping it here for
-    # posterity, in the hopes that we might find a way to make it work one day.
-    ALMOST_REGEX = re.compile(
-        r"^((?P<date>\d\d\d\d\d\d\d\d\d\d\d\d\d\dZ){separator})?"
-        r"((?P<correspondent>{non_separated_word}+){separator})??"
-        r"(?P<title>{non_separated_word}+)"
-        r"({separator}(?P<tags>[a-z,0-9-]+))?"
-        r"\.(?P<extension>[a-zA-Z.-]+)$".format(
-            separator=r"\s+-\s+",
-            non_separated_word=r"([\w,. ]|([^\s]-))"
-        )
-    )
     REGEXES = OrderedDict([
-        ("created-correspondent-title-tags", re.compile(
-            r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - "
-            r"(?P<correspondent>.*) - "
-            r"(?P<title>.*) - "
-            r"(?P<tags>[a-z0-9\-,]*)$",
-            flags=re.IGNORECASE
-        )),
-        ("created-title-tags", re.compile(
-            r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - "
-            r"(?P<title>.*) - "
-            r"(?P<tags>[a-z0-9\-,]*)$",
-            flags=re.IGNORECASE
-        )),
-        ("created-correspondent-title", re.compile(
-            r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - "
-            r"(?P<correspondent>.*) - "
-            r"(?P<title>.*)$",
-            flags=re.IGNORECASE
-        )),
         ("created-title", re.compile(
             r"^(?P<created>\d\d\d\d\d\d\d\d(\d\d\d\d\d\d)?Z) - "
             r"(?P<title>.*)$",
             flags=re.IGNORECASE
         )),
-        ("correspondent-title-tags", re.compile(
-            r"(?P<correspondent>.*) - "
-            r"(?P<title>.*) - "
-            r"(?P<tags>[a-z0-9\-,]*)$",
-            flags=re.IGNORECASE
-        )),
-        ("correspondent-title", re.compile(
-            r"(?P<correspondent>.*) - "
-            r"(?P<title>.*)?$",
-            flags=re.IGNORECASE
-        )),
         ("title", re.compile(
             r"(?P<title>.*)$",
             flags=re.IGNORECASE
@@ -427,23 +385,10 @@ def _get_created(cls, created):
         except ValueError:
             return None
 
-    @classmethod
-    def _get_correspondent(cls, name):
-        if not name:
-            return None
-        return Correspondent.objects.get_or_create(name=name)[0]
-
     @classmethod
     def _get_title(cls, title):
         return title
 
-    @classmethod
-    def _get_tags(cls, tags):
-        r = []
-        for t in tags.split(","):
-            r.append(Tag.objects.get_or_create(name=t)[0])
-        return tuple(r)
-
     @classmethod
     def _mangle_property(cls, properties, name):
         if name in properties:
@@ -453,15 +398,6 @@ def _mangle_property(cls, properties, name):
 
     @classmethod
     def from_filename(cls, filename):
-        """
-        We use a crude naming convention to make handling the correspondent,
-        title, and tags easier:
-          "<date> - <correspondent> - <title> - <tags>"
-          "<correspondent> - <title> - <tags>"
-          "<correspondent> - <title>"
-          "<title>"
-        """
-
         # Mutate filename in-place before parsing its components
         # by applying at most one of the configured transformations.
         for (pattern, repl) in settings.FILENAME_PARSE_TRANSFORMS:
@@ -492,7 +428,5 @@ def from_filename(cls, filename):
             if m:
                 properties = m.groupdict()
                 cls._mangle_property(properties, "created")
-                cls._mangle_property(properties, "correspondent")
                 cls._mangle_property(properties, "title")
-                cls._mangle_property(properties, "tags")
                 return cls(**properties)
diff --git a/src/documents/tests/test_consumer.py b/src/documents/tests/test_consumer.py
index 75d6aa16b..f53981850 100644
--- a/src/documents/tests/test_consumer.py
+++ b/src/documents/tests/test_consumer.py
@@ -29,81 +29,6 @@ def _test_guess_attributes_from_name(self, filename, sender, title, tags):
 
         self.assertEqual(tuple([t.name for t in file_info.tags]), tags, filename)
 
-    def test_guess_attributes_from_name0(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Title.pdf", "Sender", "Title", ())
-
-    def test_guess_attributes_from_name1(self):
-        self._test_guess_attributes_from_name(
-            "Spaced Sender - Title.pdf", "Spaced Sender", "Title", ())
-
-    def test_guess_attributes_from_name2(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Spaced Title.pdf", "Sender", "Spaced Title", ())
-
-    def test_guess_attributes_from_name3(self):
-        self._test_guess_attributes_from_name(
-            "Dashed-Sender - Title.pdf", "Dashed-Sender", "Title", ())
-
-    def test_guess_attributes_from_name4(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Dashed-Title.pdf", "Sender", "Dashed-Title", ())
-
-    def test_guess_attributes_from_name5(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Title - tag1,tag2,tag3.pdf",
-            "Sender",
-            "Title",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name6(self):
-        self._test_guess_attributes_from_name(
-            "Spaced Sender - Title - tag1,tag2,tag3.pdf",
-            "Spaced Sender",
-            "Title",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name7(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Spaced Title - tag1,tag2,tag3.pdf",
-            "Sender",
-            "Spaced Title",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name8(self):
-        self._test_guess_attributes_from_name(
-            "Dashed-Sender - Title - tag1,tag2,tag3.pdf",
-            "Dashed-Sender",
-            "Title",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name9(self):
-        self._test_guess_attributes_from_name(
-            "Sender - Dashed-Title - tag1,tag2,tag3.pdf",
-            "Sender",
-            "Dashed-Title",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name10(self):
-        self._test_guess_attributes_from_name(
-            "Σενδερ - Τιτλε - tag1,tag2,tag3.pdf",
-            "Σενδερ",
-            "Τιτλε",
-            self.TAGS
-        )
-
-    def test_guess_attributes_from_name_when_correspondent_empty(self):
-        self._test_guess_attributes_from_name(
-            ' - weird empty correspondent but should not break.pdf',
-            None,
-            'weird empty correspondent but should not break',
-            ()
-        )
 
     def test_guess_attributes_from_name_when_title_starts_with_dash(self):
         self._test_guess_attributes_from_name(
@@ -121,28 +46,6 @@ def test_guess_attributes_from_name_when_title_ends_with_dash(self):
             ()
         )
 
-    def test_guess_attributes_from_name_when_title_is_empty(self):
-        self._test_guess_attributes_from_name(
-            'weird correspondent but should not break - .pdf',
-            'weird correspondent but should not break',
-            '',
-            ()
-        )
-
-    def test_case_insensitive_tag_creation(self):
-        """
-        Tags should be detected and created as lower case.
-        :return:
-        """
-
-        filename = "Title - Correspondent - tAg1,TAG2.pdf"
-        self.assertEqual(len(FileInfo.from_filename(filename).tags), 2)
-
-        path = "Title - Correspondent - tag1,tag2.pdf"
-        self.assertEqual(len(FileInfo.from_filename(filename).tags), 2)
-
-        self.assertEqual(Tag.objects.all().count(), 2)
-
 
 class TestFieldPermutations(TestCase):
 
@@ -199,69 +102,7 @@ def test_just_title(self):
             filename = template.format(**spec)
             self._test_guessed_attributes(filename, **spec)
 
-    def test_title_and_correspondent(self):
-        template = '{correspondent} - {title}.pdf'
-        for correspondent in self.valid_correspondents:
-            for title in self.valid_titles:
-                spec = dict(correspondent=correspondent, title=title)
-                filename = template.format(**spec)
-                self._test_guessed_attributes(filename, **spec)
-
-    def test_title_and_correspondent_and_tags(self):
-        template = '{correspondent} - {title} - {tags}.pdf'
-        for correspondent in self.valid_correspondents:
-            for title in self.valid_titles:
-                for tags in self.valid_tags:
-                    spec = dict(correspondent=correspondent, title=title,
-                                tags=tags)
-                    filename = template.format(**spec)
-                    self._test_guessed_attributes(filename, **spec)
-
-    def test_created_and_correspondent_and_title_and_tags(self):
-
-        template = (
-            "{created} - "
-            "{correspondent} - "
-            "{title} - "
-            "{tags}.pdf"
-        )
-
-        for created in self.valid_dates:
-            for correspondent in self.valid_correspondents:
-                for title in self.valid_titles:
-                    for tags in self.valid_tags:
-                        spec = {
-                            "created": created,
-                            "correspondent": correspondent,
-                            "title": title,
-                            "tags": tags,
-                        }
-                        self._test_guessed_attributes(
-                            template.format(**spec), **spec)
-
-    def test_created_and_correspondent_and_title(self):
-
-        template = "{created} - {correspondent} - {title}.pdf"
-
-        for created in self.valid_dates:
-            for correspondent in self.valid_correspondents:
-                for title in self.valid_titles:
-
-                    # Skip cases where title looks like a tag as we can't
-                    # accommodate such cases.
-                    if title.lower() == title:
-                        continue
-
-                    spec = {
-                        "created": created,
-                        "correspondent": correspondent,
-                        "title": title
-                    }
-                    self._test_guessed_attributes(
-                        template.format(**spec), **spec)
-
     def test_created_and_title(self):
-
         template = "{created} - {title}.pdf"
 
         for created in self.valid_dates:
@@ -273,21 +114,6 @@ def test_created_and_title(self):
                 self._test_guessed_attributes(
                     template.format(**spec), **spec)
 
-    def test_created_and_title_and_tags(self):
-
-        template = "{created} - {title} - {tags}.pdf"
-
-        for created in self.valid_dates:
-            for title in self.valid_titles:
-                for tags in self.valid_tags:
-                    spec = {
-                        "created": created,
-                        "title": title,
-                        "tags": tags
-                    }
-                    self._test_guessed_attributes(
-                        template.format(**spec), **spec)
-
     def test_invalid_date_format(self):
         info = FileInfo.from_filename("06112017Z - title.pdf")
         self.assertEqual(info.title, "title")
@@ -336,32 +162,6 @@ def test_filename_parse_transforms(self):
             info = FileInfo.from_filename(filename)
             self.assertEqual(info.title, "anotherall")
 
-        # Complex transformation without date in replacement string
-        with self.settings(
-                FILENAME_PARSE_TRANSFORMS=[(exact_patt, repl1)]):
-            info = FileInfo.from_filename(filename)
-            self.assertEqual(info.title, "0001")
-            self.assertEqual(len(info.tags), 2)
-            self.assertEqual(info.tags[0].name, "tag1")
-            self.assertEqual(info.tags[1].name, "tag2")
-            self.assertIsNone(info.created)
-
-        # Complex transformation with date in replacement string
-        with self.settings(
-            FILENAME_PARSE_TRANSFORMS=[
-                (none_patt, "none.gif"),
-                (exact_patt, repl2),    # <-- matches
-                (exact_patt, repl1),
-                (all_patt, "all.gif")]):
-            info = FileInfo.from_filename(filename)
-            self.assertEqual(info.title, "0001")
-            self.assertEqual(len(info.tags), 2)
-            self.assertEqual(info.tags[0].name, "tag1")
-            self.assertEqual(info.tags[1].name, "tag2")
-            self.assertEqual(info.created.year, 2019)
-            self.assertEqual(info.created.month, 9)
-            self.assertEqual(info.created.day, 8)
-
 
 class DummyParser(DocumentParser):
 
@@ -476,15 +276,13 @@ def testNormalOperation(self):
 
     def testOverrideFilename(self):
         filename = self.get_test_file()
-        override_filename = "My Bank - Statement for November.pdf"
+        override_filename = "Statement for November.pdf"
 
         document = self.consumer.try_consume_file(filename, override_filename=override_filename)
 
-        self.assertEqual(document.correspondent.name, "My Bank")
         self.assertEqual(document.title, "Statement for November")
 
     def testOverrideTitle(self):
-
         document = self.consumer.try_consume_file(self.get_test_file(), override_title="Override Title")
         self.assertEqual(document.title, "Override Title")
 
@@ -594,11 +392,10 @@ def testPostSaveError(self, m):
     def testFilenameHandling(self):
         filename = self.get_test_file()
 
-        document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs")
+        document = self.consumer.try_consume_file(filename, override_title="new docs")
 
         self.assertEqual(document.title, "new docs")
-        self.assertEqual(document.correspondent.name, "Bank")
-        self.assertEqual(document.filename, "Bank/new docs.pdf")
+        self.assertEqual(document.filename, "none/new docs.pdf")
 
     @override_settings(PAPERLESS_FILENAME_FORMAT="{correspondent}/{title}")
     @mock.patch("documents.signals.handlers.generate_unique_filename")
@@ -617,10 +414,9 @@ def get_filename():
 
         Tag.objects.create(name="test", is_inbox_tag=True)
 
-        document = self.consumer.try_consume_file(filename, override_filename="Bank - Test.pdf", override_title="new docs")
+        document = self.consumer.try_consume_file(filename, override_title="new docs")
 
         self.assertEqual(document.title, "new docs")
-        self.assertEqual(document.correspondent.name, "Bank")
         self.assertIsNotNone(os.path.isfile(document.title))
         self.assertTrue(os.path.isfile(document.source_path))
 

From ee31fdc650c6f9df91ec2331b24c289cb03b466b Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 20 Dec 2020 13:59:25 +0100
Subject: [PATCH 51/78] removed unused code

---
 src/paperless_tesseract/languages.py | 194 ---------------------------
 src/paperless_text/parsers.py        |  12 --
 2 files changed, 206 deletions(-)
 delete mode 100644 src/paperless_tesseract/languages.py

diff --git a/src/paperless_tesseract/languages.py b/src/paperless_tesseract/languages.py
deleted file mode 100644
index 5ea560654..000000000
--- a/src/paperless_tesseract/languages.py
+++ /dev/null
@@ -1,194 +0,0 @@
-# Thanks to the Library of Congress and some creative use of sed and awk:
-# http://www.loc.gov/standards/iso639-2/php/English_list.php
-
-ISO639 = {
-
-    "aa": "aar",
-    "ab": "abk",
-    "ae": "ave",
-    "af": "afr",
-    "ak": "aka",
-    "am": "amh",
-    "an": "arg",
-    "ar": "ara",
-    "as": "asm",
-    "av": "ava",
-    "ay": "aym",
-    "az": "aze",
-    "ba": "bak",
-    "be": "bel",
-    "bg": "bul",
-    "bh": "bih",
-    "bi": "bis",
-    "bm": "bam",
-    "bn": "ben",
-    "bo": "bod",
-    "br": "bre",
-    "bs": "bos",
-    "ca": "cat",
-    "ce": "che",
-    "ch": "cha",
-    "co": "cos",
-    "cr": "cre",
-    "cs": "ces",
-    "cu": "chu",
-    "cv": "chv",
-    "cy": "cym",
-    "da": "dan",
-    "de": "deu",
-    "dv": "div",
-    "dz": "dzo",
-    "ee": "ewe",
-    "el": "ell",
-    "en": "eng",
-    "eo": "epo",
-    "es": "spa",
-    "et": "est",
-    "eu": "eus",
-    "fa": "fas",
-    "ff": "ful",
-    "fi": "fin",
-    "fj": "fij",
-    "fo": "fao",
-    "fr": "fra",
-    "fy": "fry",
-    "ga": "gle",
-    "gd": "gla",
-    "gl": "glg",
-    "gn": "grn",
-    "gu": "guj",
-    "gv": "glv",
-    "ha": "hau",
-    "he": "heb",
-    "hi": "hin",
-    "ho": "hmo",
-    "hr": "hrv",
-    "ht": "hat",
-    "hu": "hun",
-    "hy": "hye",
-    "hz": "her",
-    "ia": "ina",
-    "id": "ind",
-    "ie": "ile",
-    "ig": "ibo",
-    "ii": "iii",
-    "ik": "ipk",
-    "io": "ido",
-    "is": "isl",
-    "it": "ita",
-    "iu": "iku",
-    "ja": "jpn",
-    "jv": "jav",
-    "ka": "kat",
-    "kg": "kon",
-    "ki": "kik",
-    "kj": "kua",
-    "kk": "kaz",
-    "kl": "kal",
-    "km": "khm",
-    "kn": "kan",
-    "ko": "kor",
-    "kr": "kau",
-    "ks": "kas",
-    "ku": "kur",
-    "kv": "kom",
-    "kw": "cor",
-    "ky": "kir",
-    "la": "lat",
-    "lb": "ltz",
-    "lg": "lug",
-    "li": "lim",
-    "ln": "lin",
-    "lo": "lao",
-    "lt": "lit",
-    "lu": "lub",
-    "lv": "lav",
-    "mg": "mlg",
-    "mh": "mah",
-    "mi": "mri",
-    "mk": "mkd",
-    "ml": "mal",
-    "mn": "mon",
-    "mr": "mar",
-    "ms": "msa",
-    "mt": "mlt",
-    "my": "mya",
-    "na": "nau",
-    "nb": "nob",
-    "nd": "nde",
-    "ne": "nep",
-    "ng": "ndo",
-    "nl": "nld",
-    "no": "nor",
-    "nr": "nbl",
-    "nv": "nav",
-    "ny": "nya",
-    "oc": "oci",
-    "oj": "oji",
-    "om": "orm",
-    "or": "ori",
-    "os": "oss",
-    "pa": "pan",
-    "pi": "pli",
-    "pl": "pol",
-    "ps": "pus",
-    "pt": "por",
-    "qu": "que",
-    "rm": "roh",
-    "rn": "run",
-    "ro": "ron",
-    "ru": "rus",
-    "rw": "kin",
-    "sa": "san",
-    "sc": "srd",
-    "sd": "snd",
-    "se": "sme",
-    "sg": "sag",
-    "si": "sin",
-    "sk": "slk",
-    "sl": "slv",
-    "sm": "smo",
-    "sn": "sna",
-    "so": "som",
-    "sq": "sqi",
-    "sr": "srp",
-    "ss": "ssw",
-    "st": "sot",
-    "su": "sun",
-    "sv": "swe",
-    "sw": "swa",
-    "ta": "tam",
-    "te": "tel",
-    "tg": "tgk",
-    "th": "tha",
-    "ti": "tir",
-    "tk": "tuk",
-    "tl": "tgl",
-    "tn": "tsn",
-    "to": "ton",
-    "tr": "tur",
-    "ts": "tso",
-    "tt": "tat",
-    "tw": "twi",
-    "ty": "tah",
-    "ug": "uig",
-    "uk": "ukr",
-    "ur": "urd",
-    "uz": "uzb",
-    "ve": "ven",
-    "vi": "vie",
-    "vo": "vol",
-    "wa": "wln",
-    "wo": "wol",
-    "xh": "xho",
-    "yi": "yid",
-    "yo": "yor",
-    "za": "zha",
-
-    # Tessdata contains two values for Chinese, "chi_sim" and "chi_tra".  I
-    # have no idea which one is better, so I just picked the bigger file.
-    "zh": "chi_tra",
-
-    "zu": "zul"
-
-}
diff --git a/src/paperless_text/parsers.py b/src/paperless_text/parsers.py
index 7e488ca37..030c2c2c2 100644
--- a/src/paperless_text/parsers.py
+++ b/src/paperless_text/parsers.py
@@ -35,15 +35,3 @@ def read_text():
     def parse(self, document_path, mime_type):
         with open(document_path, 'r') as f:
             self.text = f.read()
-
-
-def run_command(*args):
-    environment = os.environ.copy()
-    if settings.CONVERT_MEMORY_LIMIT:
-        environment["MAGICK_MEMORY_LIMIT"] = settings.CONVERT_MEMORY_LIMIT
-    if settings.CONVERT_TMPDIR:
-        environment["MAGICK_TMPDIR"] = settings.CONVERT_TMPDIR
-
-    if not subprocess.Popen(' '.join(args), env=environment,
-                            shell=True).wait() == 0:
-        raise ParseError("Convert failed at {}".format(args))

From 01c2fe508e084e9b310ba7693a094f249fa78dbe Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 20 Dec 2020 14:39:17 +0100
Subject: [PATCH 52/78] more tests

---
 src/documents/tests/test_admin.py | 57 +++++++++++++++++++++++++++++++
 1 file changed, 57 insertions(+)
 create mode 100644 src/documents/tests/test_admin.py

diff --git a/src/documents/tests/test_admin.py b/src/documents/tests/test_admin.py
new file mode 100644
index 000000000..b280c43ea
--- /dev/null
+++ b/src/documents/tests/test_admin.py
@@ -0,0 +1,57 @@
+from unittest import mock
+
+from django.contrib.admin.sites import AdminSite
+from django.test import TestCase
+from django.utils import timezone
+
+from documents.admin import DocumentAdmin
+from documents.models import Document, Tag
+
+
+class TestDocumentAdmin(TestCase):
+
+    def setUp(self) -> None:
+        self.doc_admin = DocumentAdmin(model=Document, admin_site=AdminSite())
+
+    @mock.patch("documents.admin.index.add_or_update_document")
+    def test_save_model(self, m):
+        doc = Document.objects.create(title="test")
+        doc.title = "new title"
+        self.doc_admin.save_model(None, doc, None, None)
+        self.assertEqual(Document.objects.get(id=doc.id).title, "new title")
+        m.assert_called_once()
+
+    def test_tags(self):
+        doc = Document.objects.create(title="test")
+        doc.tags.create(name="t1")
+        doc.tags.create(name="t2")
+
+        self.assertEqual(self.doc_admin.tags_(doc), "<span >t1, </span><span >t2, </span>")
+
+    def test_tags_empty(self):
+        doc = Document.objects.create(title="test")
+
+        self.assertEqual(self.doc_admin.tags_(doc), "")
+
+    @mock.patch("documents.admin.index.remove_document")
+    def test_delete_model(self, m):
+        doc = Document.objects.create(title="test")
+        self.doc_admin.delete_model(None, doc)
+        self.assertRaises(Document.DoesNotExist, Document.objects.get, id=doc.id)
+        m.assert_called_once()
+
+    @mock.patch("documents.admin.index.remove_document")
+    def test_delete_queryset(self, m):
+        for i in range(42):
+            Document.objects.create(title="Many documents with the same title", checksum=f"{i:02}")
+
+        self.assertEqual(Document.objects.count(), 42)
+
+        self.doc_admin.delete_queryset(None, Document.objects.all())
+
+        self.assertEqual(m.call_count, 42)
+        self.assertEqual(Document.objects.count(), 0)
+
+    def test_created(self):
+        doc = Document.objects.create(title="test", created=timezone.datetime(2020, 4, 12))
+        self.assertEqual(self.doc_admin.created_(doc), "2020-04-12")

From b10e7abbe8f42af1ea241dc8adcdd2cfb58b342a Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 20 Dec 2020 15:25:47 +0100
Subject: [PATCH 53/78] don't know how that got in here

---
 src/paperless/checks.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/src/paperless/checks.py b/src/paperless/checks.py
index 1e74b30a9..1329ad679 100644
--- a/src/paperless/checks.py
+++ b/src/paperless/checks.py
@@ -15,7 +15,6 @@
 
 def path_check(var, directory):
     messages = []
-    print(directory)
     if directory:
         if not os.path.exists(directory):
             messages.append(Error(

From 240d5b9da214f5d56fc1b4e3229ceb3578e76ab1 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 20 Dec 2020 15:59:37 +0100
Subject: [PATCH 54/78] reorganized docker build.

---
 docker-compose.env.example | 34 ++++++++++++++++++++++++++++++++++
 1 file changed, 34 insertions(+)
 create mode 100644 docker-compose.env.example

diff --git a/docker-compose.env.example b/docker-compose.env.example
new file mode 100644
index 000000000..4271bce6e
--- /dev/null
+++ b/docker-compose.env.example
@@ -0,0 +1,34 @@
+# The UID and GID of the user used to run paperless in the container. Set this
+# to your UID and GID on the host so that you have write access to the
+# consumption directory.
+#USERMAP_UID=1000
+#USERMAP_GID=1000
+
+# Additional languages to install for text recognition, separated by a
+# whitespace. Note that this is
+# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the
+# default language used when guessing the language from the OCR output.
+# The container installs English, German, Italian, Spanish and French by
+# default.
+# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster
+# for available languages.
+#PAPERLESS_OCR_LANGUAGES=tur ces
+
+###############################################################################
+# Paperless-specific settings                                                 #
+###############################################################################
+
+# All settings defined in the paperless.conf.example can be used here. The
+# Docker setup does not use the configuration file.
+# A few commonly adjusted settings are provided below.
+
+# Adjust this key if you plan to make paperless available publicly. It should
+# be a very long sequence of random characters. You don't need to remember it.
+#PAPERLESS_SECRET_KEY=change-me
+
+# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC.
+#PAPERLESS_TIME_ZONE=America/Los_Angeles
+
+# The default language to use for OCR. Set this to the language most of your
+# documents are written in.
+#PAPERLESS_OCR_LANGUAGE=eng

From c6b9e2b5447a4c4021ec48ac8ea6ac590eb20b7e Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 20 Dec 2020 16:00:11 +0100
Subject: [PATCH 55/78] revert last commit

---
 docker-compose.env.example | 34 ----------------------------------
 1 file changed, 34 deletions(-)
 delete mode 100644 docker-compose.env.example

diff --git a/docker-compose.env.example b/docker-compose.env.example
deleted file mode 100644
index 4271bce6e..000000000
--- a/docker-compose.env.example
+++ /dev/null
@@ -1,34 +0,0 @@
-# The UID and GID of the user used to run paperless in the container. Set this
-# to your UID and GID on the host so that you have write access to the
-# consumption directory.
-#USERMAP_UID=1000
-#USERMAP_GID=1000
-
-# Additional languages to install for text recognition, separated by a
-# whitespace. Note that this is
-# different from PAPERLESS_OCR_LANGUAGE (default=eng), which defines the
-# default language used when guessing the language from the OCR output.
-# The container installs English, German, Italian, Spanish and French by
-# default.
-# See https://packages.debian.org/search?keywords=tesseract-ocr-&searchon=names&suite=buster
-# for available languages.
-#PAPERLESS_OCR_LANGUAGES=tur ces
-
-###############################################################################
-# Paperless-specific settings                                                 #
-###############################################################################
-
-# All settings defined in the paperless.conf.example can be used here. The
-# Docker setup does not use the configuration file.
-# A few commonly adjusted settings are provided below.
-
-# Adjust this key if you plan to make paperless available publicly. It should
-# be a very long sequence of random characters. You don't need to remember it.
-#PAPERLESS_SECRET_KEY=change-me
-
-# Use this variable to set a timezone for the Paperless Docker containers. If not specified, defaults to UTC.
-#PAPERLESS_TIME_ZONE=America/Los_Angeles
-
-# The default language to use for OCR. Set this to the language most of your
-# documents are written in.
-#PAPERLESS_OCR_LANGUAGE=eng

From 665863e3953a3696f9d87d4585847e5841588f84 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Sun, 20 Dec 2020 17:18:23 +0100
Subject: [PATCH 56/78] Display name of current user on the dashboard

---
 .../dashboard/dashboard.component.html        |  2 +-
 .../dashboard/dashboard.component.ts          | 24 ++++++++++++++++++-
 src/documents/templates/index.html            |  2 ++
 src/documents/views.py                        |  2 ++
 4 files changed, 28 insertions(+), 2 deletions(-)

diff --git a/src-ui/src/app/components/dashboard/dashboard.component.html b/src-ui/src/app/components/dashboard/dashboard.component.html
index 627e7ff22..541255a68 100644
--- a/src-ui/src/app/components/dashboard/dashboard.component.html
+++ b/src-ui/src/app/components/dashboard/dashboard.component.html
@@ -1,4 +1,4 @@
-<app-page-header title="Dashboard" subTitle="Welcome to paperless-ng!">
+<app-page-header title="Dashboard" [subTitle]="subtitle">
   <img src="assets/logo.svg" height="80" class="m-2 d-none d-md-block">
 </app-page-header>
 
diff --git a/src-ui/src/app/components/dashboard/dashboard.component.ts b/src-ui/src/app/components/dashboard/dashboard.component.ts
index a14ec5e90..db9b5d425 100644
--- a/src-ui/src/app/components/dashboard/dashboard.component.ts
+++ b/src-ui/src/app/components/dashboard/dashboard.component.ts
@@ -1,4 +1,5 @@
 import { Component, OnInit } from '@angular/core';
+import { Meta } from '@angular/platform-browser';
 import { PaperlessSavedView } from 'src/app/data/paperless-saved-view';
 import { SavedViewService } from 'src/app/services/rest/saved-view.service';
 
@@ -11,8 +12,29 @@ import { SavedViewService } from 'src/app/services/rest/saved-view.service';
 export class DashboardComponent implements OnInit {
 
   constructor(
-    private savedViewService: SavedViewService) { }
+    private savedViewService: SavedViewService,
+    private meta: Meta
+  ) { }
 
+  get displayName() {
+    let tagFullName = this.meta.getTag('name=full_name')
+    let tagUsername = this.meta.getTag('name=username')
+    if (tagFullName && tagFullName.content) {
+      return tagFullName.content
+    } else if (tagUsername && tagUsername.content) {
+      return tagUsername.content
+    } else {
+      return null
+    }
+  }
+
+  get subtitle() {
+    if (this.displayName) {
+      return `Hello ${this.displayName}, welcome to Paperless-ng!`
+    } else {
+      return `Welcome to Paperless-ng!`
+    }
+  }
 
   savedViews: PaperlessSavedView[] = []
 
diff --git a/src/documents/templates/index.html b/src/documents/templates/index.html
index d086be0fe..47a352cd5 100644
--- a/src/documents/templates/index.html
+++ b/src/documents/templates/index.html
@@ -8,6 +8,8 @@
   <title>Paperless-ng</title>
   <base href="/">
   <meta name="viewport" content="width=device-width, initial-scale=1">
+	<meta name="username" content="{{username}}">
+	<meta name="full_name" content="{{full_name}}">
 	<meta name="cookie_prefix" content="{{cookie_prefix}}">
   <link rel="icon" type="image/x-icon" href="favicon.ico">
 <link rel="stylesheet" href="{% static 'frontend/styles.css' %}"></head>
diff --git a/src/documents/views.py b/src/documents/views.py
index 54d0de3f6..43e06065f 100755
--- a/src/documents/views.py
+++ b/src/documents/views.py
@@ -57,6 +57,8 @@ class IndexView(TemplateView):
     def get_context_data(self, **kwargs):
         context = super().get_context_data(**kwargs)
         context['cookie_prefix'] = settings.COOKIE_PREFIX
+        context['username'] = self.request.user.username
+        context['full_name'] = self.request.user.get_full_name()
         return context
 
 

From e75534c0f222ad256c4525c43691cbb49bb0efc5 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Mon, 21 Dec 2020 17:35:05 +0100
Subject: [PATCH 57/78] fixes #165

---
 .../management/commands/document_importer.py  | 31 +++++++++++++------
 .../tests/test_management_exporter.py         | 30 +++++++++++++++---
 2 files changed, 47 insertions(+), 14 deletions(-)

diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py
index 70d05d98b..8e9a79219 100644
--- a/src/documents/management/commands/document_importer.py
+++ b/src/documents/management/commands/document_importer.py
@@ -1,18 +1,29 @@
 import json
 import os
 import shutil
+from contextlib import contextmanager
 
 from django.conf import settings
 from django.core.management import call_command
 from django.core.management.base import BaseCommand, CommandError
+from django.db.models.signals import post_save, m2m_changed
 from filelock import FileLock
 
 from documents.models import Document
 from documents.settings import EXPORTER_FILE_NAME, EXPORTER_THUMBNAIL_NAME, \
     EXPORTER_ARCHIVE_NAME
-from ...file_handling import create_source_path_directory, \
-    generate_unique_filename
+from ...file_handling import create_source_path_directory
 from ...mixins import Renderable
+from ...signals.handlers import update_filename_and_move_files
+
+
+@contextmanager
+def disable_signal(sig, receiver, sender):
+    try:
+        sig.disconnect(receiver=receiver, sender=sender)
+        yield
+    finally:
+        sig.connect(receiver=receiver, sender=sender)
 
 
 class Command(Renderable, BaseCommand):
@@ -47,11 +58,16 @@ def handle(self, *args, **options):
             self.manifest = json.load(f)
 
         self._check_manifest()
+        with disable_signal(post_save,
+                            receiver=update_filename_and_move_files,
+                            sender=Document):
+            with disable_signal(m2m_changed,
+                                receiver=update_filename_and_move_files,
+                                sender=Document.tags.through):
+                # Fill up the database with whatever is in the manifest
+                call_command("loaddata", manifest_path)
 
-        # Fill up the database with whatever is in the manifest
-        call_command("loaddata", manifest_path)
-
-        self._import_files_from_manifest()
+                self._import_files_from_manifest()
 
     @staticmethod
     def _check_manifest_exists(path):
@@ -117,9 +133,6 @@ def _import_files_from_manifest(self):
             document.storage_type = Document.STORAGE_TYPE_UNENCRYPTED
 
             with FileLock(settings.MEDIA_LOCK):
-                document.filename = generate_unique_filename(
-                    document, settings.ORIGINALS_DIR)
-
                 if os.path.isfile(document.source_path):
                     raise FileExistsError(document.source_path)
 
diff --git a/src/documents/tests/test_management_exporter.py b/src/documents/tests/test_management_exporter.py
index 22d6fc7f6..d6ab7eadd 100644
--- a/src/documents/tests/test_management_exporter.py
+++ b/src/documents/tests/test_management_exporter.py
@@ -24,11 +24,17 @@ def test_exporter(self):
 
         file = os.path.join(self.dirs.originals_dir, "0000001.pdf")
 
-        Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", mime_type="application/pdf")
-        Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
-        Tag.objects.create(name="t")
-        DocumentType.objects.create(name="dt")
-        Correspondent.objects.create(name="c")
+        d1 = Document.objects.create(content="Content", checksum="42995833e01aea9b3edee44bbfdd7ce1", archive_checksum="62acb0bcbfbcaa62ca6ad3668e4e404b", title="wow", filename="0000001.pdf", mime_type="application/pdf")
+        d2 = Document.objects.create(content="Content", checksum="9c9691e51741c1f4f41a20896af31770", title="wow", filename="0000002.pdf.gpg", mime_type="application/pdf", storage_type=Document.STORAGE_TYPE_GPG)
+        t1 = Tag.objects.create(name="t")
+        dt1 = DocumentType.objects.create(name="dt")
+        c1 = Correspondent.objects.create(name="c")
+
+        d1.tags.add(t1)
+        d1.correspondents = c1
+        d1.document_type = dt1
+        d1.save()
+        d2.save()
 
         target = tempfile.mkdtemp()
         self.addCleanup(shutil.rmtree, target)
@@ -59,11 +65,25 @@ def test_exporter(self):
                     self.assertEqual(checksum, element['fields']['archive_checksum'])
 
         with paperless_environment() as dirs:
+            self.assertEqual(Document.objects.count(), 2)
+            Document.objects.all().delete()
+            Correspondent.objects.all().delete()
+            DocumentType.objects.all().delete()
+            Tag.objects.all().delete()
+            self.assertEqual(Document.objects.count(), 0)
+
             call_command('document_importer', target)
+            self.assertEqual(Document.objects.count(), 2)
             messages = check_sanity()
             # everything is alright after the test
             self.assertEqual(len(messages), 0, str([str(m) for m in messages]))
 
+    @override_settings(
+        PAPERLESS_FILENAME_FORMAT="{title}"
+    )
+    def test_exporter_with_filename_format(self):
+        self.test_exporter()
+
     def test_export_missing_files(self):
 
         target = tempfile.mkdtemp()

From b653e44f65bfbe9a80e5e1a4e21cb364bfc6abb7 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Mon, 21 Dec 2020 18:15:28 +0100
Subject: [PATCH 58/78] changed field order, updated ng-select for tag color
 selection

---
 .../common/input/select/select.component.html    |  2 +-
 .../correspondent-edit-dialog.component.html     |  5 ++---
 .../document-type-edit-dialog.component.html     |  4 ++--
 .../tag-edit-dialog.component.html               | 16 +++++++++++++---
 4 files changed, 18 insertions(+), 9 deletions(-)

diff --git a/src-ui/src/app/components/common/input/select/select.component.html b/src-ui/src/app/components/common/input/select/select.component.html
index 655adbe74..d33dae425 100644
--- a/src-ui/src/app/components/common/input/select/select.component.html
+++ b/src-ui/src/app/components/common/input/select/select.component.html
@@ -1,7 +1,7 @@
 <div class="form-group paperless-input-select">
   <label [for]="inputId">{{title}}</label>
   <div [class.input-group]="showPlusButton()">
-    <ng-select name="correspondent" [(ngModel)]="value"
+    <ng-select name="inputId" [(ngModel)]="value"
       [disabled]="disabled"
       [style.color]="textColor"
       [style.background]="backgroundColor"
diff --git a/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.html b/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.html
index 307c78c3c..e09ea38bf 100644
--- a/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.html
+++ b/src-ui/src/app/components/manage/correspondent-list/correspondent-edit-dialog/correspondent-edit-dialog.component.html
@@ -8,10 +8,9 @@ <h4 class="modal-title" id="modal-basic-title">{{getTitle()}}</h4>
   <div class="modal-body">
     
     <app-input-text title="Name" formControlName="name"></app-input-text>
-    <app-input-text title="Match" formControlName="match"></app-input-text>
     <app-input-select title="Matching algorithm" [items]="getMatchingAlgorithms()" formControlName="matching_algorithm"></app-input-select>
-    <app-input-check title="Case insensitive" formControlName="is_insensitive"></app-input-check>
-
+    <app-input-text title="Match" formControlName="match" hint="Auto matching does not require you to fill in this field."></app-input-text>
+    <app-input-check title="Case insensitive" formControlName="is_insensitive" hint="Auto matching ignores this option."></app-input-check>
   </div>
   <div class="modal-footer">
     <button type="button" class="btn btn-outline-dark" (click)="cancel()">Cancel</button>
diff --git a/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html b/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html
index 013c5a947..3338c40c3 100644
--- a/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html
+++ b/src-ui/src/app/components/manage/document-type-list/document-type-edit-dialog/document-type-edit-dialog.component.html
@@ -8,9 +8,9 @@ <h4 class="modal-title" id="modal-basic-title">{{getTitle()}}</h4>
     <div class="modal-body">
       
       <app-input-text title="Name" formControlName="name"></app-input-text>
-      <app-input-text title="Match" formControlName="match"></app-input-text>
       <app-input-select title="Matching algorithm" [items]="getMatchingAlgorithms()" formControlName="matching_algorithm"></app-input-select>
-      <app-input-check title="Case insensitive" formControlName="is_insensitive"></app-input-check>
+      <app-input-text title="Match" formControlName="match" hint="Auto matching does not require you to fill in this field."></app-input-text>
+      <app-input-check title="Case insensitive" formControlName="is_insensitive" hint="Auto matching ignores this option."></app-input-check>
 
     </div>
     <div class="modal-footer">
diff --git a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html
index 8048b0c80..2d9f93252 100644
--- a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html
+++ b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html
@@ -7,11 +7,21 @@ <h4 class="modal-title" id="modal-basic-title">{{getTitle()}}</h4>
     </div>
     <div class="modal-body">
       <app-input-text title="Name" formControlName="name"></app-input-text>
-      <app-input-select title="Colour" [items]="getColours()" formControlName="colour" [textColor]="getColor(objectForm.value.colour).textColor" [backgroundColor]="getColor(objectForm.value.colour).value"></app-input-select>
+
+
+      <div class="form-group paperless-input-select">
+        <label for="colour">Colour</label>
+        <ng-select name="colour" formControlName="colour" [items]="getColours()" bindValue="id" bindLabel="name">
+          <ng-template ng-option-tmp ng-label-tmp let-item="item">
+            <span class="badge" [style.background]="item.value" [style.color]="item.textColor">{{item.name}}</span>
+          </ng-template>
+        </ng-select>
+      </div>
+     
       <app-input-check title="Inbox tag" formControlName="is_inbox_tag" hint="Inbox tags are automatically assigned to all consumed documents."></app-input-check>
-      <app-input-text title="Match" formControlName="match"></app-input-text>
       <app-input-select title="Matching algorithm" [items]="getMatchingAlgorithms()" formControlName="matching_algorithm"></app-input-select>
-      <app-input-check title="Case insensitive" formControlName="is_insensitive"></app-input-check>
+      <app-input-text title="Match" formControlName="match" hint="Auto matching does not require you to fill in this field."></app-input-text>
+      <app-input-check title="Case insensitive" formControlName="is_insensitive" hint="Auto matching ignores this option."></app-input-check>
     </div>
     <div class="modal-footer">
       <button type="button" class="btn btn-outline-dark" (click)="cancel()">Cancel</button>

From 5892941d8a7c9b3dd227666334ebbb5825d45a70 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Mon, 21 Dec 2020 18:17:21 +0100
Subject: [PATCH 59/78] updated tag coloring for easier readability

---
 src-ui/src/app/data/paperless-tag.ts | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src-ui/src/app/data/paperless-tag.ts b/src-ui/src/app/data/paperless-tag.ts
index 551c6e03a..979a200a8 100644
--- a/src-ui/src/app/data/paperless-tag.ts
+++ b/src-ui/src/app/data/paperless-tag.ts
@@ -6,14 +6,14 @@ export const TAG_COLOURS = [
     {id: 1, value: "#a6cee3", name: "Light Blue", textColor: "#000000"},
     {id: 2, value: "#1f78b4", name: "Blue", textColor: "#ffffff"},
     {id: 3, value: "#b2df8a", name: "Light Green", textColor: "#000000"},
-    {id: 4, value: "#33a02c", name: "Green", textColor: "#000000"},
+    {id: 4, value: "#33a02c", name: "Green", textColor: "#ffffff"},
     {id: 5, value: "#fb9a99", name: "Light Red", textColor: "#000000"},
     {id: 6, value: "#e31a1c", name: "Red ", textColor: "#ffffff"},
     {id: 7, value: "#fdbf6f", name: "Light Orange", textColor: "#000000"},
     {id: 8, value: "#ff7f00", name: "Orange", textColor: "#000000"},
     {id: 9, value: "#cab2d6", name: "Light Violet", textColor: "#000000"},
     {id: 10, value: "#6a3d9a", name: "Violet", textColor: "#ffffff"},
-    {id: 11, value: "#b15928", name: "Brown", textColor: "#000000"},
+    {id: 11, value: "#b15928", name: "Brown", textColor: "#ffffff"},
     {id: 12, value: "#000000", name: "Black", textColor: "#ffffff"},
     {id: 13, value: "#cccccc", name: "Light Grey", textColor: "#000000"}
 ]

From c527b274b65fb4f32617cabb0848c18649a9451d Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Mon, 21 Dec 2020 18:23:06 +0100
Subject: [PATCH 60/78] disallow clearing tag color

---
 .../tag-list/tag-edit-dialog/tag-edit-dialog.component.html     | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html
index 2d9f93252..138d3e7cd 100644
--- a/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html
+++ b/src-ui/src/app/components/manage/tag-list/tag-edit-dialog/tag-edit-dialog.component.html
@@ -11,7 +11,7 @@ <h4 class="modal-title" id="modal-basic-title">{{getTitle()}}</h4>
 
       <div class="form-group paperless-input-select">
         <label for="colour">Colour</label>
-        <ng-select name="colour" formControlName="colour" [items]="getColours()" bindValue="id" bindLabel="name">
+        <ng-select name="colour" formControlName="colour" [items]="getColours()" bindValue="id" bindLabel="name" [clearable]="false">
           <ng-template ng-option-tmp ng-label-tmp let-item="item">
             <span class="badge" [style.background]="item.value" [style.color]="item.textColor">{{item.name}}</span>
           </ng-template>

From ae3e2a70632642ce63abdd534aead833a45d7868 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Mon, 21 Dec 2020 21:28:21 +0100
Subject: [PATCH 61/78] updated documentation

---
 docs/advanced_usage.rst | 79 -----------------------------------------
 docs/configuration.rst  |  5 ---
 2 files changed, 84 deletions(-)

diff --git a/docs/advanced_usage.rst b/docs/advanced_usage.rst
index 48a86384c..8f6b91b4c 100644
--- a/docs/advanced_usage.rst
+++ b/docs/advanced_usage.rst
@@ -5,85 +5,6 @@ Advanced topics
 Paperless offers a couple features that automate certain tasks and make your life
 easier.
 
-Guesswork
-#########
-
-
-Any document you put into the consumption directory will be consumed, but if
-you name the file right, it'll automatically set some values in the database
-for you.  This is is the logic the consumer follows:
-
-1. Try to find the correspondent, title, and tags in the file name following
-   the pattern: ``Date - Correspondent - Title - tag,tag,tag.pdf``.  Note that
-   the format of the date is **rigidly defined** as ``YYYYMMDDHHMMSSZ`` or
-   ``YYYYMMDDZ``.  The ``Z`` refers "Zulu time" AKA "UTC".
-   The tags are optional, so the format ``Date - Correspondent - Title.pdf``
-   works as well.
-2. If that doesn't work, we skip the date and try this pattern:
-   ``Correspondent - Title - tag,tag,tag.pdf``.
-3. If that doesn't work, we try to find the correspondent and title in the file
-   name following the pattern: ``Correspondent - Title.pdf``.
-4. If that doesn't work, just assume that the name of the file is the title.
-
-So given the above, the following examples would work as you'd expect:
-
-* ``20150314000700Z - Some Company Name - Invoice 2016-01-01 - money,invoices.pdf``
-* ``20150314Z - Some Company Name - Invoice 2016-01-01 - money,invoices.pdf``
-* ``Some Company Name - Invoice 2016-01-01 - money,invoices.pdf``
-* ``Another Company - Letter of Reference.jpg``
-* ``Dad's Recipe for Pancakes.png``
-
-These however wouldn't work:
-
-* ``2015-03-14 00:07:00 UTC - Some Company Name, Invoice 2016-01-01, money, invoices.pdf``
-* ``2015-03-14 - Some Company Name, Invoice 2016-01-01, money, invoices.pdf``
-* ``Some Company Name, Invoice 2016-01-01, money, invoices.pdf``
-* ``Another Company- Letter of Reference.jpg``
-
-Do I have to be so strict about naming?
-=======================================
-
-Rather than using the strict document naming rules, one can also set the option
-``PAPERLESS_FILENAME_DATE_ORDER`` in ``paperless.conf`` to any date order
-that is accepted by dateparser_. Doing so will cause ``paperless`` to default
-to any date format that is found in the title, instead of a date pulled from
-the document's text, without requiring the strict formatting of the document
-filename as described above.
-
-.. _dateparser: https://github.com/scrapinghub/dateparser/blob/v0.7.0/docs/usage.rst#settings
-
-.. _advanced-transforming_filenames:
-
-Transforming filenames for parsing
-==================================
-
-Some devices can't produce filenames that can be parsed by the default
-parser. By configuring the option ``PAPERLESS_FILENAME_PARSE_TRANSFORMS`` in
-``paperless.conf`` one can add transformations that are applied to the filename
-before it's parsed.
-
-The option contains a list of dictionaries of regular expressions (key:
-``pattern``) and replacements (key: ``repl``) in JSON format, which are
-applied in order by passing them to ``re.subn``. Transformation stops
-after the first match, so at most one transformation is applied. The general
-syntax is
-
-.. code:: python
-
-   [{"pattern":"pattern1", "repl":"repl1"}, {"pattern":"pattern2", "repl":"repl2"}, ..., {"pattern":"patternN", "repl":"replN"}]
-
-The example below is for a Brother ADS-2400N, a scanner that allows
-different names to different hardware buttons (useful for handling
-multiple entities in one instance), but insists on adding ``_<count>``
-to the filename.
-
-.. code:: python
-
-   # Brother profile configuration, support "Name_Date_Count" (the default
-   # setting) and "Name_Count" (use "Name" as tag and "Count" as title).
-   PAPERLESS_FILENAME_PARSE_TRANSFORMS=[{"pattern":"^([a-z]+)_(\\d{8})_(\\d{6})_([0-9]+)\\.", "repl":"\\2\\3Z - \\4 - \\1."}, {"pattern":"^([a-z]+)_([0-9]+)\\.", "repl":" - \\2 - \\1."}]
-
-
 .. _advanced-matching:
 
 Matching tags, correspondents and document types
diff --git a/docs/configuration.rst b/docs/configuration.rst
index d3f47215b..efc1a9db1 100644
--- a/docs/configuration.rst
+++ b/docs/configuration.rst
@@ -400,11 +400,6 @@ PAPERLESS_FILENAME_DATE_ORDER=<format>
 
     Defaults to none, which disables this feature.
 
-PAPERLESS_FILENAME_PARSE_TRANSFORMS
-    Transforms filenames before they are processed by paperless. See
-    :ref:`advanced-transforming_filenames` for details.
-
-    Defaults to none, which disables this feature.
 
 Binaries
 ########

From e466158ce218b7ba66abe99f10bebffb77cf4024 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 01:17:07 +0100
Subject: [PATCH 62/78] bugfix

---
 .../widgets/saved-view-widget/saved-view-widget.component.ts    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts
index 5bfecc640..f200d8db9 100644
--- a/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts
+++ b/src-ui/src/app/components/dashboard/widgets/saved-view-widget/saved-view-widget.component.ts
@@ -23,7 +23,7 @@ export class SavedViewWidgetComponent implements OnInit {
   documents: PaperlessDocument[] = []
 
   ngOnInit(): void {
-    this.documentService.list(1,10,this.savedView.sort_field, this.savedView.sort_reverse, this.savedView.filter_rules).subscribe(result => {
+    this.documentService.listFiltered(1,10,this.savedView.sort_field, this.savedView.sort_reverse, this.savedView.filter_rules).subscribe(result => {
       this.documents = result.results
     })
   }

From 5863060585e2ccb9c8c850333318d8881244a1a5 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 02:43:18 +0100
Subject: [PATCH 63/78] added selection to the large cards

---
 .../document-card-large.component.html        | 14 ++++++++---
 .../document-card-large.component.scss        | 25 +++++++++++++++++--
 .../document-card-large.component.ts          | 19 ++++++++++++++
 .../document-list.component.html              |  2 +-
 4 files changed, 54 insertions(+), 6 deletions(-)

diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
index 5bf0c9af2..8f52f707f 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.html
@@ -1,7 +1,15 @@
-<div class="card mb-3 bg-light shadow-sm">
+<div class="card mb-3 bg-light shadow-sm" [class.card-selected]="selected" [class.document-card]="selectable">
   <div class="row no-gutters">
-    <div class="col-md-2 d-none d-lg-block">
-      <img [src]="getThumbUrl()" class="card-img doc-img border-right">
+    <div class="col-md-2 d-none d-lg-block doc-img-background" [class.doc-img-background-selected]="selected">
+      <img [src]="getThumbUrl()" class="card-img doc-img border-right" (click)="selected = selectable ? !selected : false">
+
+      <div style="top: 0; left: 0" class="position-absolute border-right border-bottom bg-light p-1" [class.document-card-check]="!selected">
+        <div class="custom-control custom-checkbox">
+          <input type="checkbox" class="custom-control-input" id="smallCardCheck{{document.id}}" [checked]="selected" (change)="selected = $event.target.checked">
+          <label class="custom-control-label" for="smallCardCheck{{document.id}}"></label>
+        </div>
+      </div>
+
     </div>
     <div class="col">
       <div class="card-body">
diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss
index a20a56672..eb744d2af 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.scss
@@ -1,5 +1,6 @@
+@import "/src/theme";
+
 .result-content {
-  color: darkgray;
   overflow-wrap: anywhere;
 }
 
@@ -8,11 +9,31 @@
   object-position: top;
   height: 100%;
   position: absolute;
-
+  mix-blend-mode: multiply;
 }
 
 .search-score-bar {
   width: 100px;
   height: 5px;
   margin-top: 2px;
+}
+
+.document-card-check {
+  display: none
+}
+
+.document-card:hover .document-card-check {
+  display: block;
+}
+
+.card-selected {
+  border-color: $primary;
+}
+
+.doc-img-background {
+  background-color: white;
+}
+
+.doc-img-background-selected {
+  background-color: $primaryFaded;
 }
\ No newline at end of file
diff --git a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
index bcc1b1f3c..336af41da 100644
--- a/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
+++ b/src-ui/src/app/components/document-list/document-card-large/document-card-large.component.ts
@@ -12,6 +12,25 @@ export class DocumentCardLargeComponent implements OnInit {
 
   constructor(private documentService: DocumentService, private sanitizer: DomSanitizer) { }
 
+  _selected = false
+
+  get selected() {
+    return this._selected
+  }
+
+  @Input()
+  set selected(value: boolean) {
+    this._selected = value
+    this.selectedChange.emit(value)
+  }
+
+  @Output()
+  selectedChange = new EventEmitter<boolean>()
+
+  get selectable() {
+    return this.selectedChange.observers.length > 0
+  }
+
   @Input()
   moreLikeThis: boolean = false
 
diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html
index f1101610b..4b05d5e61 100644
--- a/src-ui/src/app/components/document-list/document-list.component.html
+++ b/src-ui/src/app/components/document-list/document-list.component.html
@@ -97,7 +97,7 @@
 </div>
 
 <div *ngIf="displayMode == 'largeCards'">
-  <app-document-card-large *ngFor="let d of list.documents" [document]="d" [details]="d.content" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)">
+  <app-document-card-large [selected]="list.isSelected(d)" (selectedChange)="list.setSelected(d, $event)"   *ngFor="let d of list.documents" [document]="d" [details]="d.content" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)">
   </app-document-card-large>
 </div>
 

From f42e5bf1e5324dec65a21ed0cf7fd9acd76aaff9 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 02:59:09 +0100
Subject: [PATCH 64/78] added tracking to the document list

---
 .../components/document-list/document-list.component.html   | 6 +++---
 .../app/components/document-list/document-list.component.ts | 4 ++++
 2 files changed, 7 insertions(+), 3 deletions(-)

diff --git a/src-ui/src/app/components/document-list/document-list.component.html b/src-ui/src/app/components/document-list/document-list.component.html
index 9657f35e3..7c7541b4d 100644
--- a/src-ui/src/app/components/document-list/document-list.component.html
+++ b/src-ui/src/app/components/document-list/document-list.component.html
@@ -73,7 +73,7 @@
 </div>
 
 <div *ngIf="displayMode == 'largeCards'">
-  <app-document-card-large *ngFor="let d of list.documents" [document]="d" [details]="d.content" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)">
+  <app-document-card-large *ngFor="let d of list.documents; trackBy: trackByDocumentId" [document]="d" [details]="d.content" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)">
   </app-document-card-large>
 </div>
 
@@ -87,7 +87,7 @@
     <th class="d-none d-xl-table-cell">Added</th>
   </thead>
   <tbody>
-    <tr *ngFor="let d of list.documents">
+    <tr *ngFor="let d of list.documents; trackBy: trackByDocumentId">
       <td class="d-none d-lg-table-cell">
         {{d.archive_serial_number}}
       </td>
@@ -117,5 +117,5 @@
 
 
 <div class="m-n2 row row-cols-paperless-cards" *ngIf="displayMode == 'smallCards'">
-  <app-document-card-small [document]="d" *ngFor="let d of list.documents" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)"></app-document-card-small>
+  <app-document-card-small [document]="d" *ngFor="let d of list.documents; trackBy: trackByDocumentId" (clickTag)="clickTag($event)" (clickCorrespondent)="clickCorrespondent($event)"></app-document-card-small>
 </div>
diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts
index 25d92e9db..2d5257804 100644
--- a/src-ui/src/app/components/document-list/document-list.component.ts
+++ b/src-ui/src/app/components/document-list/document-list.component.ts
@@ -2,6 +2,7 @@ import { Component, OnInit, ViewChild } from '@angular/core';
 import { ActivatedRoute, Router } from '@angular/router';
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap';
 import { FILTER_CORRESPONDENT } from 'src/app/data/filter-rule-type';
+import { PaperlessDocument } from 'src/app/data/paperless-document';
 import { PaperlessSavedView } from 'src/app/data/paperless-saved-view';
 import { DocumentListViewService } from 'src/app/services/document-list-view.service';
 import { DOCUMENT_SORT_FIELDS } from 'src/app/services/rest/document.service';
@@ -113,4 +114,7 @@ export class DocumentListComponent implements OnInit {
     this.filterEditor.toggleDocumentType(documentTypeID)
   }
 
+  trackByDocumentId(index, item: PaperlessDocument) {
+    return item.id
+  }
 }

From 147a8774c6b7a029a73faa03475965c3302646b5 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 12:40:06 +0100
Subject: [PATCH 65/78] more tests

---
 src/documents/tests/test_api.py | 52 +++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)

diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py
index 5d2e6a3c5..28aa05513 100644
--- a/src/documents/tests/test_api.py
+++ b/src/documents/tests/test_api.py
@@ -64,6 +64,58 @@ def testDocuments(self):
 
         self.assertEqual(len(Document.objects.all()), 0)
 
+    def test_document_fields(self):
+        c = Correspondent.objects.create(name="c", pk=41)
+        dt = DocumentType.objects.create(name="dt", pk=63)
+        tag = Tag.objects.create(name="t", pk=85)
+        doc = Document.objects.create(title="WOW", content="the content", correspondent=c, document_type=dt, checksum="123", mime_type="application/pdf")
+
+        response = self.client.get("/api/documents/", format='json')
+        self.assertEqual(response.status_code, 200)
+        results_full = response.data['results']
+        self.assertTrue("content" in results_full[0])
+        self.assertTrue("id" in results_full[0])
+
+        response = self.client.get("/api/documents/?fields=id", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertFalse("content" in results[0])
+        self.assertTrue("id" in results[0])
+        self.assertEqual(len(results[0]), 1)
+
+        response = self.client.get("/api/documents/?fields=content", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertTrue("content" in results[0])
+        self.assertFalse("id" in results[0])
+        self.assertEqual(len(results[0]), 1)
+
+        response = self.client.get("/api/documents/?fields=id,content", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertTrue("content" in results[0])
+        self.assertTrue("id" in results[0])
+        self.assertEqual(len(results[0]), 2)
+
+        response = self.client.get("/api/documents/?fields=id,conteasdnt", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertFalse("content" in results[0])
+        self.assertTrue("id" in results[0])
+        self.assertEqual(len(results[0]), 1)
+
+        response = self.client.get("/api/documents/?fields=", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertEqual(results_full, results)
+
+        response = self.client.get("/api/documents/?fields=dgfhs", format='json')
+        self.assertEqual(response.status_code, 200)
+        results = response.data['results']
+        self.assertEqual(len(results[0]), 0)
+
+
+
     def test_document_actions(self):
 
         _, filename = tempfile.mkstemp(dir=self.dirs.originals_dir)

From 1c3b85249c7880d8728de9ceb85b77d1f92e400d Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 13:03:50 +0100
Subject: [PATCH 66/78] more tests

---
 src/documents/tests/test_api.py | 130 +++++++++++++++++++++++++++++++-
 1 file changed, 126 insertions(+), 4 deletions(-)

diff --git a/src/documents/tests/test_api.py b/src/documents/tests/test_api.py
index 28aa05513..bce2a433d 100644
--- a/src/documents/tests/test_api.py
+++ b/src/documents/tests/test_api.py
@@ -735,7 +735,6 @@ def test_add_tag(self):
         args, kwargs = self.async_task.call_args
         self.assertCountEqual(kwargs['document_ids'], [self.doc1.id, self.doc3.id])
 
-
     def test_remove_tag(self):
         self.assertEqual(Document.objects.filter(tags__id=self.t1.id).count(), 2)
         bulk_edit.remove_tag([self.doc1.id, self.doc3.id, self.doc4.id], self.t1.id)
@@ -750,15 +749,103 @@ def test_delete(self):
         self.assertEqual(Document.objects.count(), 3)
         self.assertCountEqual([doc.id for doc in Document.objects.all()], [self.doc3.id, self.doc4.id, self.doc5.id])
 
-    def test_api(self):
-        self.assertEqual(Document.objects.count(), 5)
+    @mock.patch("documents.serialisers.bulk_edit.set_correspondent")
+    def test_api_set_correspondent(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "set_correspondent",
+            "parameters": {"correspondent": self.c1.id}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertEqual(kwargs['correspondent'], self.c1.id)
+
+    @mock.patch("documents.serialisers.bulk_edit.set_correspondent")
+    def test_api_unset_correspondent(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "set_correspondent",
+            "parameters": {"correspondent": None}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertIsNone(kwargs['correspondent'])
+
+    @mock.patch("documents.serialisers.bulk_edit.set_document_type")
+    def test_api_set_type(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "set_document_type",
+            "parameters": {"document_type": self.dt1.id}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertEqual(kwargs['document_type'], self.dt1.id)
+
+    @mock.patch("documents.serialisers.bulk_edit.set_document_type")
+    def test_api_unset_type(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "set_document_type",
+            "parameters": {"document_type": None}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertIsNone(kwargs['document_type'])
+
+    @mock.patch("documents.serialisers.bulk_edit.add_tag")
+    def test_api_add_tag(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "add_tag",
+            "parameters": {"tag": self.t1.id}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertEqual(kwargs['tag'], self.t1.id)
+
+    @mock.patch("documents.serialisers.bulk_edit.remove_tag")
+    def test_api_remove_tag(self, m):
+        m.return_value = "OK"
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc1.id],
+            "method": "remove_tag",
+            "parameters": {"tag": self.t1.id}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 200)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertEqual(kwargs['tag'], self.t1.id)
+
+    @mock.patch("documents.serialisers.bulk_edit.delete")
+    def test_api_delete(self, m):
+        m.return_value = "OK"
         response = self.client.post("/api/documents/bulk_edit/", json.dumps({
             "documents": [self.doc1.id],
             "method": "delete",
             "parameters": {}
         }), content_type='application/json')
         self.assertEqual(response.status_code, 200)
-        self.assertEqual(Document.objects.count(), 4)
+        m.assert_called_once()
+        args, kwargs = m.call_args
+        self.assertEqual(args[0], [self.doc1.id])
+        self.assertEqual(len(kwargs), 0)
 
     def test_api_invalid_doc(self):
         self.assertEqual(Document.objects.count(), 5)
@@ -779,3 +866,38 @@ def test_api_invalid_method(self):
         }), content_type='application/json')
         self.assertEqual(response.status_code, 400)
         self.assertEqual(Document.objects.count(), 5)
+
+    def test_api_invalid_correspondent(self):
+        self.assertEqual(self.doc2.correspondent, self.c1)
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc2.id],
+            "method": "set_correspondent",
+            "parameters": {'correspondent': 345657}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+
+        doc2 = Document.objects.get(id=self.doc2.id)
+        self.assertEqual(doc2.correspondent, self.c1)
+
+    def test_api_invalid_document_type(self):
+        self.assertEqual(self.doc2.document_type, self.dt1)
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc2.id],
+            "method": "set_document_type",
+            "parameters": {'document_type': 345657}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+
+        doc2 = Document.objects.get(id=self.doc2.id)
+        self.assertEqual(doc2.document_type, self.dt1)
+
+    def test_api_invalid_tag(self):
+        self.assertEqual(list(self.doc2.tags.all()), [self.t1])
+        response = self.client.post("/api/documents/bulk_edit/", json.dumps({
+            "documents": [self.doc2.id],
+            "method": "add_tag",
+            "parameters": {'document_type': 345657}
+        }), content_type='application/json')
+        self.assertEqual(response.status_code, 400)
+
+        self.assertEqual(list(self.doc2.tags.all()), [self.t1])

From b7d310ef90d0373dfa93efdddd86e1c6ca524627 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 13:04:08 +0100
Subject: [PATCH 67/78] more tests

---
 src/documents/checks.py            |  2 +-
 src/documents/tests/test_checks.py | 15 ++++++++++++++-
 2 files changed, 15 insertions(+), 2 deletions(-)

diff --git a/src/documents/checks.py b/src/documents/checks.py
index b6da5bfc9..ba55b1397 100644
--- a/src/documents/checks.py
+++ b/src/documents/checks.py
@@ -51,6 +51,6 @@ def parser_check(app_configs, **kwargs):
 
     if len(parsers) == 0:
         return [Error("No parsers found. This is a bug. The consumer won't be "
-                      "able to onsume any documents without parsers.")]
+                      "able to consume any documents without parsers.")]
     else:
         return []
diff --git a/src/documents/tests/test_checks.py b/src/documents/tests/test_checks.py
index 1027c11a0..ee4fbe8d1 100644
--- a/src/documents/tests/test_checks.py
+++ b/src/documents/tests/test_checks.py
@@ -1,9 +1,12 @@
 import unittest
+from unittest import mock
 
+from django.core.checks import Error
 from django.test import TestCase
 
 from .factories import DocumentFactory
-from ..checks import changed_password_check
+from .. import document_consumer_declaration
+from ..checks import changed_password_check, parser_check
 from ..models import Document
 
 
@@ -15,3 +18,13 @@ def test_changed_password_check_empty_db(self):
     def test_changed_password_check_no_encryption(self):
         DocumentFactory.create(storage_type=Document.STORAGE_TYPE_UNENCRYPTED)
         self.assertEqual(changed_password_check(None), [])
+
+    def test_parser_check(self):
+
+        self.assertEqual(parser_check(None), [])
+
+        with mock.patch('documents.checks.document_consumer_declaration.send') as m:
+            m.return_value = []
+
+            self.assertEqual(parser_check(None), [Error("No parsers found. This is a bug. The consumer won't be "
+                                                        "able to consume any documents without parsers.")])

From 6968e228e17391bc78cf1e1142f8537199036012 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 13:04:22 +0100
Subject: [PATCH 68/78] increase indexing speed

---
 src/documents/management/commands/document_index.py | 11 ++++++-----
 1 file changed, 6 insertions(+), 5 deletions(-)

diff --git a/src/documents/management/commands/document_index.py b/src/documents/management/commands/document_index.py
index 7dfdbaa42..08e20e1d2 100644
--- a/src/documents/management/commands/document_index.py
+++ b/src/documents/management/commands/document_index.py
@@ -1,4 +1,5 @@
 from django.core.management import BaseCommand
+from django.db import transaction
 
 from documents.mixins import Renderable
 from documents.tasks import index_reindex, index_optimize
@@ -18,8 +19,8 @@ def add_arguments(self, parser):
     def handle(self, *args, **options):
 
         self.verbosity = options["verbosity"]
-
-        if options['command'] == 'reindex':
-            index_reindex()
-        elif options['command'] == 'optimize':
-            index_optimize()
+        with transaction.atomic():
+            if options['command'] == 'reindex':
+                index_reindex()
+            elif options['command'] == 'optimize':
+                index_optimize()

From 2f599ca1d3a79fe5539f3135d8b39e953b2b35c5 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 13:32:30 +0100
Subject: [PATCH 69/78] fixes an issue with open documents not refreshing after
 bulk operations

---
 .../document-list/document-list.component.ts      | 15 ++++++++-------
 src-ui/src/app/services/open-documents.service.ts | 12 +++++++++++-
 2 files changed, 19 insertions(+), 8 deletions(-)

diff --git a/src-ui/src/app/components/document-list/document-list.component.ts b/src-ui/src/app/components/document-list/document-list.component.ts
index 5eb97b08d..f72a92aa9 100644
--- a/src-ui/src/app/components/document-list/document-list.component.ts
+++ b/src-ui/src/app/components/document-list/document-list.component.ts
@@ -2,8 +2,7 @@ import { Component, OnInit, ViewChild } from '@angular/core';
 import { ActivatedRoute, Router } from '@angular/router';
 import { NgbModal } from '@ng-bootstrap/ng-bootstrap';
 import { Observable } from 'rxjs';
-import { map } from 'rxjs/operators';
-import { FILTER_CORRESPONDENT } from 'src/app/data/filter-rule-type';
+import { tap } from 'rxjs/operators';
 import { PaperlessDocument } from 'src/app/data/paperless-document';
 import { PaperlessSavedView } from 'src/app/data/paperless-saved-view';
 import { DocumentListViewService } from 'src/app/services/document-list-view.service';
@@ -17,6 +16,7 @@ import { FilterEditorComponent } from '../filter-editor/filter-editor.component'
 import { ConfirmDialogComponent } from '../common/confirm-dialog/confirm-dialog.component';
 import { SelectDialogComponent } from '../common/select-dialog/select-dialog.component';
 import { SaveViewConfigDialogComponent } from './save-view-config-dialog/save-view-config-dialog.component';
+import { OpenDocumentsService } from 'src/app/services/open-documents.service';
 
 @Component({
   selector: 'app-document-list',
@@ -35,7 +35,8 @@ export class DocumentListComponent implements OnInit {
     private correspondentService: CorrespondentService,
     private documentTypeService: DocumentTypeService,
     private tagService: TagService,
-    private documentService: DocumentService) { }
+    private documentService: DocumentService,
+    private openDocumentService: OpenDocumentsService) { }
 
   @ViewChild("filterEditor")
   private filterEditor: FilterEditorComponent
@@ -131,12 +132,12 @@ export class DocumentListComponent implements OnInit {
 
   private executeBulkOperation(method: string, args): Observable<any> {
     return this.documentService.bulkEdit(Array.from(this.list.selected), method, args).pipe(
-      map(r => {
-
+      tap(() => {
         this.list.reload()
+        this.list.selected.forEach(id => {
+          this.openDocumentService.refreshDocument(id)
+        })
         this.list.selectNone()
-
-        return r
       })
     )
   }
diff --git a/src-ui/src/app/services/open-documents.service.ts b/src-ui/src/app/services/open-documents.service.ts
index e37f5db8c..c91031f83 100644
--- a/src-ui/src/app/services/open-documents.service.ts
+++ b/src-ui/src/app/services/open-documents.service.ts
@@ -1,6 +1,7 @@
 import { Injectable } from '@angular/core';
 import { PaperlessDocument } from '../data/paperless-document';
 import { OPEN_DOCUMENT_SERVICE } from '../data/storage-keys';
+import { DocumentService } from './rest/document.service';
 
 @Injectable({
   providedIn: 'root'
@@ -9,7 +10,7 @@ export class OpenDocumentsService {
 
   private MAX_OPEN_DOCUMENTS = 5
 
-  constructor() { 
+  constructor(private documentService: DocumentService) { 
     if (sessionStorage.getItem(OPEN_DOCUMENT_SERVICE.DOCUMENTS)) {
       try {
         this.openDocuments = JSON.parse(sessionStorage.getItem(OPEN_DOCUMENT_SERVICE.DOCUMENTS))
@@ -22,6 +23,15 @@ export class OpenDocumentsService {
 
   private openDocuments: PaperlessDocument[] = []
 
+  refreshDocument(id: number) {
+    let index = this.openDocuments.findIndex(doc => doc.id == id)
+    if (index > -1) {
+      this.documentService.get(id).subscribe(doc => {
+        this.openDocuments[index] = doc
+      })
+    }
+  }
+
   getOpenDocuments(): PaperlessDocument[] {
     return this.openDocuments
   }

From 0aad31b4bc6bd9866fdfcb9550b89ffca41c24f3 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 13:39:46 +0100
Subject: [PATCH 70/78] fixes clearing fields that should not be clearable

---
 .../app/components/common/input/select/select.component.html  | 1 +
 .../components/document-detail/document-detail.component.html | 4 ++--
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/src-ui/src/app/components/common/input/select/select.component.html b/src-ui/src/app/components/common/input/select/select.component.html
index d33dae425..780dc5686 100644
--- a/src-ui/src/app/components/common/input/select/select.component.html
+++ b/src-ui/src/app/components/common/input/select/select.component.html
@@ -5,6 +5,7 @@
       [disabled]="disabled"
       [style.color]="textColor"
       [style.background]="backgroundColor"
+      [clearable]="allowNull"
       (change)="onChange(value)"
       (blur)="onTouched()">
       <ng-option *ngFor="let i of items" [value]="i.id">{{i.name}}</ng-option>
diff --git a/src-ui/src/app/components/document-detail/document-detail.component.html b/src-ui/src/app/components/document-detail/document-detail.component.html
index 228264378..ae3fb0c0a 100644
--- a/src-ui/src/app/components/document-detail/document-detail.component.html
+++ b/src-ui/src/app/components/document-detail/document-detail.component.html
@@ -67,9 +67,9 @@
                                 formControlName='archive_serial_number'>
                         </div>
                         <app-input-date-time titleDate="Date created" formControlName="created"></app-input-date-time>
-                        <app-input-select [items]="correspondents" title="Correspondent" formControlName="correspondent"
+                        <app-input-select [items]="correspondents" title="Correspondent" formControlName="correspondent" [allowNull]="true"
                             (createNew)="createCorrespondent()"></app-input-select>
-                        <app-input-select [items]="documentTypes" title="Document type" formControlName="document_type"
+                        <app-input-select [items]="documentTypes" title="Document type" formControlName="document_type" [allowNull]="true"
                             (createNew)="createDocumentType()"></app-input-select>
                         <app-input-tags formControlName="tags" title="Tags"></app-input-tags>
 

From 0dbd598116d720d9d371015adccd06b42919ebdb Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 13:54:41 +0100
Subject: [PATCH 71/78] changelog

---
 docs/changelog.rst | 32 ++++++++++++++++++++++++++++++++
 1 file changed, 32 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index a993eb530..795582260 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -6,6 +6,38 @@ Changelog
 *********
 
 
+paperless-ng 0.9.9
+##################
+
+Christmas release!
+
+Bulk editing
+
+* Paperless now supports bulk editing.
+* The following operations are available: Add and remove correspondents, tags, document types from selected documents, as well as mass-deleting documents.
+* We've got a more fancy UI in the works that makes these features more accessible, but that's not quite ready yet.
+
+Searching
+
+* Paperless now supports searching for similar documents ("More like this") both from the document detail page as well as from individual search results.
+* A search score indicates how well a document matches the search query, or how similar a document is to a given reference document.
+
+Changes
+
+* Clarification in the UI that the fields "Match" and "Is insensitive" are not relevant for the Auto matching algorithm.
+* New select interface for tags, types and correspondents allows filtering. This also improves tag selection. Thanks again to `Michael Shamoon`_!
+* Page navigation controls for the document viewer, thanks to `Michael Shamoon`_.
+
+Fixes
+
+* An error that caused the document importer to crash was fixed.
+* An issue with changes not being possible when ``PAPERLESS_COOKIE_PREFIX`` is used was fixed.
+* The date selection filters now allow manual entry of dates.
+
+Feature Removal
+
+* Most of the guesswork features have been removed. Paperless no longer tries to extract correspondents and tags from file names.
+
 paperless-ng 0.9.8
 ##################
 

From 3e389a3140c5acea15035d778afb7340fc61a8d8 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 14:09:38 +0100
Subject: [PATCH 72/78] versions

---
 docker/hub/docker-compose.postgres.yml      | 2 +-
 docker/hub/docker-compose.sqlite.yml        | 2 +-
 src-ui/src/environments/environment.prod.ts | 2 +-
 src/paperless/version.py                    | 2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)

diff --git a/docker/hub/docker-compose.postgres.yml b/docker/hub/docker-compose.postgres.yml
index d33e4c38d..e7a32bec0 100644
--- a/docker/hub/docker-compose.postgres.yml
+++ b/docker/hub/docker-compose.postgres.yml
@@ -15,7 +15,7 @@ services:
       POSTGRES_PASSWORD: paperless
 
   webserver:
-    image: jonaswinkler/paperless-ng:0.9.8
+    image: jonaswinkler/paperless-ng:0.9.9
     restart: always
     depends_on:
       - db
diff --git a/docker/hub/docker-compose.sqlite.yml b/docker/hub/docker-compose.sqlite.yml
index c130dfef6..98b7d70a2 100644
--- a/docker/hub/docker-compose.sqlite.yml
+++ b/docker/hub/docker-compose.sqlite.yml
@@ -5,7 +5,7 @@ services:
     restart: always
 
   webserver:
-    image: jonaswinkler/paperless-ng:0.9.8
+    image: jonaswinkler/paperless-ng:0.9.9
     restart: always
     depends_on:
       - broker
diff --git a/src-ui/src/environments/environment.prod.ts b/src-ui/src/environments/environment.prod.ts
index f12c6a7cb..ab6b07c73 100644
--- a/src-ui/src/environments/environment.prod.ts
+++ b/src-ui/src/environments/environment.prod.ts
@@ -2,5 +2,5 @@ export const environment = {
   production: true,
   apiBaseUrl: "/api/",
   appTitle: "Paperless-ng",
-  version: "0.9.8"
+  version: "0.9.9"
 };
diff --git a/src/paperless/version.py b/src/paperless/version.py
index 10283c145..b1dfc590c 100644
--- a/src/paperless/version.py
+++ b/src/paperless/version.py
@@ -1 +1 @@
-__version__ = (0, 9, 8)
+__version__ = (0, 9, 9)

From 978bb3c33980dc2d9bea056a47db26cdbde02d0b Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 14:13:58 +0100
Subject: [PATCH 73/78] more change log

---
 docs/changelog.rst | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index 795582260..db3378011 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -27,6 +27,8 @@ Changes
 * Clarification in the UI that the fields "Match" and "Is insensitive" are not relevant for the Auto matching algorithm.
 * New select interface for tags, types and correspondents allows filtering. This also improves tag selection. Thanks again to `Michael Shamoon`_!
 * Page navigation controls for the document viewer, thanks to `Michael Shamoon`_.
+* Layout changes to the small cards document list.
+* The dashboard now displays the username (or full name if specified in the admin) on the dashboard.
 
 Fixes
 

From b08a5e62d89560fef4dc5a8ae27749190c68bad5 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 14:43:16 +0100
Subject: [PATCH 74/78] documentation

---
 docs/setup.rst           |  2 ++
 docs/troubleshooting.rst | 28 +++++++++++++++++++++++++++-
 2 files changed, 29 insertions(+), 1 deletion(-)

diff --git a/docs/setup.rst b/docs/setup.rst
index e5e6526ea..908704fc2 100644
--- a/docs/setup.rst
+++ b/docs/setup.rst
@@ -120,6 +120,8 @@ The `bare metal route`_ is more complicated to setup but makes it easier
 should you want to contribute some code back. You need to configure and
 run the above mentioned components yourself.
 
+.. _setup-docker_route:
+
 Docker Route
 ============
 
diff --git a/docs/troubleshooting.rst b/docs/troubleshooting.rst
index dc5bf7f5d..4c06ec4cd 100644
--- a/docs/troubleshooting.rst
+++ b/docs/troubleshooting.rst
@@ -39,7 +39,7 @@ Operation not permitted
 
 You might see errors such as:
 
-.. code::
+.. code:: shell-session
 
     chown: changing ownership of '../export': Operation not permitted
 
@@ -49,3 +49,29 @@ to these folders. This happens when pointing these directories to NFS shares,
 for example.
 
 Ensure that `chown` is possible on these directories.
+
+Classifier error: No training data available
+############################################
+
+This indicates that the Auto matching algorithm found no documents to learn from.
+This may have two reasons:
+
+*   You don't use the Auto matching algorithm: The error can be safely ignored in this case.
+*   You are using the Auto matching algorithm: The classifier explicitly excludes documents
+    with Inbox tags. Verify that there are documents in your archive without inbox tags.
+    The algorithm will only learn from documents not in your inbox.
+
+Permission denied errors in the consumption directory
+#####################################################
+
+You might encounter errors such as:
+
+.. code:: shell-session
+
+    The following error occured while consuming document.pdf: [Errno 13] Permission denied: '/usr/src/paperless/src/../consume/document.pdf'
+
+This happens when paperless does not have permission to delete files inside the consumption directory.
+Ensure that ``USERMAP_UID`` and ``USERMAP_GID`` are set to the user id and group id you use on the host operating system, if these are
+different from ``1000``. See :ref:`setup-docker_route`.
+
+Also ensure that you are able to read and write to the consumption directory on the host.

From 403de420311e0a3f9324d507c181f9b65bfa3d49 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 15:18:42 +0100
Subject: [PATCH 75/78] discard selection when switching views

---
 src-ui/src/app/services/document-list-view.service.ts | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/src-ui/src/app/services/document-list-view.service.ts b/src-ui/src/app/services/document-list-view.service.ts
index 00f84048b..b148d4087 100644
--- a/src-ui/src/app/services/document-list-view.service.ts
+++ b/src-ui/src/app/services/document-list-view.service.ts
@@ -40,10 +40,14 @@ export class DocumentListViewService {
   }
 
   set savedView(value: PaperlessSavedView) {
-    if (value) {
+    if (value && !this._savedViewConfig || value && value.id != this._savedViewConfig.id) {
+      //saved view inactive and should be active now, or saved view active, but a different view is requested
       //this is here so that we don't modify value, which might be the actual instance of the saved view.
+      this.selectNone()
       this._savedViewConfig = Object.assign({}, value)
-    } else {
+    } else if (this._savedViewConfig && !value) {
+      //saved view active, but document list requested
+      this.selectNone()
       this._savedViewConfig = null
     }
   }

From 8dc15352effac4cb9739135fb70d1166be418523 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 15:22:24 +0100
Subject: [PATCH 76/78] changelog layout

---
 docs/changelog.rst | 38 +++++++++++++++++++-------------------
 1 file changed, 19 insertions(+), 19 deletions(-)

diff --git a/docs/changelog.rst b/docs/changelog.rst
index db3378011..e63c19d7d 100644
--- a/docs/changelog.rst
+++ b/docs/changelog.rst
@@ -11,34 +11,34 @@ paperless-ng 0.9.9
 
 Christmas release!
 
-Bulk editing
+* Bulk editing
 
-* Paperless now supports bulk editing.
-* The following operations are available: Add and remove correspondents, tags, document types from selected documents, as well as mass-deleting documents.
-* We've got a more fancy UI in the works that makes these features more accessible, but that's not quite ready yet.
+  * Paperless now supports bulk editing.
+  * The following operations are available: Add and remove correspondents, tags, document types from selected documents, as well as mass-deleting documents.
+  * We've got a more fancy UI in the works that makes these features more accessible, but that's not quite ready yet.
 
-Searching
+* Searching
 
-* Paperless now supports searching for similar documents ("More like this") both from the document detail page as well as from individual search results.
-* A search score indicates how well a document matches the search query, or how similar a document is to a given reference document.
+  * Paperless now supports searching for similar documents ("More like this") both from the document detail page as well as from individual search results.
+  * A search score indicates how well a document matches the search query, or how similar a document is to a given reference document.
 
-Changes
+* Other additions and changes
 
-* Clarification in the UI that the fields "Match" and "Is insensitive" are not relevant for the Auto matching algorithm.
-* New select interface for tags, types and correspondents allows filtering. This also improves tag selection. Thanks again to `Michael Shamoon`_!
-* Page navigation controls for the document viewer, thanks to `Michael Shamoon`_.
-* Layout changes to the small cards document list.
-* The dashboard now displays the username (or full name if specified in the admin) on the dashboard.
+  * Clarification in the UI that the fields "Match" and "Is insensitive" are not relevant for the Auto matching algorithm.
+  * New select interface for tags, types and correspondents allows filtering. This also improves tag selection. Thanks again to `Michael Shamoon`_!
+  * Page navigation controls for the document viewer, thanks to `Michael Shamoon`_.
+  * Layout changes to the small cards document list.
+  * The dashboard now displays the username (or full name if specified in the admin) on the dashboard.
 
-Fixes
+* Fixes
 
-* An error that caused the document importer to crash was fixed.
-* An issue with changes not being possible when ``PAPERLESS_COOKIE_PREFIX`` is used was fixed.
-* The date selection filters now allow manual entry of dates.
+  * An error that caused the document importer to crash was fixed.
+  * An issue with changes not being possible when ``PAPERLESS_COOKIE_PREFIX`` is used was fixed.
+  * The date selection filters now allow manual entry of dates.
 
-Feature Removal
+* Feature Removal
 
-* Most of the guesswork features have been removed. Paperless no longer tries to extract correspondents and tags from file names.
+  * Most of the guesswork features have been removed. Paperless no longer tries to extract correspondents and tags from file names.
 
 paperless-ng 0.9.8
 ##################

From ebcb2cf694a0b92e24b23ea3b5ebe1afc185322a Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 15:50:27 +0100
Subject: [PATCH 77/78] progress bar for the document importer

---
 .../management/commands/document_importer.py       | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py
index 8e9a79219..344be32be 100644
--- a/src/documents/management/commands/document_importer.py
+++ b/src/documents/management/commands/document_importer.py
@@ -1,8 +1,10 @@
 import json
+import logging
 import os
 import shutil
 from contextlib import contextmanager
 
+import tqdm
 from django.conf import settings
 from django.core.management import call_command
 from django.core.management.base import BaseCommand, CommandError
@@ -43,6 +45,8 @@ def __init__(self, *args, **kwargs):
 
     def handle(self, *args, **options):
 
+        logging.getLogger().handlers[0].level = logging.ERROR
+
         self.source = options["source"]
 
         if not os.path.exists(self.source):
@@ -111,10 +115,13 @@ def _import_files_from_manifest(self):
         os.makedirs(settings.THUMBNAIL_DIR, exist_ok=True)
         os.makedirs(settings.ARCHIVE_DIR, exist_ok=True)
 
-        for record in self.manifest:
+        print("Copy files into paperless...")
 
-            if not record["model"] == "documents.document":
-                continue
+        manifest_documents = list(filter(
+            lambda r: r["model"] == "documents.document",
+            self.manifest))
+
+        for record in tqdm.tqdm(manifest_documents):
 
             document = Document.objects.get(pk=record["pk"])
 
@@ -138,7 +145,6 @@ def _import_files_from_manifest(self):
 
                 create_source_path_directory(document.source_path)
 
-                print(f"Moving {document_path} to {document.source_path}")
                 shutil.copy(document_path, document.source_path)
                 shutil.copy(thumbnail_path, document.thumbnail_path)
                 if archive_path:

From 0fa3c3188d9c324de15ee71243f8b34e456daf50 Mon Sep 17 00:00:00 2001
From: jonaswinkler <jonas.winkler@jpwinkler.de>
Date: Tue, 22 Dec 2020 15:53:04 +0100
Subject: [PATCH 78/78] update index after import

---
 src/documents/management/commands/document_importer.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/src/documents/management/commands/document_importer.py b/src/documents/management/commands/document_importer.py
index 344be32be..6df14a82c 100644
--- a/src/documents/management/commands/document_importer.py
+++ b/src/documents/management/commands/document_importer.py
@@ -73,6 +73,9 @@ def handle(self, *args, **options):
 
                 self._import_files_from_manifest()
 
+        print("Updating search index...")
+        call_command('document_index', 'reindex')
+
     @staticmethod
     def _check_manifest_exists(path):
         if not os.path.exists(path):