Overhaul moneybird (#3446)

* Fix moneybird mandate synchronization bug * Rename stuff * Add starting point for testing * Implement bulk moneybird payment creation * Push most invoices at night * Make EventRegistration.payment_amount queryable * Push EventRegistration invoices nightly * Remove customer_id from contact * Improve displaying of transactions * Fix * Add SEPA fields to transactions * Improve SEPA fields * Improve SEPA fields * Improve SEPA fields * Fix timezone * Slightly improved admins * Slightly improved admins * Actually remove customer ID * Make moneybird client retry on ratelimit * Organize services * Use loggers * Implement deleting eventregistration invoices * Implement updating invoices * Send error report email after logging * Make migration set needs_synchronization False * Implement deleting invoices * Improve admins * Refactor services * Never use thaliawebsite.settings * Improve model constraints * Write a test * Add a few tests * Add test for FoodOrders * Fix and test pushing contacts * Add sales test * Write last test * Fix query on postgres --------- Co-authored-by: Ties <[email protected]> Co-authored-by: Job Doesburg <[email protected]> Co-authored-by: movieminer <[email protected]>
svthalia · Oct 26, 2023 · 0c690c4 · 0c690c4
1 parent 56692f6
commit 0c690c4
Show file tree

Hide file tree

Showing 20 changed files with 1,306 additions and 205 deletions.
diff --git a/website/events/models/event_registration.py b/website/events/models/event_registration.py
@@ -7,7 +7,7 @@
 from django.utils.translation import gettext_lazy as _
 
 from queryable_properties.managers import QueryablePropertiesManager
-from queryable_properties.properties import AnnotationProperty
+from queryable_properties.properties import AnnotationProperty, queryable_property
 
 from payments.models import PaymentAmountField
 
@@ -176,10 +176,18 @@ def is_late_cancellation(self):
     def is_paid(self):
         return self.payment
 
-    @property
+    @queryable_property
     def payment_amount(self):
         return self.event.price if not self.special_price else self.special_price
 
+    @payment_amount.annotater
+    @classmethod
+    def payment_amount(cls):
+        return Case(
+            When(Q(special_price__isnull=False), then=F("special_price")),
+            default=F("event__price"),
+        )
+
     def would_cancel_after_deadline(self):
         now = timezone.now()
         if not self.event.registration_required:

diff --git a/website/events/tests/test_views.py b/website/events/tests/test_views.py
@@ -1,5 +1,6 @@
 import datetime
 
+from django.conf import settings
 from django.contrib.auth.models import Permission
 from django.core import mail
 from django.test import Client, TestCase, override_settings
@@ -17,7 +18,6 @@
 )
 from mailinglists.models import MailingList
 from members.models import Member
-from thaliawebsite import settings
 
 
 @override_settings(SUSPEND_SIGNALS=True)

diff --git a/website/moneybirdsynchronization/admin.py b/website/moneybirdsynchronization/admin.py
@@ -1,4 +1,5 @@
 from django.contrib import admin
+from django.contrib.admin import RelatedOnlyFieldListFilter
 
 from .models import MoneybirdContact, MoneybirdExternalInvoice, MoneybirdPayment
 
@@ -7,14 +8,95 @@
 class MoneybirdContactAdmin(admin.ModelAdmin):
     """Manage moneybird contacts."""
 
-    list_display = ("member", "moneybird_id")
+    list_display = (
+        "member",
+        "moneybird_id",
+        "moneybird_sepa_mandate_id",
+    )
+
+    fields = (
+        "member",
+        "moneybird_id",
+        "moneybird_sepa_mandate_id",
+    )
+
+    raw_id_fields = ("member",)
+
+    search_fields = (
+        "member__first_name",
+        "member__last_name",
+        "member__username",
+        "member__email",
+        "moneybird_id",
+    )
+
+    def get_readonly_fields(self, request, obj=None):
+        if not obj:
+            return ()
+        return ("member",)
 
 
 @admin.register(MoneybirdExternalInvoice)
 class MoneybirdExternalInvoiceAdmin(admin.ModelAdmin):
-    list_display = ("payable_object",)
+    """Manage moneybird external invoices."""
+
+    list_display = (
+        "payable_object",
+        "payable_model",
+        "moneybird_invoice_id",
+        "needs_synchronization",
+        "needs_deletion",
+    )
+
+    fields = (
+        "payable_object",
+        "payable_model",
+        "object_id",
+        "moneybird_invoice_id",
+        "needs_synchronization",
+        "needs_deletion",
+    )
+
+    readonly_fields = ("payable_object", "needs_synchronization", "needs_deletion")
+
+    search_fields = (
+        "payable_model__app_label",
+        "payable_model__model",
+        "moneybird_invoice_id",
+    )
+
+    list_filter = (
+        "needs_synchronization",
+        "needs_deletion",
+        ("payable_model", RelatedOnlyFieldListFilter),
+    )
 
 
 @admin.register(MoneybirdPayment)
 class MoneybirdPaymentAdmin(admin.ModelAdmin):
-    list_display = ("payment",)
+    """Manage moneybird payments."""
+
+    list_display = (
+        "payment",
+        "moneybird_financial_statement_id",
+        "moneybird_financial_mutation_id",
+    )
+
+    fields = (
+        "payment",
+        "moneybird_financial_statement_id",
+        "moneybird_financial_mutation_id",
+    )
+
+    raw_id_fields = ("payment",)
+
+    search_fields = (
+        "payment__amount",
+        "moneybird_financial_mutation_id",
+        "moneybird_financial_statement_id",
+    )
+
+    def get_readonly_fields(self, request, obj=None):
+        if not obj:
+            return ()
+        return ("payment",)
diff --git a/website/moneybirdsynchronization/administration.py b/website/moneybirdsynchronization/administration.py
@@ -4,18 +4,21 @@
 licensed under the MIT license. The source code of moneybird-python
 can be found on GitHub: https://github.com/jjkester/moneybird-python.
 """
+import functools
 import json
 import logging
+import time
 from abc import ABC, abstractmethod
+from datetime import datetime
 from functools import reduce
 from typing import Optional, Union
 from urllib.parse import urljoin
 
-from django.utils import timezone
+from django.conf import settings
 
 import requests
 
-from thaliawebsite import settings
+logger = logging.getLogger(__name__)
 
 
 class Administration(ABC):
@@ -68,6 +71,8 @@ class InvalidData(Error):
     class Throttled(Error):
         """The client sent too many requests."""
 
+        retry_after: int
+
     class ServerError(Error):
         """An error happened on the server."""
 
@@ -90,10 +95,10 @@ def _build_url(self, resource_path: str) -> str:
         return reduce(urljoin, url_parts)
 
     def _process_response(self, response: requests.Response) -> Union[dict, None]:
-        logging.debug(f"Response {response.status_code}: {response.text}")
+        logger.debug(f"Response {response.status_code}: {response.text}")
 
         if response.next:
-            logging.debug(f"Received paginated response: {response.next}")
+            logger.debug(f"Received paginated response: {response.next}")
 
         good_codes = {200, 201, 204}
         bad_codes = {
@@ -112,25 +117,31 @@ def _process_response(self, response: requests.Response) -> Union[dict, None]:
         code_is_known: bool = code in good_codes | bad_codes.keys()
 
         if not code_is_known:
-            logging.warning(f"Unknown response code {code}")
+            logger.warning(f"Unknown response code {code}")
             raise Administration.Error(
                 code, "API response contained unknown status code"
             )
 
         if code in bad_codes:
             error = bad_codes[code]
             if error == Administration.Throttled:
-                throttled_retry_after = response.headers.get("Retry-After")
-                error_description = f"Retry after {timezone.datetime.fromtimestamp(float(throttled_retry_after)):'%Y-%m-%d %H:%M:%S'}"
+                e = Administration.Throttled(code, "Throttled")
+                e.retry_after = response.headers.get("Retry-After")
+                e.rate_limit_remaining = response.headers.get("RateLimit-Remaining")
+                e.rate_limit_limit = response.headers.get("RateLimit-Limit")
+                e.rate_limit_reset = response.headers.get("RateLimit-Reset")
+                error_description = f"retry after {e.retry_after}"
             else:
                 try:
                     error_description = response.json()["error"]
                 except (AttributeError, TypeError, KeyError, ValueError):
                     error_description = None
 
-            logging.warning(f"API error {code}: {error_description}")
+                e = error(code, error_description)
+
+            logger.warning(f"API error {code}: {e}")
 
-            raise error(code, error_description)
+            raise e
 
         if code == 204:
             return {}
@@ -141,6 +152,33 @@ def _process_response(self, response: requests.Response) -> Union[dict, None]:
         return response.json()
 
 
+def _retry_if_throttled():
+    max_retries = 3
+
+    def decorator_retry(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            retries = 0
+            while retries < max_retries:
+                try:
+                    return func(*args, **kwargs)
+                except HttpsAdministration.Throttled as e:
+                    retries += 1
+                    retry_after = datetime.fromtimestamp(float(e.retry_after))
+                    now = datetime.now()
+                    sleep_seconds = int(retry_after.timestamp() - now.timestamp()) + 1
+                    if retries < max_retries:
+                        logger.info(f"Retrying in {sleep_seconds} seconds...")
+                        time.sleep(sleep_seconds)
+                    else:
+                        logger.warning("Max retries reached. Giving up.")
+            return None
+
+        return wrapper
+
+    return decorator_retry
+
+
 class HttpsAdministration(Administration):
     """The HTTPS implementation of the MoneyBird Administration interface."""
 
@@ -155,33 +193,37 @@ def _create_session(self) -> requests.Session:
         session.headers.update({"Authorization": f"Bearer {self.key}"})
         return session
 
+    @_retry_if_throttled()
     def get(self, resource_path: str, params: Optional[dict] = None):
         """Do a GET on the Moneybird administration."""
         url = self._build_url(resource_path)
-        logging.debug(f"GET {url} {params}")
+        logger.debug(f"GET {url} {params}")
         response = self.session.get(url, params=params)
         return self._process_response(response)
 
+    @_retry_if_throttled()
     def post(self, resource_path: str, data: dict):
         """Do a POST request on the Moneybird administration."""
         url = self._build_url(resource_path)
         data = json.dumps(data)
-        logging.debug(f"POST {url} with {data}")
+        logger.debug(f"POST {url} with {data}")
         response = self.session.post(url, data=data)
         return self._process_response(response)
 
+    @_retry_if_throttled()
     def patch(self, resource_path: str, data: dict):
         """Do a PATCH request on the Moneybird administration."""
         url = self._build_url(resource_path)
         data = json.dumps(data)
-        logging.debug(f"PATCH {url} with {data}")
+        logger.debug(f"PATCH {url} with {data}")
         response = self.session.patch(url, data=data)
         return self._process_response(response)
 
+    @_retry_if_throttled()
     def delete(self, resource_path: str, data: Optional[dict] = None):
         """Do a DELETE on the Moneybird administration."""
         url = self._build_url(resource_path)
-        logging.debug(f"DELETE {url}")
+        logger.debug(f"DELETE {url}")
         response = self.session.delete(url, data=data)
         return self._process_response(response)
 

diff --git a/...te/moneybirdsynchronization/migrations/0005_moneybirdcontact_moneybird_sepa_mandate_id.py b/...te/moneybirdsynchronization/migrations/0005_moneybirdcontact_moneybird_sepa_mandate_id.py
@@ -0,0 +1,23 @@
+# Generated by Django 4.2.5 on 2023-10-04 18:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        ("moneybirdsynchronization", "0004_moneybirdproject"),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="moneybirdcontact",
+            name="moneybird_sepa_mandate_id",
+            field=models.CharField(
+                blank=True,
+                max_length=255,
+                null=True,
+                unique=True,
+                verbose_name="Moneybird SEPA mandate ID",
+            ),
+        ),
+    ]
diff --git a/...oneybirdsynchronization/migrations/0006_moneybirdexternalinvoice_needs_synchronization.py b/...oneybirdsynchronization/migrations/0006_moneybirdexternalinvoice_needs_synchronization.py
@@ -0,0 +1,42 @@
+# Generated by Django 4.2.5 on 2023-10-20 09:12
+
+from django.db import migrations, models
+
+
+def set_needs_synchronization_false(apps, schema_editor):
+    MoneybirdExternalInvoice = apps.get_model(
+        "moneybirdsynchronization", "MoneybirdExternalInvoice"
+    )
+    MoneybirdExternalInvoice.objects.all().update(needs_synchronization=False)
+
+
+class Migration(migrations.Migration):
+    dependencies = [
+        (
+            "moneybirdsynchronization",
+            "0005_moneybirdcontact_moneybird_sepa_mandate_id",
+        ),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name="moneybirdexternalinvoice",
+            name="needs_synchronization",
+            field=models.BooleanField(
+                default=True,
+                help_text="Indicates that the invoice has to be synchronized (again).",
+            ),
+        ),
+        migrations.AddField(
+            model_name="moneybirdexternalinvoice",
+            name="needs_deletion",
+            field=models.BooleanField(
+                default=False,
+                help_text="Indicates that the invoice has to be deleted from moneybird.",
+            ),
+        ),
+        migrations.RunPython(
+            set_needs_synchronization_false,
+            reverse_code=migrations.RunPython.noop,
+        ),
+    ]