From df1237fbd7b1d3f44fe5567d0cc3a57eb2591f4c Mon Sep 17 00:00:00 2001 From: Lars Holm Nielsen Date: Thu, 8 Feb 2024 21:52:54 +0100 Subject: [PATCH] models: change tracking of domains --- invenio_accounts/api.py | 14 +++++++++ invenio_accounts/datastore.py | 13 +++++---- invenio_accounts/domains.py | 1 + invenio_accounts/models.py | 8 +++--- invenio_accounts/tasks.py | 54 +++++++++++++++++++---------------- 5 files changed, 57 insertions(+), 33 deletions(-) diff --git a/invenio_accounts/api.py b/invenio_accounts/api.py index e1c98a75..b462c811 100644 --- a/invenio_accounts/api.py +++ b/invenio_accounts/api.py @@ -16,8 +16,10 @@ def __init__(self): """Constructor.""" self.updated_users = set() self.updated_roles = set() + self.updated_domains = set() self.deleted_users = set() self.deleted_roles = set() + self.deleted_domains = set() class DBUsersChangeHistory: @@ -43,6 +45,12 @@ def add_updated_role(self, session_id, role_id): session = self._get_session(session_id) session.updated_roles.add(role_id) + def add_updated_domain(self, session_id, domain_id): + """Adds a user to the updated domains list.""" + assert domain_id is not None + session = self._get_session(session_id) + session.updated_domains.add(domain_id) + def add_deleted_user(self, session_id, user_id): """Adds a user to the deleted users list.""" assert user_id is not None @@ -55,6 +63,12 @@ def add_deleted_role(self, session_id, role_id): session = self._get_session(session_id) session.deleted_roles.add(role_id) + def add_deleted_domain(self, session_id, domain_id): + """Adds a role to the deleted domain list.""" + assert domain_id is not None + session = self._get_session(session_id) + session.deleted_domains.add(domain_id) + def clear_dirty_sets(self, session): """Removes session object.""" sid = id(session) diff --git a/invenio_accounts/datastore.py b/invenio_accounts/datastore.py index 5bad177d..95e0b6e1 100644 --- a/invenio_accounts/datastore.py +++ b/invenio_accounts/datastore.py @@ -12,6 +12,7 @@ from flask import current_app from flask_security import SQLAlchemyUserDatastore, user_confirmed +from sqlalchemy.orm import joinedload from .models import Domain, Role, User from .proxies import current_db_change_history @@ -27,10 +28,9 @@ def verify_user(self, user): now = datetime.utcnow() user.blocked_at = None user.verified_at = now + user.active = True if user.confirmed_at is None: user.confirmed_at = now - if not user.active: - user.active = True return True def block_user(self, user): @@ -38,8 +38,7 @@ def block_user(self, user): now = datetime.utcnow() user.blocked_at = now user.verified_at = None - if user.active: - user.active = False + user.active = False delete_user_sessions(user) return True @@ -113,7 +112,11 @@ def find_role_by_id(self, role_id): def find_domain(self, domain): """Find a domain.""" - return Domain.query.filter_by(domain=domain).one_or_none() + return ( + Domain.query.filter_by(domain=domain) + .options(joinedload(Domain.category_name)) + .one_or_none() + ) def create_domain(self, domain, **kwargs): """Create a new domain.""" diff --git a/invenio_accounts/domains.py b/invenio_accounts/domains.py index 640e98cc..4652afef 100644 --- a/invenio_accounts/domains.py +++ b/invenio_accounts/domains.py @@ -20,6 +20,7 @@ def on_user_confirmed(app, user): domain = datastore.find_domain(user.domain) if domain is None: domain = datastore.create_domain(user.domain) + datastore.mark_changed(id(datastore.db.session), model=domain) # Verify user if domain is verified. if domain.status == DomainStatus.verified: diff --git a/invenio_accounts/models.py b/invenio_accounts/models.py index 9955268c..1fed7ad5 100644 --- a/invenio_accounts/models.py +++ b/invenio_accounts/models.py @@ -498,9 +498,6 @@ class DomainOrg(db.Model): parent = db.relationship("DomainOrg", remote_side=[id]) """Relationship to parent.""" - domains = db.relationship("Domain", back_populates="org") - """Relationship to domains for this organisation.""" - @classmethod def create(cls, pid, name, json=None, parent=None): """Create a domain organisation.""" @@ -564,12 +561,15 @@ class Domain(db.Model, Timestamp): org_id = db.Column(db.Integer(), db.ForeignKey(DomainOrg.id), nullable=True) """Organisation associated with domain.""" - org = db.relationship("DomainOrg", back_populates="domains") + org = db.relationship("DomainOrg", backref="domains") # spammer, mail-provider, organisation, company category = db.Column(db.Integer(), db.ForeignKey(DomainCategory.id), nullable=True) """Category of domain.""" + category_name = db.relationship("DomainCategory", backref="domains") + """Relationship to category""" + num_users = db.Column(db.Integer(), default=0, nullable=False) """Computed property to store number of users in domain.""" diff --git a/invenio_accounts/tasks.py b/invenio_accounts/tasks.py index 9fd92b54..2133f2e0 100644 --- a/invenio_accounts/tasks.py +++ b/invenio_accounts/tasks.py @@ -122,27 +122,33 @@ def update_domain_status(): # If statistics are updated regularly, the number of updates is relatively # low and hence fit in memory. We read all data first, to avoid starting # to modify the same table we're reading from. - domain_updates = [] - for row in stmt.all(): - domain_updates.append(row) - - # Update the database - i = 0 - for row in domain_updates: - domain, users, active, inactive, confirmed, verified, blocked = row - db.session.query(Domain).filter(Domain.domain == domain).update( - { - "num_users": users, - "num_active": active, - "num_inactive": inactive, - "num_confirmed": confirmed, - "num_verified": verified, - "num_blocked": blocked, - } - ) - i += 1 - # Commit batches of 500 updates - if i % 500 == 0: - db.session.commit() - if i % 500 != 0: - db.session.commit() + domain_updates = list(stmt.all()) + + # Commit batches of 500 updates + batch_size = 500 + now = datetime.utcnow() + + # Process updates in batches + for i in range(0, len(domain_updates), batch_size): + with db.session.begin_nested(): # Use nested transactions for safety + for ( + domain, + users, + active, + inactive, + confirmed, + verified, + blocked, + ) in domain_updates[i : i + batch_size]: + db.session.query(Domain).filter(Domain.domain == domain).update( + { + "num_users": users, + "num_active": active, + "num_inactive": inactive, + "num_confirmed": confirmed, + "num_verified": verified, + "num_blocked": blocked, + "updated": now, + } + ) + db.session.commit() # Commit after each batch