global: improved domain/user indexing, data flow, search and REST API

.github/workflows/tests.yml

@@ -38,7 +38,7 @@ jobs:
         cache-service: [redis]
         db-service: [postgresql14]
         mq-service: [rabbitmq]
-        search-service: [opensearch2, elasticsearch7]
+        search-service: [opensearch2]
 
     env:
       CACHE: ${{ matrix.cache-service }}

docs/conf.py

@@ -333,3 +333,6 @@
 
 # Autodoc configuraton.
 autoclass_content = "both"
+
+
+nitpick_ignore = [("py:class", "types.StrSequenceOrSet")]

invenio_users_resources/config.py

@@ -10,7 +10,9 @@
 """Invenio module providing management APIs for users and roles/groups."""
 
 from invenio_i18n import lazy_gettext as _
+from marshmallow import Schema, fields, validate
 
+from invenio_users_resources.services.domains import facets as domainfacets
 from invenio_users_resources.services.schemas import UserSchema
 from invenio_users_resources.services.users import facets
 
@@ -48,36 +50,54 @@
 """Schema used by the users service."""
 
 USERS_RESOURCES_SEARCH = {
-    "sort": [
-        "email",
-        "username",
-        "email_domain",
-    ],
-    "facets": ["email_domain", "affiliations"],
+    "sort": ["bestmatch", "username", "email", "domain", "newest", "oldest", "updated"],
+    "facets": ["status", "visibility", "domain_status", "domain", "affiliations"],
 }
-"""User search configuration (i.e list of banners)."""
+"""User search configuration."""
 
 USERS_RESOURCES_SORT_OPTIONS = {
+    "bestmatch": dict(
+        title=_("Best match"),
+        fields=["_score"],
+    ),
     "username": dict(
         title=_("Username"),
-        fields=["username"],
+        fields=["username", "-created"],
     ),
     "email": dict(
         title=_("Email"),
-        fields=["email"],
+        fields=["email_hidden", "-created"],
+    ),
+    "domain": dict(
+        title=_("Domain"),
+        fields=["domain", "-created"],
     ),
-    "email_domain": dict(
-        title=_("Email domain"),
-        fields=["email.domain"],
+    "newest": dict(
+        title=_("Newest"),
+        fields=["-created"],
+    ),
+    "oldest": dict(
+        title=_("Oldest"),
+        fields=["created"],
+    ),
+    "updated": dict(
+        title=_("Recently updated"),
+        fields=["-updated"],
     ),
 }
 """Definitions of available Users sort options. """
 
 USERS_RESOURCES_SEARCH_FACETS = {
-    "email_domain": {
-        "facet": facets.email_domain,
+    "domain": {
+        "facet": facets.domain,
+        "ui": {
+            "field": "domain",
+        },
+    },
+    "domain_status": {
+        "facet": facets.domain_status,
         "ui": {
-            "field": "email.domain",
+            "field": "domain_status",
         },
     },
     "affiliations": {
@@ -86,6 +106,18 @@
             "field": "profile.affiliations.keyword",
         },
     },
+    "status": {
+        "facet": facets.status,
+        "ui": {
+            "field": "status",
+        },
+    },
+    "visibility": {
+        "facet": facets.visibility,
+        "ui": {
+            "field": "visibility",
+        },
+    },
 }
 """Invenio requests facets."""
 
@@ -97,3 +129,114 @@
 
 USERS_RESOURCES_MODERATION_LOCK_RENEWAL_TIMEOUT = 120
 """Renewal timeout, in seconds, to increase the lock time for a user when moderating."""
+
+
+USERS_RESOURCES_DOMAINS_SEARCH = {
+    "sort": [
+        "bestmatch",
+        "domain",
+        "newest",
+        "oldest",
+        "updated",
+        "num-users",
+        "num-active",
+        "num-inactive",
+        "num-confirmed",
+        "num-verified",
+        "num-blocked",
+    ],
+    "facets": ["status", "flagged", "category", "organisation", "tld"],
+}
+"""User search configuration."""
+
+USERS_RESOURCES_DOMAINS_SORT_OPTIONS = {
+    "bestmatch": dict(
+        title=_("Best match"),
+        fields=["_score"],
+    ),
+    "domain": dict(
+        title=_("Domain"),
+        fields=["domain", "-created"],
+    ),
+    "newest": dict(
+        title=_("Newest"),
+        fields=["-created"],
+    ),
+    "oldest": dict(
+        title=_("Oldest"),
+        fields=["created"],
+    ),
+    "updated": dict(
+        title=_("Recently updated"),
+        fields=["-updated"],
+    ),
+    "num-users": dict(
+        title=_("# Users"),
+        fields=["-num_users"],
+    ),
+    "num-active": dict(
+        title=_("# Active"),
+        fields=["-num_active"],
+    ),
+    "num-inactive": dict(
+        title=_("# Inactive"),
+        fields=["-num_inactive"],
+    ),
+    "num-confirmed": dict(
+        title=_("# Confirmed"),
+        fields=["-num_confirmed"],
+    ),
+    "num-verified": dict(
+        title=_("# Verified"),
+        fields=["-num_verified"],
+    ),
+    "num-blocked": dict(
+        title=_("# Blocked"),
+        fields=["-num_blocked"],
+    ),
+}
+"""Definitions of available Users sort options. """
+
+USERS_RESOURCES_DOMAINS_SEARCH_FACETS = {
+    "status": {
+        "facet": domainfacets.status,
+        "ui": {
+            "field": "status",
+        },
+    },
+    "flagged": {
+        "facet": domainfacets.flagged,
+        "ui": {
+            "field": "flagged",
+        },
+    },
+    "category": {
+        "facet": domainfacets.category,
+        "ui": {
+            "field": "category",
+        },
+    },
+    "organisation": {
+        "facet": domainfacets.organisation,
+        "ui": {
+            "field": "organisation",
+        },
+    },
+    "tld": {
+        "facet": domainfacets.tld,
+        "ui": {
+            "field": "tld",
+        },
+    },
+}
+"""Invenio domains facets."""
+
+
+class OrgPropsSchema(Schema):
+    """Schema for validating domain org properties."""
+
+    country = fields.String(validate=validate.Length(min=2, max=3))
+
+
+USERS_RESOURCES_DOMAINS_ORG_SCHEMA = OrgPropsSchema
+"""Domains organisation schema config."""

invenio_users_resources/ext.py

@@ -18,12 +18,16 @@
 from . import config
 from .records.hooks import post_commit, pre_commit
 from .resources import (
+    DomainsResource,
+    DomainsResourceConfig,
     GroupsResource,
     GroupsResourceConfig,
     UsersResource,
     UsersResourceConfig,
 )
 from .services import (
+    DomainsService,
+    DomainsServiceConfig,
     GroupsService,
     GroupsServiceConfig,
     UsersService,
@@ -58,6 +62,7 @@ def init_services(self, app):
         """Initialize the services for users and user groups."""
         self.users_service = UsersService(config=UsersServiceConfig.build(app))
         self.groups_service = GroupsService(config=GroupsServiceConfig)
+        self.domains_service = DomainsService(config=DomainsServiceConfig.build(app))
 
     def init_resources(self, app):
         """Initialize the resources for users and user groups."""
@@ -70,6 +75,11 @@ def init_resources(self, app):
             config=GroupsResourceConfig,
         )
 
+        self.domains_resource = DomainsResource(
+            service=self.domains_service,
+            config=DomainsResourceConfig,
+        )
+
     def init_db_hooks(self):
         """Initialize the database hooks for reindexing updated users/roles."""
         # make sure that the hooks are only registered once per DB connection

invenio_users_resources/proxies.py

@@ -26,6 +26,11 @@
 )
 """Proxy for the currently instantiated user groups service."""
 
+current_domains_service = LocalProxy(
+    lambda: current_app.extensions["invenio-users-resources"].domains_service
+)
+"""Proxy for the currently instantiated user groups service."""
+
 current_actions_registry = LocalProxy(
     lambda: current_app.extensions["invenio-users-resources"].actions_registry
 )