global: domains rest api and underlying service

Co-authored-by: Karolina Przerwa <[email protected]>
Co-authored-by: Sam Arbid <[email protected]>

.github/workflows/tests.yml

@@ -38,7 +38,7 @@ jobs:
         cache-service: [redis]
         db-service: [postgresql14]
         mq-service: [rabbitmq]
-        search-service: [opensearch2, elasticsearch7]
+        search-service: [opensearch2]
 
     env:
       CACHE: ${{ matrix.cache-service }}

docs/conf.py

@@ -333,3 +333,6 @@
 
 # Autodoc configuraton.
 autoclass_content = "both"
+
+
+nitpick_ignore = [("py:class", "types.StrSequenceOrSet")]

invenio_users_resources/config.py

@@ -10,7 +10,9 @@
 """Invenio module providing management APIs for users and roles/groups."""
 
 from invenio_i18n import lazy_gettext as _
+from marshmallow import Schema, fields, validate
 
+from invenio_users_resources.services.domains import facets as domainfacets
 from invenio_users_resources.services.schemas import UserSchema
 from invenio_users_resources.services.users import facets
 
@@ -127,3 +129,114 @@
 
 USERS_RESOURCES_MODERATION_LOCK_RENEWAL_TIMEOUT = 120
 """Renewal timeout, in seconds, to increase the lock time for a user when moderating."""
+
+
+USERS_RESOURCES_DOMAINS_SEARCH = {
+    "sort": [
+        "bestmatch",
+        "domain",
+        "newest",
+        "oldest",
+        "updated",
+        "num-users",
+        "num-active",
+        "num-inactive",
+        "num-confirmed",
+        "num-verified",
+        "num-blocked",
+    ],
+    "facets": ["status", "flagged", "category", "organisation", "tld"],
+}
+"""User search configuration."""
+
+USERS_RESOURCES_DOMAINS_SORT_OPTIONS = {
+    "bestmatch": dict(
+        title=_("Best match"),
+        fields=["_score"],
+    ),
+    "domain": dict(
+        title=_("Domain"),
+        fields=["domain", "-created"],
+    ),
+    "newest": dict(
+        title=_("Newest"),
+        fields=["-created"],
+    ),
+    "oldest": dict(
+        title=_("Oldest"),
+        fields=["created"],
+    ),
+    "updated": dict(
+        title=_("Recently updated"),
+        fields=["-updated"],
+    ),
+    "num-users": dict(
+        title=_("# Users"),
+        fields=["-num_users"],
+    ),
+    "num-active": dict(
+        title=_("# Active"),
+        fields=["-num_active"],
+    ),
+    "num-inactive": dict(
+        title=_("# Inactive"),
+        fields=["-num_inactive"],
+    ),
+    "num-confirmed": dict(
+        title=_("# Confirmed"),
+        fields=["-num_confirmed"],
+    ),
+    "num-verified": dict(
+        title=_("# Verified"),
+        fields=["-num_verified"],
+    ),
+    "num-blocked": dict(
+        title=_("# Blocked"),
+        fields=["-num_blocked"],
+    ),
+}
+"""Definitions of available Users sort options. """
+
+USERS_RESOURCES_DOMAINS_SEARCH_FACETS = {
+    "status": {
+        "facet": domainfacets.status,
+        "ui": {
+            "field": "status",
+        },
+    },
+    "flagged": {
+        "facet": domainfacets.flagged,
+        "ui": {
+            "field": "flagged",
+        },
+    },
+    "category": {
+        "facet": domainfacets.category,
+        "ui": {
+            "field": "category",
+        },
+    },
+    "organisation": {
+        "facet": domainfacets.organisation,
+        "ui": {
+            "field": "organisation",
+        },
+    },
+    "tld": {
+        "facet": domainfacets.tld,
+        "ui": {
+            "field": "tld",
+        },
+    },
+}
+"""Invenio domains facets."""
+
+
+class OrgPropsSchema(Schema):
+    """Schema for validating domain org properties."""
+
+    country = fields.String(validate=validate.Length(min=2, max=3))
+
+
+USERS_RESOURCES_DOMAINS_ORG_SCHEMA = OrgPropsSchema
+"""Domains organisation schema config."""

invenio_users_resources/ext.py

@@ -18,12 +18,16 @@
 from . import config
 from .records.hooks import post_commit, pre_commit
 from .resources import (
+    DomainsResource,
+    DomainsResourceConfig,
     GroupsResource,
     GroupsResourceConfig,
     UsersResource,
     UsersResourceConfig,
 )
 from .services import (
+    DomainsService,
+    DomainsServiceConfig,
     GroupsService,
     GroupsServiceConfig,
     UsersService,
@@ -58,6 +62,7 @@ def init_services(self, app):
         """Initialize the services for users and user groups."""
         self.users_service = UsersService(config=UsersServiceConfig.build(app))
         self.groups_service = GroupsService(config=GroupsServiceConfig)
+        self.domains_service = DomainsService(config=DomainsServiceConfig.build(app))
 
     def init_resources(self, app):
         """Initialize the resources for users and user groups."""
@@ -70,6 +75,11 @@ def init_resources(self, app):
             config=GroupsResourceConfig,
         )
 
+        self.domains_resource = DomainsResource(
+            service=self.domains_service,
+            config=DomainsResourceConfig,
+        )
+
     def init_db_hooks(self):
         """Initialize the database hooks for reindexing updated users/roles."""
         # make sure that the hooks are only registered once per DB connection

invenio_users_resources/proxies.py

@@ -26,6 +26,11 @@
 )
 """Proxy for the currently instantiated user groups service."""
 
+current_domains_service = LocalProxy(
+    lambda: current_app.extensions["invenio-users-resources"].domains_service
+)
+"""Proxy for the currently instantiated user groups service."""
+
 current_actions_registry = LocalProxy(
     lambda: current_app.extensions["invenio-users-resources"].actions_registry
 )

invenio_users_resources/records/api.py

@@ -10,27 +10,62 @@
 """API classes for user and group management in Invenio."""
 
 import unicodedata
+from collections import namedtuple
 from datetime import datetime
 
 from flask import current_app
+from invenio_accounts.models import Domain
 from invenio_accounts.proxies import current_datastore
 from invenio_db import db
-from invenio_records.dumpers import SearchDumper
+from invenio_records.dumpers import SearchDumper, SearchDumperExt
 from invenio_records.dumpers.indexedat import IndexedAtDumperExt
 from invenio_records.systemfields import ModelField
 from invenio_records_resources.records.api import Record
 from invenio_records_resources.records.systemfields import IndexField
+from sqlalchemy.exc import NoResultFound
 
 from .dumpers import EmailFieldDumperExt
-from .models import GroupAggregateModel, UserAggregateModel
+from .models import DomainAggregateModel, GroupAggregateModel, UserAggregateModel
 from .systemfields import (
     AccountStatusField,
     AccountVisibilityField,
+    DomainCategoryNameField,
     DomainField,
+    DomainOrgField,
+    DomainStatusNameField,
     IsNotNoneField,
     UserIdentitiesField,
 )
 
+EmulatedPID = namedtuple("EmulatedPID", ["pid_value"])
+"""Emulated PID"""
+
+
+class AggregatePID:
+    """Helper emulate a PID field."""
+
+    def __init__(self, pid_field):
+        """Constructor."""
+        self._pid_field = pid_field
+
+    def __get__(self, record, owner=None):
+        """Evaluate the property."""
+        if record is None:
+            return GetRecordResolver(owner)
+        return EmulatedPID(record[self._pid_field])
+
+
+class GetRecordResolver(object):
+    """Resolver that simply uses get record."""
+
+    def __init__(self, record_cls):
+        """Initialize resolver."""
+        self._record_cls = record_cls
+
+    def resolve(self, pid_value):
+        """Simply get the record."""
+        return self._record_cls.get_record(pid_value)
+
 
 class BaseAggregate(Record):
     """An aggregate of information about a user group/role."""
@@ -67,6 +102,10 @@ def commit(self):
         # You can only commit if you have an underlying model object.
         if self.model._model_obj is None:
             raise ValueError(f"{self.__class__.__name__} not backed by a model.")
+        if self.model._model_obj not in db.session:
+            with db.session.begin_nested():
+                # make sure we get an id assigned
+                db.session.add(self.model._model_obj)
         # Basically re-parses the model object.
         model = self.model_cls(model_obj=self.model._model_obj)
         self.model = model
@@ -285,3 +324,112 @@ def get_record_by_name(cls, name):
             if role is None:
                 return None
             return cls.from_model(role)
+
+
+class OrgNameDumperExt(SearchDumperExt):
+    """Custom fields dumper extension."""
+
+    def dump(self, record, data):
+        """Dump for faceting."""
+        org = data.get("org", None)
+        if org and len(org) > 0:
+            data["org_names"] = [o["name"] for o in org]
+
+    def load(self, data, record_cls):
+        """Remove data from object."""
+        data.pop("org_names", None)
+
+
+class DomainAggregate(BaseAggregate):
+    """An aggregate of information about a user."""
+
+    model_cls = DomainAggregateModel
+    """The model class for the request."""
+
+    # NOTE: the "uuid" isn't a UUID but contains the same value as the "id"
+    #       field, which is currently an integer for User objects!
+    dumper = SearchDumper(
+        extensions=[
+            IndexedAtDumperExt(),
+            OrgNameDumperExt(),
+        ],
+        model_fields={
+            "id": ("uuid", int),
+        },
+    )
+    """Search dumper with configured extensions."""
+
+    index = IndexField("domains-domain-v1.0.0", search_alias="domains")
+    """The search engine index to use."""
+
+    pid = AggregatePID("domain")
+    """Needed to emulate pid access."""
+
+    id = ModelField("id", dump_type=int)
+    """The user identifier."""
+
+    domain = ModelField("domain", dump_type=str)
+    """The domain of the users' email address."""
+
+    tld = ModelField("tld", dump_type=str)
+    """Top level domain."""
+
+    status = ModelField("status", dump_type=int)
+    """Domain status."""
+
+    status_name = DomainStatusNameField(index=True)
+    """Domain status name."""
+
+    flagged = ModelField("flagged", dump_type=bool)
+    """Flagged."""
+
+    flagged_source = ModelField("flagged_source", dump_type=str)
+    """Source of flagging."""
+
+    category = ModelField("category", dump_type=int)
+    """Domain category."""
+
+    category_name = DomainCategoryNameField(use_cache=True, index=True)
+    """Domain category name."""
+
+    org_id = ModelField("org_id", dump_type=int)
+    """Number of users."""
+
+    org = DomainOrgField("org", use_cache=True, index=True)
+    """Organization behind the domain."""
+
+    num_users = ModelField("num_users", dump_type=int)
+    """Number of users."""
+
+    num_active = ModelField("num_active", dump_type=int)
+    """Number of active users."""
+
+    num_inactive = ModelField("num_inactive", dump_type=int)
+    """Number of inactive users."""
+
+    num_confirmed = ModelField("num_confirmed", dump_type=int)
+    """Number of confirmed users."""
+
+    num_verified = ModelField("num_verified", dump_type=int)
+    """Number of verified users."""
+
+    num_blocked = ModelField("num_blocked", dump_type=int)
+    """Number of blocked users."""
+
+    @classmethod
+    def get_record(cls, id_):
+        """Get the user via the specified ID."""
+        with db.session.no_autoflush:
+            domain = current_datastore.find_domain(id_)
+        if domain is None:
+            raise NoResultFound()
+        return cls.from_model(domain)
+
+    @classmethod
+    def create(cls, data, id_=None, **kwargs):
+        """Create a domain."""
+        return DomainAggregate(data, model=DomainAggregateModel(model_obj=Domain()))
+
+    def delete(self, force=True):
+        """Delete the domain."""
+        db.session.delete(self.model.model_obj)

invenio_users_resources/records/dumpers/email.py

@@ -28,9 +28,7 @@ def dump(self, record, data):
         email_visible = record.preferences["email_visibility"]
         if email_visible == "public":
             data[self.field] = email
-            data[self.hidden_field] = email
-        else:
-            data[self.hidden_field] = email
+        data[self.hidden_field] = email
 
     def load(self, data, record_cls):
         """Load the data."""