From 520e2e0076faaef7b6651a8bd515fc5c6781949a Mon Sep 17 00:00:00 2001 From: Lars Holm Nielsen Date: Thu, 8 Feb 2024 21:53:29 +0100 Subject: [PATCH] global: domains rest api and underlying service Co-authored-by: Karolina Przerwa Co-authored-by: Sam Arbid <36583694+Samk13@users.noreply.github.com> --- .github/workflows/tests.yml | 2 +- docs/conf.py | 3 + invenio_users_resources/config.py | 113 ++++++++++ invenio_users_resources/ext.py | 10 + invenio_users_resources/proxies.py | 5 + invenio_users_resources/records/api.py | 152 +++++++++++++- .../records/dumpers/email.py | 4 +- invenio_users_resources/records/hooks.py | 31 ++- .../mappings/os-v1/domains/domain-v1.0.0.json | 105 ++++++++++ .../mappings/os-v2/domains/domain-v1.0.0.json | 105 ++++++++++ invenio_users_resources/records/models.py | 58 ++++++ .../records/systemfields/__init__.py | 62 +++++- invenio_users_resources/resources/__init__.py | 3 + .../resources/domains/__init__.py | 17 ++ .../resources/domains/config.py | 43 ++++ .../resources/domains/resource.py | 32 +++ .../resources/users/config.py | 7 +- invenio_users_resources/services/__init__.py | 3 + .../services/domains/__init__.py | 17 ++ .../services/domains/components.py | 56 +++++ .../services/domains/config.py | 103 ++++++++++ .../services/domains/facets.py | 51 +++++ .../services/domains/service.py | 26 +++ .../services/domains/tasks.py | 38 ++++ .../services/permissions.py | 10 + invenio_users_resources/services/schemas.py | 125 +++++++++++- .../services/users/config.py | 10 +- invenio_users_resources/views.py | 8 + setup.cfg | 2 + tests/conftest.py | 103 +++++++++- tests/resources/test_resources_domains.py | 193 ++++++++++++++++++ tests/services/users/test_service_users.py | 3 +- tests/services/users/test_user_moderation.py | 18 +- 33 files changed, 1485 insertions(+), 33 deletions(-) create mode 100644 invenio_users_resources/records/mappings/os-v1/domains/domain-v1.0.0.json create mode 100644 invenio_users_resources/records/mappings/os-v2/domains/domain-v1.0.0.json create mode 100644 invenio_users_resources/resources/domains/__init__.py create mode 100644 invenio_users_resources/resources/domains/config.py create mode 100644 invenio_users_resources/resources/domains/resource.py create mode 100644 invenio_users_resources/services/domains/__init__.py create mode 100644 invenio_users_resources/services/domains/components.py create mode 100644 invenio_users_resources/services/domains/config.py create mode 100644 invenio_users_resources/services/domains/facets.py create mode 100644 invenio_users_resources/services/domains/service.py create mode 100644 invenio_users_resources/services/domains/tasks.py create mode 100644 tests/resources/test_resources_domains.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ab99d0b..4fb6e59 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -38,7 +38,7 @@ jobs: cache-service: [redis] db-service: [postgresql14] mq-service: [rabbitmq] - search-service: [opensearch2, elasticsearch7] + search-service: [opensearch2] env: CACHE: ${{ matrix.cache-service }} diff --git a/docs/conf.py b/docs/conf.py index bfc9b3c..e8066cd 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -333,3 +333,6 @@ # Autodoc configuraton. autoclass_content = "both" + + +nitpick_ignore = [("py:class", "types.StrSequenceOrSet")] diff --git a/invenio_users_resources/config.py b/invenio_users_resources/config.py index 1747bc8..ff25de5 100644 --- a/invenio_users_resources/config.py +++ b/invenio_users_resources/config.py @@ -10,7 +10,9 @@ """Invenio module providing management APIs for users and roles/groups.""" from invenio_i18n import lazy_gettext as _ +from marshmallow import Schema, fields, validate +from invenio_users_resources.services.domains import facets as domainfacets from invenio_users_resources.services.schemas import UserSchema from invenio_users_resources.services.users import facets @@ -127,3 +129,114 @@ USERS_RESOURCES_MODERATION_LOCK_RENEWAL_TIMEOUT = 120 """Renewal timeout, in seconds, to increase the lock time for a user when moderating.""" + + +USERS_RESOURCES_DOMAINS_SEARCH = { + "sort": [ + "bestmatch", + "domain", + "newest", + "oldest", + "updated", + "num-users", + "num-active", + "num-inactive", + "num-confirmed", + "num-verified", + "num-blocked", + ], + "facets": ["status", "flagged", "category", "organisation", "tld"], +} +"""User search configuration.""" + +USERS_RESOURCES_DOMAINS_SORT_OPTIONS = { + "bestmatch": dict( + title=_("Best match"), + fields=["_score"], + ), + "domain": dict( + title=_("Domain"), + fields=["domain", "-created"], + ), + "newest": dict( + title=_("Newest"), + fields=["-created"], + ), + "oldest": dict( + title=_("Oldest"), + fields=["created"], + ), + "updated": dict( + title=_("Recently updated"), + fields=["-updated"], + ), + "num-users": dict( + title=_("# Users"), + fields=["-num_users"], + ), + "num-active": dict( + title=_("# Active"), + fields=["-num_active"], + ), + "num-inactive": dict( + title=_("# Inactive"), + fields=["-num_inactive"], + ), + "num-confirmed": dict( + title=_("# Confirmed"), + fields=["-num_confirmed"], + ), + "num-verified": dict( + title=_("# Verified"), + fields=["-num_verified"], + ), + "num-blocked": dict( + title=_("# Blocked"), + fields=["-num_blocked"], + ), +} +"""Definitions of available Users sort options. """ + +USERS_RESOURCES_DOMAINS_SEARCH_FACETS = { + "status": { + "facet": domainfacets.status, + "ui": { + "field": "status", + }, + }, + "flagged": { + "facet": domainfacets.flagged, + "ui": { + "field": "flagged", + }, + }, + "category": { + "facet": domainfacets.category, + "ui": { + "field": "category", + }, + }, + "organisation": { + "facet": domainfacets.organisation, + "ui": { + "field": "organisation", + }, + }, + "tld": { + "facet": domainfacets.tld, + "ui": { + "field": "tld", + }, + }, +} +"""Invenio domains facets.""" + + +class OrgPropsSchema(Schema): + """Schema for validating domain org properties.""" + + country = fields.String(validate=validate.Length(min=2, max=3)) + + +USERS_RESOURCES_DOMAINS_ORG_SCHEMA = OrgPropsSchema +"""Domains organisation schema config.""" diff --git a/invenio_users_resources/ext.py b/invenio_users_resources/ext.py index 91bbda0..817c605 100644 --- a/invenio_users_resources/ext.py +++ b/invenio_users_resources/ext.py @@ -18,12 +18,16 @@ from . import config from .records.hooks import post_commit, pre_commit from .resources import ( + DomainsResource, + DomainsResourceConfig, GroupsResource, GroupsResourceConfig, UsersResource, UsersResourceConfig, ) from .services import ( + DomainsService, + DomainsServiceConfig, GroupsService, GroupsServiceConfig, UsersService, @@ -58,6 +62,7 @@ def init_services(self, app): """Initialize the services for users and user groups.""" self.users_service = UsersService(config=UsersServiceConfig.build(app)) self.groups_service = GroupsService(config=GroupsServiceConfig) + self.domains_service = DomainsService(config=DomainsServiceConfig.build(app)) def init_resources(self, app): """Initialize the resources for users and user groups.""" @@ -70,6 +75,11 @@ def init_resources(self, app): config=GroupsResourceConfig, ) + self.domains_resource = DomainsResource( + service=self.domains_service, + config=DomainsResourceConfig, + ) + def init_db_hooks(self): """Initialize the database hooks for reindexing updated users/roles.""" # make sure that the hooks are only registered once per DB connection diff --git a/invenio_users_resources/proxies.py b/invenio_users_resources/proxies.py index bd0c3a5..3db4281 100644 --- a/invenio_users_resources/proxies.py +++ b/invenio_users_resources/proxies.py @@ -26,6 +26,11 @@ ) """Proxy for the currently instantiated user groups service.""" +current_domains_service = LocalProxy( + lambda: current_app.extensions["invenio-users-resources"].domains_service +) +"""Proxy for the currently instantiated user groups service.""" + current_actions_registry = LocalProxy( lambda: current_app.extensions["invenio-users-resources"].actions_registry ) diff --git a/invenio_users_resources/records/api.py b/invenio_users_resources/records/api.py index 375f72f..1d906e5 100644 --- a/invenio_users_resources/records/api.py +++ b/invenio_users_resources/records/api.py @@ -10,27 +10,62 @@ """API classes for user and group management in Invenio.""" import unicodedata +from collections import namedtuple from datetime import datetime from flask import current_app +from invenio_accounts.models import Domain from invenio_accounts.proxies import current_datastore from invenio_db import db -from invenio_records.dumpers import SearchDumper +from invenio_records.dumpers import SearchDumper, SearchDumperExt from invenio_records.dumpers.indexedat import IndexedAtDumperExt from invenio_records.systemfields import ModelField from invenio_records_resources.records.api import Record from invenio_records_resources.records.systemfields import IndexField +from sqlalchemy.exc import NoResultFound from .dumpers import EmailFieldDumperExt -from .models import GroupAggregateModel, UserAggregateModel +from .models import DomainAggregateModel, GroupAggregateModel, UserAggregateModel from .systemfields import ( AccountStatusField, AccountVisibilityField, + DomainCategoryNameField, DomainField, + DomainOrgField, + DomainStatusNameField, IsNotNoneField, UserIdentitiesField, ) +EmulatedPID = namedtuple("EmulatedPID", ["pid_value"]) +"""Emulated PID""" + + +class AggregatePID: + """Helper emulate a PID field.""" + + def __init__(self, pid_field): + """Constructor.""" + self._pid_field = pid_field + + def __get__(self, record, owner=None): + """Evaluate the property.""" + if record is None: + return GetRecordResolver(owner) + return EmulatedPID(record[self._pid_field]) + + +class GetRecordResolver(object): + """Resolver that simply uses get record.""" + + def __init__(self, record_cls): + """Initialize resolver.""" + self._record_cls = record_cls + + def resolve(self, pid_value): + """Simply get the record.""" + return self._record_cls.get_record(pid_value) + class BaseAggregate(Record): """An aggregate of information about a user group/role.""" @@ -67,6 +102,10 @@ def commit(self): # You can only commit if you have an underlying model object. if self.model._model_obj is None: raise ValueError(f"{self.__class__.__name__} not backed by a model.") + if self.model._model_obj not in db.session: + with db.session.begin_nested(): + # make sure we get an id assigned + db.session.add(self.model._model_obj) # Basically re-parses the model object. model = self.model_cls(model_obj=self.model._model_obj) self.model = model @@ -285,3 +324,112 @@ def get_record_by_name(cls, name): if role is None: return None return cls.from_model(role) + + +class OrgNameDumperExt(SearchDumperExt): + """Custom fields dumper extension.""" + + def dump(self, record, data): + """Dump for faceting.""" + org = data.get("org", None) + if org and len(org) > 0: + data["org_names"] = [o["name"] for o in org] + + def load(self, data, record_cls): + """Remove data from object.""" + data.pop("org_names", None) + + +class DomainAggregate(BaseAggregate): + """An aggregate of information about a user.""" + + model_cls = DomainAggregateModel + """The model class for the request.""" + + # NOTE: the "uuid" isn't a UUID but contains the same value as the "id" + # field, which is currently an integer for User objects! + dumper = SearchDumper( + extensions=[ + IndexedAtDumperExt(), + OrgNameDumperExt(), + ], + model_fields={ + "id": ("uuid", int), + }, + ) + """Search dumper with configured extensions.""" + + index = IndexField("domains-domain-v1.0.0", search_alias="domains") + """The search engine index to use.""" + + pid = AggregatePID("domain") + """Needed to emulate pid access.""" + + id = ModelField("id", dump_type=int) + """The user identifier.""" + + domain = ModelField("domain", dump_type=str) + """The domain of the users' email address.""" + + tld = ModelField("tld", dump_type=str) + """Top level domain.""" + + status = ModelField("status", dump_type=int) + """Domain status.""" + + status_name = DomainStatusNameField(index=True) + """Domain status name.""" + + flagged = ModelField("flagged", dump_type=bool) + """Flagged.""" + + flagged_source = ModelField("flagged_source", dump_type=str) + """Source of flagging.""" + + category = ModelField("category", dump_type=int) + """Domain category.""" + + category_name = DomainCategoryNameField(use_cache=True, index=True) + """Domain category name.""" + + org_id = ModelField("org_id", dump_type=int) + """Number of users.""" + + org = DomainOrgField("org", use_cache=True, index=True) + """Organization behind the domain.""" + + num_users = ModelField("num_users", dump_type=int) + """Number of users.""" + + num_active = ModelField("num_active", dump_type=int) + """Number of active users.""" + + num_inactive = ModelField("num_inactive", dump_type=int) + """Number of inactive users.""" + + num_confirmed = ModelField("num_confirmed", dump_type=int) + """Number of confirmed users.""" + + num_verified = ModelField("num_verified", dump_type=int) + """Number of verified users.""" + + num_blocked = ModelField("num_blocked", dump_type=int) + """Number of blocked users.""" + + @classmethod + def get_record(cls, id_): + """Get the user via the specified ID.""" + with db.session.no_autoflush: + domain = current_datastore.find_domain(id_) + if domain is None: + raise NoResultFound() + return cls.from_model(domain) + + @classmethod + def create(cls, data, id_=None, **kwargs): + """Create a domain.""" + return DomainAggregate(data, model=DomainAggregateModel(model_obj=Domain())) + + def delete(self, force=True): + """Delete the domain.""" + db.session.delete(self.model.model_obj) diff --git a/invenio_users_resources/records/dumpers/email.py b/invenio_users_resources/records/dumpers/email.py index 39c4cc0..4fe6c15 100644 --- a/invenio_users_resources/records/dumpers/email.py +++ b/invenio_users_resources/records/dumpers/email.py @@ -28,9 +28,7 @@ def dump(self, record, data): email_visible = record.preferences["email_visibility"] if email_visible == "public": data[self.field] = email - data[self.hidden_field] = email - else: - data[self.hidden_field] = email + data[self.hidden_field] = email def load(self, data, record_cls): """Load the data.""" diff --git a/invenio_users_resources/records/hooks.py b/invenio_users_resources/records/hooks.py index d5fe544..a115811 100644 --- a/invenio_users_resources/records/hooks.py +++ b/invenio_users_resources/records/hooks.py @@ -9,9 +9,10 @@ """Invenio users DB hooks.""" -from invenio_accounts.models import Role, User +from invenio_accounts.models import Domain, Role, User from invenio_accounts.proxies import current_db_change_history +from ..services.domains.tasks import delete_domains, reindex_domains from ..services.groups.tasks import reindex_groups, unindex_groups from ..services.users.tasks import reindex_users, unindex_users @@ -36,12 +37,16 @@ def pre_commit(sender, session): current_db_change_history.add_updated_user(sid, item.id) if isinstance(item, Role): current_db_change_history.add_updated_role(sid, item.id) + if isinstance(item, Domain): + current_db_change_history.add_updated_domain(sid, item.id) for item in deleted: if isinstance(item, User): current_db_change_history.add_deleted_user(sid, item.id) if isinstance(item, Role): current_db_change_history.add_deleted_role(sid, item.id) + if isinstance(item, Domain): + current_db_change_history.add_deleted_domain(sid, item.id) def post_commit(sender, session): @@ -54,14 +59,30 @@ def post_commit(sender, session): if current_db_change_history.sessions.get(sid): # Handle updates user_ids_updated = list(current_db_change_history.sessions[sid].updated_users) - reindex_users.delay(user_ids_updated) + if user_ids_updated: + reindex_users.delay(user_ids_updated) group_ids_updated = list(current_db_change_history.sessions[sid].updated_roles) - reindex_groups.delay(group_ids_updated) + if group_ids_updated: + reindex_groups.delay(group_ids_updated) + + domain_ids_updated = list( + current_db_change_history.sessions[sid].updated_domains + ) + if domain_ids_updated: + reindex_domains.delay(domain_ids_updated) # Handle deletes user_ids_deleted = list(current_db_change_history.sessions[sid].deleted_users) - unindex_users.delay(user_ids_deleted) + if user_ids_deleted: + unindex_users.delay(user_ids_deleted) group_ids_deleted = list(current_db_change_history.sessions[sid].deleted_roles) - unindex_groups.delay(group_ids_deleted) + if group_ids_deleted: + unindex_groups.delay(group_ids_deleted) + + domain_ids_deleted = list( + current_db_change_history.sessions[sid].deleted_domains + ) + if domain_ids_deleted: + delete_domains.delay(domain_ids_deleted) diff --git a/invenio_users_resources/records/mappings/os-v1/domains/domain-v1.0.0.json b/invenio_users_resources/records/mappings/os-v1/domains/domain-v1.0.0.json new file mode 100644 index 0000000..4c8e45e --- /dev/null +++ b/invenio_users_resources/records/mappings/os-v1/domains/domain-v1.0.0.json @@ -0,0 +1,105 @@ +{ + "mappings": { + "dynamic": "strict", + "dynamic_templates": [ + { + "org": { + "path_match": "org.props", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "uuid": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "version_id" : { + "type": "integer" + }, + "domain": { + "type": "keyword" + }, + "tld": { + "type": "keyword" + }, + "status": { + "type": "integer" + }, + "status_name": { + "type": "keyword" + }, + "flagged": { + "type": "boolean" + }, + "flagged_source": { + "type": "keyword" + }, + "org_names": { + "type": "keyword" + }, + "org_id": { + "type": "keyword" + }, + "org": { + "type" : "nested", + "properties": { + "id" : { + "type": "integer" + }, + "pid" : { + "type": "keyword" + }, + "name" : { + "type": "text" + }, + "props" : { + "type": "object", + "properties": {}, + "dynamic": true + }, + "is_parent" : { + "type": "boolean" + } + } + }, + "category": { + "type": "integer" + }, + "category_name": { + "type": "keyword" + }, + "num_users": { + "type": "integer" + }, + "num_active": { + "type": "integer" + }, + "num_inactive": { + "type": "integer" + }, + "num_confirmed": { + "type": "integer" + }, + "num_verified": { + "type": "integer" + }, + "num_blocked": { + "type": "integer" + }, + "created": { + "type": "date" + }, + "updated": { + "type": "date" + }, + "indexed_at": { + "type": "date" + } + } + } +} diff --git a/invenio_users_resources/records/mappings/os-v2/domains/domain-v1.0.0.json b/invenio_users_resources/records/mappings/os-v2/domains/domain-v1.0.0.json new file mode 100644 index 0000000..4c8e45e --- /dev/null +++ b/invenio_users_resources/records/mappings/os-v2/domains/domain-v1.0.0.json @@ -0,0 +1,105 @@ +{ + "mappings": { + "dynamic": "strict", + "dynamic_templates": [ + { + "org": { + "path_match": "org.props", + "mapping": { + "type": "keyword" + } + } + } + ], + "properties": { + "uuid": { + "type": "integer" + }, + "id": { + "type": "integer" + }, + "version_id" : { + "type": "integer" + }, + "domain": { + "type": "keyword" + }, + "tld": { + "type": "keyword" + }, + "status": { + "type": "integer" + }, + "status_name": { + "type": "keyword" + }, + "flagged": { + "type": "boolean" + }, + "flagged_source": { + "type": "keyword" + }, + "org_names": { + "type": "keyword" + }, + "org_id": { + "type": "keyword" + }, + "org": { + "type" : "nested", + "properties": { + "id" : { + "type": "integer" + }, + "pid" : { + "type": "keyword" + }, + "name" : { + "type": "text" + }, + "props" : { + "type": "object", + "properties": {}, + "dynamic": true + }, + "is_parent" : { + "type": "boolean" + } + } + }, + "category": { + "type": "integer" + }, + "category_name": { + "type": "keyword" + }, + "num_users": { + "type": "integer" + }, + "num_active": { + "type": "integer" + }, + "num_inactive": { + "type": "integer" + }, + "num_confirmed": { + "type": "integer" + }, + "num_verified": { + "type": "integer" + }, + "num_blocked": { + "type": "integer" + }, + "created": { + "type": "date" + }, + "updated": { + "type": "date" + }, + "indexed_at": { + "type": "date" + } + } + } +} diff --git a/invenio_users_resources/records/models.py b/invenio_users_resources/records/models.py index 85ea096..92bc756 100644 --- a/invenio_users_resources/records/models.py +++ b/invenio_users_resources/records/models.py @@ -12,6 +12,7 @@ from flask import current_app from invenio_accounts.proxies import current_datastore +from invenio_accounts.utils import DomainStatus from invenio_db import db @@ -179,3 +180,60 @@ def model_obj(self): with db.session.no_autoflush: self._model_obj = current_datastore.find_role(name) return self._model_obj + + +class DomainAggregateModel(AggregateMetadata): + """Mock model for glueing together various parts of user group info.""" + + _properties = [ + "category", + "created", + "domain", + "flagged_source", + "flagged", + "id", + "num_active", + "num_blocked", + "num_confirmed", + "num_inactive", + "num_users", + "num_verified", + "org_id", + "status", + "tld", + "updated", + "version_id", + ] + """Properties of this object that can be accessed.""" + + _set_properties = [ + "category", + "domain", + "flagged_source", + "flagged", + "org_id", + "status", + "tld", + ] + """Properties of this object that can be set.""" + + def from_model(self, domain): + """Extract information from a user object.""" + super().from_model(domain) + # Hardcoding version id to 1 since domain model doesn't have + # a version id because we update the table often outside + # of sqlalchemy ORM. + self._data["version_id"] = 1 + # Convert enum + status = self._data.get("status", None) + if status and isinstance(status, DomainStatus): + self._data["status"] = status.value + + @property + def model_obj(self): + """The actual model object behind this mock model.""" + if self._model_obj is None: + domain = self.data.get("domain") + with db.session.no_autoflush: + self._model_obj = current_datastore.find_domain(domain) + return self._model_obj diff --git a/invenio_users_resources/records/systemfields/__init__.py b/invenio_users_resources/records/systemfields/__init__.py index b9a09d9..0704884 100644 --- a/invenio_users_resources/records/systemfields/__init__.py +++ b/invenio_users_resources/records/systemfields/__init__.py @@ -8,8 +8,9 @@ """Data-layer definitions for user and group management in Invenio.""" -from invenio_accounts.models import UserIdentity +from invenio_accounts.models import DomainOrg, UserIdentity from invenio_accounts.proxies import current_datastore +from invenio_accounts.utils import DomainStatus from invenio_records_resources.records.systemfields.calculated import CalculatedField @@ -107,7 +108,60 @@ class UserIdentitiesField(CalculatedIndexedField): def calculate(self, user_record): """Checks if a timestamp is not none.""" identities = UserIdentity.query.filter_by(id_user=user_record.id).all() - data = {} - for i in identities: - data[i.method] = i.id + data = {i.method: i.id for i in identities} return data + + +class DomainOrgField(CalculatedIndexedField): + """Get information about the user's domain.""" + + def calculate(self, domain_record): + """Checks if a timestamp is not none.""" + if not domain_record.model.org_id: + return None + + org = domain_record.model.model_obj.org + + parent_org = None + if org.parent_id is not None: + parent_org = org.parent + + data = [ + { + "id": org.id, + "pid": org.pid, + "name": org.name, + "props": org.json or {}, + "is_parent": False, + } + ] + if parent_org: + data.append( + { + "id": parent_org.id, + "pid": parent_org.pid, + "name": parent_org.name, + "props": parent_org.json or {}, + "is_parent": True, + } + ) + return data + + +class DomainCategoryNameField(CalculatedIndexedField): + """Dump the name of the category.""" + + def calculate(self, domain_record): + """Logic for calculating.""" + if domain_record.model.model_obj.category: + return domain_record.model.model_obj.category_name.label + else: + return None + + +class DomainStatusNameField(CalculatedIndexedField): + """Dump the name of the category.""" + + def calculate(self, domain_record): + """Logic for calculating.""" + return DomainStatus(domain_record.status).name diff --git a/invenio_users_resources/resources/__init__.py b/invenio_users_resources/resources/__init__.py index 1e7bb65..eb569dd 100644 --- a/invenio_users_resources/resources/__init__.py +++ b/invenio_users_resources/resources/__init__.py @@ -8,10 +8,13 @@ """Resources for users and roles/groups.""" +from .domains import DomainsResource, DomainsResourceConfig from .groups import GroupsResource, GroupsResourceConfig from .users import UsersResource, UsersResourceConfig __all__ = ( + "DomainsResource", + "DomainsResourceConfig", "GroupsResource", "GroupsResourceConfig", "UsersResource", diff --git a/invenio_users_resources/resources/domains/__init__.py b/invenio_users_resources/resources/domains/__init__.py new file mode 100644 index 0000000..a262177 --- /dev/null +++ b/invenio_users_resources/resources/domains/__init__.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022 TU Wien. +# +# Invenio-Users-Resources is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Resources for user roles/groups.""" + +from .config import DomainsResourceConfig +from .resource import DomainsResource + +__all__ = ( + "DomainsResource", + "DomainsResourceConfig", +) diff --git a/invenio_users_resources/resources/domains/config.py b/invenio_users_resources/resources/domains/config.py new file mode 100644 index 0000000..e6b4bfd --- /dev/null +++ b/invenio_users_resources/resources/domains/config.py @@ -0,0 +1,43 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 CERN. +# +# Invenio-Users-Resources is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Domains resource config.""" + +import marshmallow as ma +from flask_resources import ( + JSONDeserializer, + JSONSerializer, + RequestBodyParser, + ResponseHandler, +) +from invenio_records_resources.resources import RecordResourceConfig + + +# +# Resource config +# +class DomainsResourceConfig(RecordResourceConfig): + """User groups resource configuration.""" + + blueprint_name = "domains" + url_prefix = "/domains" + + # Request parsing + request_headers = {} + request_body_parsers = { + "application/vnd.inveniordm.v1+json": RequestBodyParser(JSONDeserializer()), + "application/json": RequestBodyParser(JSONDeserializer()), + } + default_content_type = "application/vnd.inveniordm.v1+json" + + # Response handling + response_handlers = { + "application/vnd.inveniordm.v1+json": ResponseHandler(JSONSerializer()), + "application/json": ResponseHandler(JSONSerializer()), + } + default_accept_mimetype = "application/vnd.inveniordm.v1+json" diff --git a/invenio_users_resources/resources/domains/resource.py b/invenio_users_resources/resources/domains/resource.py new file mode 100644 index 0000000..211bf4a --- /dev/null +++ b/invenio_users_resources/resources/domains/resource.py @@ -0,0 +1,32 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 CERN +# +# Invenio-Users-Resources is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Domains resource.""" + + +from flask_resources import HTTPJSONException, create_error_handler +from invenio_i18n import lazy_gettext as _ +from invenio_records_resources.resources import RecordResource +from sqlalchemy.exc import IntegrityError + + +# +# Resource +# +class DomainsResource(RecordResource): + """Resource for domains.""" + + error_handlers = { + **RecordResource.error_handlers, + IntegrityError: create_error_handler( + lambda e: HTTPJSONException( + code=400, + description=_("Domain already exists."), + ) + ), + } diff --git a/invenio_users_resources/resources/users/config.py b/invenio_users_resources/resources/users/config.py index 5bf3573..6bf3d38 100644 --- a/invenio_users_resources/resources/users/config.py +++ b/invenio_users_resources/resources/users/config.py @@ -60,7 +60,12 @@ class UsersResourceConfig(RecordResourceConfig): **ErrorHandlersMixin.error_handlers, LockAcquireFailed: create_error_handler( lambda e: ( - HTTPJSONException(code=400, description=_("User is locked due to concurrent running operation.")) + HTTPJSONException( + code=400, + description=_( + "User is locked due to concurrent running operation." + ), + ) ) ), } diff --git a/invenio_users_resources/services/__init__.py b/invenio_users_resources/services/__init__.py index ad368ca..71ac7da 100644 --- a/invenio_users_resources/services/__init__.py +++ b/invenio_users_resources/services/__init__.py @@ -8,10 +8,13 @@ """Services for users and user roles/groups.""" +from .domains import DomainsService, DomainsServiceConfig from .groups import GroupsService, GroupsServiceConfig from .users import UsersService, UsersServiceConfig __all__ = ( + "DomainsService", + "DomainsServiceConfig", "GroupsService", "GroupsServiceConfig", "UsersService", diff --git a/invenio_users_resources/services/domains/__init__.py b/invenio_users_resources/services/domains/__init__.py new file mode 100644 index 0000000..4025518 --- /dev/null +++ b/invenio_users_resources/services/domains/__init__.py @@ -0,0 +1,17 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 CERN. +# +# Invenio-Users-Resources is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Services for domains.""" + +from .config import DomainsServiceConfig +from .service import DomainsService + +__all__ = ( + "DomainsService", + "DomainsServiceConfig", +) diff --git a/invenio_users_resources/services/domains/components.py b/invenio_users_resources/services/domains/components.py new file mode 100644 index 0000000..9202ba1 --- /dev/null +++ b/invenio_users_resources/services/domains/components.py @@ -0,0 +1,56 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 CERN. +# +# Invenio-Records-Resources is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Domains service component.""" + +from invenio_accounts.models import DomainOrg +from invenio_db import db +from invenio_records_resources.services.records.components import ServiceComponent + + +class DomainComponent(ServiceComponent): + """Service component for metadata.""" + + def create(self, identity, data=None, record=None, errors=None, **kwargs): + """Inject fields into the record.""" + # Note, DB model takes care of setting tld + record.domain = data["domain"] + record.status = data["status"] + # Optional values + record.flagged = data.get("flagged", False) + record.flagged_source = data.get("flagged_source", "") + record.category = data.get("category", None) + self._handle_org(data, record) + + def update(self, identity, data=None, record=None, **kwargs): + """Inject update fields into the domain.""" + # Main part of the validation happens in the schema hence here we just + # pass on already validated properties. + + # Required values + record.status = data["status"] + # Optional values + record.flagged = data.get("flagged", record.flagged) + record.flagged_source = data.get("flagged_source", record.flagged_source) + record.category = data.get("category", record.category) + self._handle_org(data, record) + + def _handle_org(self, data, record): + # Handle organisation + if "org" in data: + if data["org"] is None: + record.org_id = None + else: + org = data["org"] + obj = DomainOrg.query.filter_by(pid=org["pid"]).one_or_none() + if obj is None: + with db.session.begin_nested(): + obj = DomainOrg.create( + org["pid"], org["name"], json=org.get("props", None) + ) + record.org_id = obj.id diff --git a/invenio_users_resources/services/domains/config.py b/invenio_users_resources/services/domains/config.py new file mode 100644 index 0000000..8d130e3 --- /dev/null +++ b/invenio_users_resources/services/domains/config.py @@ -0,0 +1,103 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022 TU Wien. +# Copyright (C) 2024 CERN. +# Copyright (C) 2023 Graz University of Technology. +# +# Invenio-Users-Resources is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Domains service configuration.""" + +from invenio_i18n import lazy_gettext as _ +from invenio_records_resources.services import ( + RecordServiceConfig, + SearchOptions, + pagination_links, +) +from invenio_records_resources.services.base.config import ( + ConfiguratorMixin, + FromConfigSearchOptions, + SearchOptionsMixin, +) +from invenio_records_resources.services.records.params import ( + FacetsParam, + PaginationParam, + QueryStrParam, + SortParam, +) +from invenio_records_resources.services.records.queryparser import QueryParser + +from ...records.api import DomainAggregate +from ..common import Link +from ..permissions import DomainPermissionPolicy +from ..schemas import DomainSchema +from .components import DomainComponent + + +class DomainsSearchOptions(SearchOptions, SearchOptionsMixin): + """Search options.""" + + pagination_options = { + "default_results_per_page": 30, + "default_max_results": 1000, + } + + query_parser_cls = QueryParser.factory( + fields=[ + "id", + "domain^3", + ], + ) + + sort_default = "bestmatch" + sort_default_no_query = "newest" + + params_interpreters_cls = [ + QueryStrParam, + SortParam, + PaginationParam, + FacetsParam, + ] + + +def domainvar(obj, vars): + """Add domain into link vars.""" + vars["domain"] = obj.domain + + +class DomainsServiceConfig(RecordServiceConfig, ConfiguratorMixin): + """Requests service configuration.""" + + # common configuration + permission_policy_cls = DomainPermissionPolicy + search = FromConfigSearchOptions( + "USERS_RESOURCES_DOMAINS_SEARCH", + "USERS_RESOURCES_DOMAINS_SORT_OPTIONS", + "USERS_RESOURCES_DOMAINS_SEARCH_FACETS", + search_option_cls=DomainsSearchOptions, + ) + + # specific configuration + service_id = "domains" + record_cls = DomainAggregate + schema = DomainSchema + indexer_queue_name = "domains" + index_dumper = None + + # links configuration + links_item = { + "self": Link("{+api}/domains/{domain}", vars=domainvar), + "admin_self_html": Link( + "{+ui}/administration/domains/{domain}", vars=domainvar + ), + "admin_users_html": Link( + "{+ui}/administration/users?q=domain:{domain}", vars=domainvar + ), + } + links_search = pagination_links("{+api}/domains{?args*}") + + components = [ + DomainComponent, + ] diff --git a/invenio_users_resources/services/domains/facets.py b/invenio_users_resources/services/domains/facets.py new file mode 100644 index 0000000..b2d9bd2 --- /dev/null +++ b/invenio_users_resources/services/domains/facets.py @@ -0,0 +1,51 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 CERN. +# +# Invenio-Users-Resources is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Domains search facets definitions.""" + +from invenio_i18n import gettext as _ +from invenio_records_resources.services.records.facets import TermsFacet + +status = TermsFacet( + field="status_name", + label=_("Status"), + value_labels={ + "new": _("New"), + "moderated": _("Moderated"), + "verified": _("Verified"), + "blocked": _("Blocked"), + }, +) + + +flagged = TermsFacet( + field="flagged", + label=_("Flagged"), + value_labels={ + True: _("Yes"), + False: _("No"), + }, +) + + +category = TermsFacet( + field="category_name", + label=_("Category"), +) + + +organisation = TermsFacet( + field="org_names", + label=_("Organisation"), +) + + +tld = TermsFacet( + field="tld", + label=_("Top-level domain"), +) diff --git a/invenio_users_resources/services/domains/service.py b/invenio_users_resources/services/domains/service.py new file mode 100644 index 0000000..25cb690 --- /dev/null +++ b/invenio_users_resources/services/domains/service.py @@ -0,0 +1,26 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2022 KTH Royal Institute of Technology +# Copyright (C) 2022 TU Wien. +# Copyright (C) 2024 CERN. +# Copyright (C) 2022 European Union. +# +# Invenio-Users-Resources is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Domains service.""" + +from invenio_accounts.models import Domain +from invenio_db import db +from invenio_records_resources.services import RecordService + + +class DomainsService(RecordService): + """Domains service.""" + + def rebuild_index(self, identity, uow=None): + """Reindex all user groups managed by this service.""" + domains = db.session.query(Domain.domain).yield_per(1000) + self.indexer.bulk_index([r[0] for r in domains]) + return True diff --git a/invenio_users_resources/services/domains/tasks.py b/invenio_users_resources/services/domains/tasks.py new file mode 100644 index 0000000..9409518 --- /dev/null +++ b/invenio_users_resources/services/domains/tasks.py @@ -0,0 +1,38 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 CERN. +# Copyright (C) 2022 TU Wien. +# +# Invenio-Users-Resources is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Users service tasks.""" + +from celery import shared_task +from flask import current_app +from invenio_search.engine import search + +from ...proxies import current_domains_service + + +@shared_task(ignore_result=True) +def reindex_domains(domain_ids): + """Reindex the given domains.""" + index = current_domains_service.record_cls.index + if current_domains_service.indexer.exists(index): + try: + current_domains_service.indexer.bulk_index(domain_ids) + except search.exceptions.ConflictError as e: + current_app.logger.warn(f"Could not bulk-reindex groups: {e}") + + +@shared_task(ignore_result=True) +def delete_domains(domain_ids): + """Delete domains from index.""" + index = current_domains_service.record_cls.index + if current_domains_service.indexer.exists(index): + try: + current_domains_service.indexer.bulk_delete(domain_ids) + except search.exceptions.ConflictError as e: + current_app.logger.warn(f"Could not bulk-unindex groups: {e}") diff --git a/invenio_users_resources/services/permissions.py b/invenio_users_resources/services/permissions.py index a1eac30..a2d3d23 100644 --- a/invenio_users_resources/services/permissions.py +++ b/invenio_users_resources/services/permissions.py @@ -61,3 +61,13 @@ class GroupsPermissionPolicy(BasePermissionPolicy): can_search = [AuthenticatedUser(), SystemProcess()] can_update = [SystemProcess()] can_delete = [SystemProcess()] + + +class DomainPermissionPolicy(BasePermissionPolicy): + """Permission policy for users and user groups.""" + + can_create = [UserManager, SystemProcess()] + can_read = [UserManager, SystemProcess()] + can_search = [UserManager, SystemProcess()] + can_update = [UserManager, SystemProcess()] + can_delete = [UserManager, SystemProcess()] diff --git a/invenio_users_resources/services/schemas.py b/invenio_users_resources/services/schemas.py index ab53305..958b808 100644 --- a/invenio_users_resources/services/schemas.py +++ b/invenio_users_resources/services/schemas.py @@ -9,19 +9,31 @@ """User and user group schemas.""" +from flask import current_app from invenio_access.permissions import system_user_id +from invenio_accounts.models import DomainCategory from invenio_accounts.profiles.schemas import ( validate_locale, validate_timezone, validate_visibility, ) +from invenio_accounts.utils import DomainStatus from invenio_i18n import lazy_gettext as _ from invenio_records_resources.services.records.schema import ( BaseGhostSchema, BaseRecordSchema, ) -from marshmallow import Schema, fields -from marshmallow_utils.fields import SanitizedUnicode, TZDateTime +from marshmallow import ( + EXCLUDE, + Schema, + ValidationError, + fields, + post_load, + pre_load, + validate, + validates_schema, +) +from marshmallow_utils.fields import Links, SanitizedUnicode, TZDateTime from marshmallow_utils.permissions import FieldPermissionsMixin @@ -156,3 +168,112 @@ class NotificationPreferences(Schema): """Schema for notification preferences.""" enabled = fields.Bool() + + +class DomainOrgSchema(Schema): + """Schema for domain orgs.""" + + id = fields.Integer(dump_only=True) + pid = fields.String(validate=validate.Length(min=1, max=255), required=True) + name = fields.String(validate=validate.Length(min=1, max=255), required=True) + props = fields.Dict( + keys=fields.String(required=True), + values=fields.String(validate=validate.Length(max=255)), + ) + is_parent = fields.Boolean(dump_only=True, dump_default=False) + + @validates_schema + def validate_props(self, data, **kwargs): + """Apply instance specific validation on props.""" + schema = current_app.config["USERS_RESOURCES_DOMAINS_ORG_SCHEMA"] + props = data.get("props", {}) + if props: + schema.load(props) + + +def validate_domain(value): + """Domain validation.""" + # Basic validation - zenodo has some pretty funky domains so we are not too + # strict here. + if len(value) > 255: + raise ValidationError("Length must be less than 255.") + value = value.lower().strip() + if "." not in value: + raise ValidationError("Not a domain name.") + prefix, tld = value.rsplit(".", 1) + if tld == "": + raise ValidationError("Not a domain name.") + + +class DomainSchema(Schema): + """Schema for user groups.""" + + id = fields.Str(dump_only=True) + domain = fields.String( + validate=validate_domain, required=True, metadata={"create_only": True} + ) + tld = fields.String(dump_only=True) + status = fields.Integer(dump_only=True) + status_name = fields.String( + validate=validate.OneOf([s.name for s in list(DomainStatus)]), + load_default=DomainStatus.new.name, + ) + category = fields.Integer(dump_only=True, metadata={"read_only": True}) + category_name = fields.String(validate=validate.Length(min=1, max=255)) + flagged = fields.Boolean(default=False, metadata={"checked": False}) + flagged_source = fields.Str(validate=validate.Length(max=255), load_default="") + org = fields.List( + fields.Nested(DomainOrgSchema), dump_default=None, load_default=None + ) + num_users = fields.Integer(dump_only=True) + num_active = fields.Integer(dump_only=True) + num_inactive = fields.Integer(dump_only=True) + num_confirmed = fields.Integer(dump_only=True) + num_verified = fields.Integer(dump_only=True) + num_blocked = fields.Integer(dump_only=True) + created = TZDateTime(dump_only=True) + updated = TZDateTime(dump_only=True) + links = Links(dump_only=True) + + class Meta: + """Schema meta.""" + + unknown = EXCLUDE + + @pre_load + def preprocess(self, data, **kwargs): + """Preprocess form data.""" + # Handle misbehaving clients. + if "org" in data and data["org"] == "": + del data["org"] + return data + + @post_load + def postprocess(self, data, **kwargs): + """Process output data.""" + data["domain"] = data["domain"].lower().strip() + data["domain"].strip() + data["status_name"] = DomainStatus[data["status_name"]] + data["status"] = data["status_name"].value + if "category_name" in data: + if data["category_name"] is None: + data["category"] = None + else: + category = DomainCategory.get(data["category_name"]) + data["category"] = category.id + if "org" in data: + org = data["org"] + if org is None or len(org) == 0: + data["org"] = None + else: + # discard parent + data["org"] = org[0] + return data + + @validates_schema + def validate_category(self, data, **kwargs): + """Validate category data.""" + if "category_name" in data and data["category_name"] is not None: + category = DomainCategory.get(data["category_name"]) + if category is None: + raise ValidationError("Invalid category_name.") diff --git a/invenio_users_resources/services/users/config.py b/invenio_users_resources/services/users/config.py index 48a676d..5542d5e 100644 --- a/invenio_users_resources/services/users/config.py +++ b/invenio_users_resources/services/users/config.py @@ -50,14 +50,8 @@ def can_manage(obj, ctx): def word_domain_status(node): """Quote DOIs.""" val = node.value - if node.value == "verified": - val = DomainStatus.verified.value - elif node.value == "blocked": - val = DomainStatus.blocked.value - elif node.value == "moderated": - val = DomainStatus.moderated.value - elif node.value == "new": - val = DomainStatus.new.value + if val in ["verified", "blocked", "moderated", "new"]: + val = DomainStatus[node.value].value return Word(f"{val}") diff --git a/invenio_users_resources/views.py b/invenio_users_resources/views.py index 88c23b9..3af93ff 100644 --- a/invenio_users_resources/views.py +++ b/invenio_users_resources/views.py @@ -32,9 +32,11 @@ def init(state): # services rr_ext.registry.register(ext.users_service) rr_ext.registry.register(ext.groups_service) + rr_ext.registry.register(ext.domains_service) idx_ext.registry.register(ext.users_service.indexer, indexer_id="users") idx_ext.registry.register(ext.groups_service.indexer, indexer_id="groups") + idx_ext.registry.register(ext.domains_service.indexer, indexer_id="domains") def create_users_resources_bp(app): @@ -47,3 +49,9 @@ def create_groups_resources_bp(app): """Create the user groups resource blueprint.""" ext = app.extensions["invenio-users-resources"] return ext.groups_resource.as_blueprint() + + +def create_domains_resources_bp(app): + """Create the domains resource blueprint.""" + ext = app.extensions["invenio-users-resources"] + return ext.domains_resource.as_blueprint() diff --git a/setup.cfg b/setup.cfg index 56b5478..3efda01 100644 --- a/setup.cfg +++ b/setup.cfg @@ -56,12 +56,14 @@ invenio_base.api_apps = invenio_base.api_blueprints = invenio_users = invenio_users_resources.views:create_users_resources_bp invenio_groups = invenio_users_resources.views:create_groups_resources_bp + invenio_domains = invenio_users_resources.views:create_domains_resources_bp invenio_users_resources = invenio_users_resources.views:blueprint invenio_base.blueprints = invenio_users_resources = invenio_users_resources.views:blueprint invenio_search.mappings = users = invenio_users_resources.records.mappings groups = invenio_users_resources.records.mappings + domains = invenio_users_resources.records.mappings invenio_i18n.translations = messages = invenio_users_resources invenio_access.actions = diff --git a/tests/conftest.py b/tests/conftest.py index abb5198..aee749d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -19,7 +19,8 @@ from flask_principal import AnonymousIdentity from invenio_access.models import ActionRoles from invenio_access.permissions import any_user as any_user_need -from invenio_accounts.models import Role +from invenio_access.permissions import system_identity +from invenio_accounts.models import Domain, DomainCategory, DomainOrg, Role from invenio_accounts.proxies import current_datastore from invenio_app.factory import create_api from invenio_cache.proxies import current_cache @@ -27,6 +28,7 @@ from invenio_users_resources.permissions import user_management_action from invenio_users_resources.proxies import ( + current_domains_service, current_groups_service, current_users_service, ) @@ -364,3 +366,102 @@ def clear_cache(): Locking is done using cache, therefore the cache must be cleared after each test to make sure that locks from previous tests are cleared. """ current_cache.cache.clear() + + +@pytest.fixture(scope="module") +def domains_data(): + """Data for domains.""" + return [ + { + "domain": "cern.ch", + "tld": "ch", + "status": 3, + "flagged": False, + "flagged_source": "", + "category": 1, + "org_id": 1, + }, + { + "domain": "inveniosoftware.org", + "tld": "org", + "status": 3, + "flagged": False, + "flagged_source": "", + "org_id": 2, + }, + { + "domain": "new.org", + "tld": "org", + "status": 1, + "flagged": False, + "flagged_source": "", + }, + { + "domain": "moderated.org", + "tld": "org", + "status": 2, + "flagged": True, + "flagged_source": "disposable", + "category": 3, + }, + { + "domain": "spammer.com", + "tld": "com", + "status": 4, + "flagged": True, + "flagged_source": "", + "category": 4, + }, + ] + + +@pytest.fixture(scope="module") +def domaincategories_data(): + """Data for domains.""" + return [ + {"id": 1, "label": "organization"}, + {"id": 2, "label": "company"}, + {"id": 3, "label": "mail-provider"}, + {"id": 4, "label": "spammer"}, + ] + + +@pytest.fixture(scope="module") +def domainorgs_data(): + """Data for domains.""" + return [ + { + "id": 1, + "pid": "https://ror.org/01ggx4157", + "name": "CERN", + "json": {"country": "ch"}, + }, + { + "id": 2, + "pid": "https://ror.org/01ggx4157::it", + "name": "IT department", + "json": {"country": "ch"}, + "parent_id": 1, + }, + ] + + +@pytest.fixture(scope="module") +def domains(app, database, domainorgs_data, domaincategories_data, domains_data): + """Test domains.""" + for d in domaincategories_data: + database.session.add(DomainCategory(**d)) + for d in domainorgs_data: + database.session.add(DomainOrg(**d)) + database.session.commit() + + domains = {} + for d in domains_data: + database.session.add(Domain(**d)) + domains[d["domain"]] = d + database.session.commit() + + current_domains_service.rebuild_index(system_identity) + current_domains_service.indexer.process_bulk_queue() + current_domains_service.record_cls.index.refresh() + return domains diff --git a/tests/resources/test_resources_domains.py b/tests/resources/test_resources_domains.py new file mode 100644 index 0000000..e0a60f5 --- /dev/null +++ b/tests/resources/test_resources_domains.py @@ -0,0 +1,193 @@ +# -*- coding: utf-8 -*- +# +# Copyright (C) 2024 CERN. +# +# Invenio-Users-Resources is free software; you can redistribute it and/or +# modify it under the terms of the MIT License; see LICENSE file for more +# details. + +"""Domains resource tests.""" + +import pytest + + +def test_domains_access(app, client, domains, user_moderator, user_pub): + res = client.get(f"/domains") + assert res.status_code == 403 + + user_pub.login(client) + res = client.get(f"/domains") + assert res.status_code == 403 + + +def test_domains_search(app, client, domains, user_moderator, user_pub): + user_moderator.login(client) + res = client.get(f"/domains") + assert res.status_code == 200 + data = res.json + assert len(data["hits"]["hits"]) == 5 + + # Props + props = [ + "id", + "domain", + "created", + "updated", + "domain", + "tld", + "status", + "status_name", + "category", + "category_name", + "flagged", + "flagged_source", + "org", + "num_users", + "num_active", + "num_inactive", + "num_confirmed", + "num_verified", + "num_blocked", + ] + cern = data["hits"]["hits"][0] + assert cern["domain"] == "cern.ch" + for p in props: + assert p in cern + + # Check aggregations and that they have content + aggs = ["status", "flagged", "category", "organisation", "tld"] + for a in aggs: + assert a in data["aggregations"] + assert ( + len(data["aggregations"][a]["buckets"]) > 0 + ), f"'{a}' is missing bucket values" + assert len(data["hits"]["hits"]) == 5 + + +def test_domains_read(app, client, domains, user_moderator): + res = client.get(f"/domains/cern.ch") + assert res.status_code == 403 + + user_moderator.login(client) + res = client.get(f"/domains/cern.ch") + assert res.json["links"]["self"].endswith("/domains/cern.ch") + assert res.status_code == 200 + d = res.json + assert d["domain"] == "cern.ch" + assert d["tld"] == "ch" + assert d["status"] == 3 + assert d["status_name"] == "verified" + assert d["flagged"] == False + assert d["flagged_source"] == "" + assert d["category"] == 1 + assert d["category_name"] == "organization" + assert d["org"] == [ + { + "id": 1, + "pid": "https://ror.org/01ggx4157", + "name": "CERN", + "props": {"country": "ch"}, + "is_parent": False, + } + ] + stats = ["users", "active", "inactive", "confirmed", "verified", "blocked"] + for s in stats: + assert f"num_{s}" in d, f"num_{s} is missing from payload" + + +def test_domains_delete(app, client, domains, user_moderator): + res = client.delete(f"/domains/cern.ch") + assert res.status_code == 403 + + user_moderator.login(client) + res = client.delete(f"/domains/cern.ch") + assert res.status_code == 204 + res = client.get(f"/domains/cern.ch") + assert res.status_code == 404 + + +def test_domains_create(app, client, domains, user_moderator): + res = client.post( + f"/domains", + json={ + "domain": "zenodo.org", + }, + headers={"content-type": "application/vnd.inveniordm.v1+json"}, + ) + assert res.status_code == 403 + + user_moderator.login(client) + # Make an update + res = client.post( + f"/domains", + json={ + "domain": "zenodo.org", + }, + headers={"content-type": "application/vnd.inveniordm.v1+json"}, + ) + assert res.status_code == 201 + # Re-read to check that it was updated + data = client.get(f"/domains/zenodo.org").json + assert data["domain"] == "zenodo.org" + assert data["tld"] == "org" + assert data["status_name"] == "new" + assert data["category_name"] == None + assert data["flagged"] == False + assert data["flagged_source"] == "" + assert data["org"] is None + + +@pytest.mark.parametrize( + "status_code,json", + [ + (400, {"status": "new"}), # missing domain + (400, {"domain": "test.com", "status_name": "invalid"}), # invalid status + (400, {"domain": "spammer.com"}), # duplicate domain + (400, {"domain": "test.com", "category_name": "invalid"}), # invalid category + ], +) +def test_domains_create_failure( + app, client, domains, user_moderator, status_code, json +): + user_moderator.login(client) + # Make an update + res = client.post( + f"/domains", + json=json, + headers={"content-type": "application/vnd.inveniordm.v1+json"}, + ) + assert res.status_code == status_code + + +def test_domains_update(app, client, domains, user_moderator): + user_moderator.login(client) + data = client.get(f"/domains/moderated.org").json + assert data["domain"] == "moderated.org" + assert data["status_name"] == "moderated" + assert data["category_name"] == "mail-provider" + assert data["flagged"] == True + assert data["flagged_source"] == "disposable" + assert data["org"] is None + # Make an update + res = client.put( + f"/domains/moderated.org", + json={ + "domain": "moderated.org", + "status_name": "verified", + "category_name": "spammer", + "flagged": False, + "flagged_source": "test", + "org": None, + }, + headers={"content-type": "application/vnd.inveniordm.v1+json"}, + ) + assert res.status_code == 200 + # Re-read to check that it was updated + data = client.get(f"/domains/moderated.org").json + assert data["domain"] == "moderated.org" + assert data["tld"] == "org" + assert data["status_name"] == "verified" + assert data["category_name"] == "spammer" + assert data["flagged"] == False + assert data["flagged_source"] == "test" + assert data["org"] is None diff --git a/tests/services/users/test_service_users.py b/tests/services/users/test_service_users.py index 5860c8a..7afe142 100644 --- a/tests/services/users/test_service_users.py +++ b/tests/services/users/test_service_users.py @@ -276,7 +276,8 @@ def test_restore(app, db, user_service, user_res, user_moderator, clear_cache): ur = user_service.read(user_moderator.identity, user_res.id) assert ur.data["active"] == True - assert ur.data["verified_at"] is not None + assert ur.data["confirmed_at"] is not None + assert ur.data["verified_at"] is None assert ur.data["blocked_at"] is None diff --git a/tests/services/users/test_user_moderation.py b/tests/services/users/test_user_moderation.py index 134364f..9fdbcbc 100644 --- a/tests/services/users/test_user_moderation.py +++ b/tests/services/users/test_user_moderation.py @@ -16,17 +16,24 @@ import pytest from invenio_access.permissions import system_identity -from invenio_cache.lock import CachedMutex +from marshmallow import ValidationError from invenio_users_resources.proxies import current_actions_registry from invenio_users_resources.services.users.lock import ModerationMutex +@pytest.fixture() +def unblocked(user_service, user_res): + try: + user_service.activate(system_identity, user_res.id) + except ValidationError: + pass + + def test_moderation_callbacks_success( - user_service, user_res, user_moderator, monkeypatch, clear_cache + user_service, user_res, user_moderator, monkeypatch, unblocked, clear_cache ): """Test moderation actions (post block / restore).""" - mocked_method = MagicMock(return_value=True) monkeypatch.setitem(current_actions_registry, "block", [mocked_method]) blocked = user_service.block(user_moderator.identity, user_res.id) @@ -37,7 +44,7 @@ def test_moderation_callbacks_success( def test_moderation_callbacks_failure( - user_service, user_res, user_moderator, monkeypatch, clear_cache + user_service, user_res, user_moderator, monkeypatch, unblocked, clear_cache ): """Test moderation actions (post block). @@ -69,7 +76,7 @@ def _block_action_failure(user_id, uow=None): def test_moderation_callbacks_lock( - app, user_service, user_res, user_moderator, monkeypatch, clear_cache + app, user_service, user_res, user_moderator, monkeypatch, unblocked, clear_cache ): """Tests the 'simplest' flow for user moderation in terms of locks (e.g. mutex). @@ -113,6 +120,7 @@ def test_moderation_callbacks_lock_renewal( renewal_timeout, expected_lock_state, clear_cache, + unblocked, ): """Tests whether the lock is renewed after moderating a user.