Merge pull request #132 from plone/issue-116-foundation-member

Foundation Member improvements

backend/instance.yaml

@@ -5,4 +5,7 @@ default_context:
     load_zcml:
         package_includes: ['ploneorg']
 
-    db_storage: direct
+    db_storage: relstorage
+    db_relstorage_postgresql_driver: psycopg2
+    db_relstorage_postgresql_dsn: dbname='ploneorg' user='ploneorg' host='localhost' password='ploneorg'
+    db_blobs_mode: cache

backend/requirements.txt

@@ -1,4 +1,6 @@
 -c constraints.txt
 -e src/ploneorg[test]
 
+relstorage==3.5.0
+psycopg2==2.9.5
 zope.testrunner

backend/src/ploneorg/src/ploneorg/profiles/default/metadata.xml

@@ -1,6 +1,6 @@
 <?xml version="1.0" encoding="utf-8"?>
 <metadata>
-  <version>20230530001</version>
+  <version>20230904001</version>
   <dependencies>
     <dependency>profile-plone.volto:default</dependency>
     <dependency>profile-plone.app.vulnerabilities:default</dependency>

backend/src/ploneorg/src/ploneorg/profiles/default/workflows/foundation_member_workflow/definition.xml

@@ -90,6 +90,7 @@
       <permission-role>Manager</permission-role>
       <permission-role>Owner</permission-role>
       <permission-role>Reviewer</permission-role>
+      <permission-role>Anonymous</permission-role>
     </permission-map>
     <permission-map acquired="False"
                     name="ploneorg: View Foundation Member Details"

backend/src/ploneorg/src/ploneorg/profiles/oauth/registry/pas.plugins.authomatic.interfaces.IPasPluginsAuthomaticSettings.xml

@@ -31,6 +31,6 @@
     }
 }</value>
     <value key="secret">4SiyrCO6ZV</value>
-    <value key="userid_factory_name">uuid</value>
+    <value key="userid_factory_name">username</value>
   </records>
 </registry>

backend/src/ploneorg/src/ploneorg/upgrades/configure.zcml

@@ -131,4 +131,26 @@
         />
   </genericsetup:upgradeSteps>
 
+  <genericsetup:upgradeSteps
+      profile="ploneorg:default"
+      source="20230530001"
+      destination="20230904001"
+      >
+    <upgradeStep
+        title="Update Foundation Member workflow"
+        description="Update permission and update security"
+        handler=".v20230904.update_workflow"
+        />
+    <upgradeStep
+        title="Update Authomatic Settings"
+        description="Modify update_userid_factory_name to username"
+        handler=".v20230904.update_userid_factory_name"
+        />
+    <upgradeStep
+        title="Update existing GitHub users to use username as user_id"
+        description="Change all existing users, update group membership and local roles."
+        handler=".v20230904.update_users"
+        />
+  </genericsetup:upgradeSteps>
+
 </configure>

backend/src/ploneorg/src/ploneorg/upgrades/v20230904/__init__.py

@@ -0,0 +1,185 @@
+from collections import defaultdict
+from dataclasses import dataclass
+from plone import api
+from ploneorg import logger
+from Products.CMFPlone.Portal import PloneSite
+from Products.GenericSetup.tool import SetupTool
+from typing import List
+
+import transaction
+
+
+PREFIX = "pas.plugins.authomatic.interfaces.IPasPluginsAuthomaticSettings"
+
+
+@dataclass
+class UserInfo:
+    user_name: str
+    user_id: str
+    fullname: str
+    email: str
+    github_username: str
+    github_id: str
+    portrait: str
+    login_time: str
+    last_login_time: str
+    groups: List[str]
+
+
+def get_gh_users(site: PloneSite) -> List[UserInfo]:
+    pas = site.acl_users
+    kw = {}
+    results = {user["userid"] for user in pas.searchUsers(**kw)}
+    users = [site.portal_membership.getMemberById(userid) for userid in results]
+    users_info = {}
+    for user in users:
+        gh_info = {}
+        github_id = ""
+        user_id = user.getUserId()
+        portrait = user.getProperty("portrait", "")
+        # Probably github account
+        if not len(user_id) == 36:
+            continue
+        gh_info = pas.authomatic._useridentities_by_userid[user_id].identity("github")
+        github_id = gh_info.get("id", "")
+        portrait = gh_info.get("picture", "")
+        user_info = UserInfo(
+            user.getUserName(),
+            user.getUserId(),
+            user.getProperty("fullname", ""),
+            user.getProperty("email", ""),
+            gh_info.get("username", ""),
+            github_id,
+            portrait,
+            user.getProperty("login_time", ""),
+            user.getProperty("last_login_time", ""),
+            [g.id for g in api.group.get_groups(user=user)],
+        )
+        users_info[user_info.user_name] = user_info
+    return users_info
+
+
+def content_by_user(users_info: List[UserInfo], site: PloneSite) -> dict:
+    content = {
+        "content": defaultdict(list),
+        "user": defaultdict(list),
+    }
+    with api.env.adopt_roles(
+        [
+            "Manager",
+            "Site Administrator",
+        ]
+    ):
+        brains = site.portal_catalog.searchResults()
+
+    for brain in brains:
+        obj = brain.getObject()
+        local_roles = obj.get_local_roles()
+        if local_roles:
+            for (g, r) in local_roles:
+                if g in users_info:
+                    content["content"][brain.UID].append((g, r))
+                    content["user"][g].append((brain.UID, r))
+    return content
+
+
+def _update_user(user_info: UserInfo, site: PloneSite):
+    pas = site.acl_users
+    authomatic = pas.authomatic
+    prop = pas.mutable_properties
+    md = site.portal_memberdata
+    old_id = user_info.user_id
+    new_id = user_info.github_username
+    github_id = user_info.github_id
+    groups = [g for g in user_info.groups if g != "AuthenticatedUsers"]
+    for group in groups:
+        api.group.remove_user(groupname=group, username=old_id)
+    # Update Authomatic
+    authomatic._useridentities_by_userid[
+        new_id
+    ] = pas.authomatic._useridentities_by_userid[old_id]
+    authomatic._useridentities_by_userid[new_id].userid = new_id
+    authomatic._userid_by_identityinfo[("github", github_id)] = new_id
+    del authomatic._useridentities_by_userid[old_id]
+    # Update Memberdata
+    prop._storage[new_id] = prop._storage[old_id]
+    md._members[new_id] = md._members[old_id]
+    del md._members[old_id]
+    del prop._storage[old_id]
+    # Update Groups info
+    for group in groups:
+        api.group.add_user(groupname=group, username=new_id)
+    user = api.user.get(userid=new_id)
+    # Update username
+    user._user._login = new_id
+    return user
+
+
+def update_userid_factory_name(_: SetupTool):
+    with transaction.manager as tm:
+        logger.info("Update userid_factory_name to username.")
+        new_value = "username"
+        key = f"{PREFIX}.userid_factory_name"
+        current = api.portal.get_registry_record(key)
+        api.portal.set_registry_record(key, new_value)
+        logger.info(f"- Configuration updated in {key} from {current} to {new_value}")
+        tm.note(f"Authomatic: Updated {key}")
+
+
+def update_users(_: SetupTool):
+    site = api.portal.get()
+    users_info = get_gh_users(site)
+    content = content_by_user(users_info, site)
+    with transaction.manager as tm:
+        for idx, (old_id, user_info) in enumerate(users_info.items()):
+            user = _update_user(user_info, site)
+            logger.info(f"-- {old_id} -> {user.getUserName()}")
+            if idx % 10 == 0:
+                logger.info(f"Users savepoint: {idx} itens")
+                tm.savepoint()
+
+        tm.note(f"Users: Updated {idx + 1} usernames and ids")
+    with transaction.manager as tm:
+        for idx, (uid, roles) in enumerate(content["content"].items()):
+            obj = api.content.get(UID=uid)
+            # Update creators field
+            creators = []
+            for creator in obj.creators:
+                actor = (
+                    users_info[creator].github_username
+                    if creator in users_info
+                    else creator
+                )
+                creators.append(actor)
+            obj.creators = tuple(creators)
+            for old_id, r in roles:
+                obj.manage_delLocalRoles(
+                    [
+                        old_id,
+                    ]
+                )
+                new_id = users_info[old_id].github_username
+                obj.manage_setLocalRoles(new_id, r)
+            obj.reindexObject(idxs=["Creator", "allowedRolesAndUsers", "listCreators"])
+            if idx % 100 == 0:
+                logger.info(f"Content savepoint: {idx} itens")
+                tm.savepoint()
+        tm.note(f"Content: Updated local roles for {idx + 1} content items")
+
+
+def update_workflow(st: SetupTool):
+    # Update Foundation Workflow
+    with transaction.manager as tm:
+        st.runImportStepFromProfile(
+            "ploneorg:default", "workflow", run_dependencies=False
+        )
+        msg = "Updated Plone Foundation Workflow"
+        logger.info(msg)
+        tm.note(msg)
+    # Update workflow mappings
+    with transaction.manager as tm:
+        wf_tool = api.portal.get_tool("portal_workflow")
+        wf_tool.updateRoleMappings()
+        msg = "Updated workflow security"
+        logger.info(msg)
+        tm.note(msg)

backend/src/ploneorg/tests/test_setup.py

@@ -34,7 +34,7 @@ def test_latest_version(self):
         """Test latest version of default profile."""
         self.assertEqual(
             self.setup.getLastVersionForProfile("ploneorg:default")[0],
-            "20230530001",
+            "20230904001",
         )
 
 

backend/src/ploneorg/tests/test_upgrades.py

@@ -39,6 +39,7 @@ def available_steps(self, src, dst) -> list:
             ("20221212003", "20230412001", 1),
             ("20230412001", "20230528001", 1),
             ("20230528001", "20230530001", 1),
+            ("20230530001", "20230904001", 1),
 
         ]
     )

docker-compose.yml

@@ -1,44 +1,16 @@
-version: "3"
-services:
-
-  webserver:
-    image: nginx
-    volumes:
-      - ./default.conf:/etc/nginx/conf.d/default.conf
-    depends_on:
-      - backend
-      - frontend
-    ports:
-      - "80:80"
+version: "3.8"
 
-
-  frontend:
-    build:
-      context: ../../frontend
-    environment:
-      RAZZLE_INTERNAL_API_PATH: http://backend:8080/Plone
-    ports:
-      - "3000:3000"
-    depends_on:
-      - backend
-
-
-  backend:
-    build:
-      context: ../../backend
-    ports:
-      - "8080:8080"
-    environment:
-      RELSTORAGE_DSN: "dbname='${DB_NAME:-plone}' user='${DB_USER:-plone}' host='${DB_HOST:-db}' password='${DB_PASSWORD:-plone}'"
-    depends_on:
-      - db
+services:
 
   db:
     image: postgres:14.2
     environment:
-      POSTGRES_USER: plone
-      POSTGRES_PASSWORD: plone
-      POSTGRES_DB: plone
+      POSTGRES_USER: ploneorg
+      POSTGRES_PASSWORD: ploneorg
+      POSTGRES_DB: ploneorg
+    command: postgres -c shared_buffers=4GB -c effective_cache_size=12GB -c maintenance_work_mem=1GB -c wal_buffers=16MB -c random_page_cost=1.1 -c effective_io_concurrency=200 -c work_mem=20971kB -c min_wal_size=2GB -c max_wal_size=8GB -c wal_keep_size=1GB -c max_locks_per_transaction=512
+    ports:
+      - 5432:5432
     volumes:
       - data:/var/lib/postgresql/data