Merge pull request #163 from plone/update_609_16-30

Update to Volto 16.30.1 and Plone 6.0.9

.github/workflows/backend-testing.yaml

@@ -69,7 +69,7 @@ jobs:
         python:
           - 3.11
         plone:
-          - "6.0.6"
+          - "6.0.9"
 
     defaults:
       run:

CHANGELOG.md

@@ -2,8 +2,23 @@
 
 ---
 
+- Use mxdev 3.1.0 and pip 23.3.2 by overriding them in mx.ini. [fredvd]
+
+- List and pin backend bootstrap packages in backend/requirements-bootstrap.txt and install them in the 'bin/pip' Makefile target while passing in the full project constraints. [fredvd]
+
+- Add zodbconvert example and instructions in devops/zodbconvert to create filestorage version of the content database from a locally running postgresql server. [fredvd]
+
+- Add alternative filestorage based configuration profile for the backend with build-dev-fs (and config-fs) targets in backend/Makefile. [fredvd]
+
+- Update docker-compose example with latest postgresql 14 and local bind mount for the postgesql data. [fredvd]
+
+- Update to Volto 16.30.1. [fredvd]
+
+- Update to Plone backend 6.0.9 [fredvd]
+
 - Update to Volto 16.25.0 [davisagli]
 
+
 ## 1.0.13 (2023-10-09)
 
 ---

README.md

@@ -82,6 +82,16 @@ There are `Makefile` commands in place:
 
 `test-acceptance`: Start Core Cypress Acceptance Tests in dev mode
 
+## Filestorage based backend alternative
+
+The default setup assumes for local development you also have a PostgreSQL server running. As a convenience an example docker compose file is provided in the project root for such a service. If you prefer to use filestorage, you can look in the backend directory its Makefile and
+look at the alternative 'build-dev-fs' target, which depends on 'config-fs'. The difference with the normal 'config' target is the usages of
+instance-filestorage.yaml instead of instanc.yaml. These files are used as input for the application server scaffolding and switch the storage
+layer to either relstorage/postgresql or direct/filestorage. 
+
+For both storages, if you want to develop for plone.org and need a copy of the data, you will need to as another community member (preferably from the website-team or the AI-team) for a database dump. 
+
+
 ## Credits
 
 **This was generated by [cookiecutter-plone-starter](https://github.com/collective/cookiecutter-plone-starter) on 2022-10-14 12:14:12**

backend/Dockerfile

@@ -1,6 +1,6 @@
-ARG PLONE_VERSION=6.0.6
+ARG PLONE_VERSION=6.0.9
 FROM plone/plone-backend:${PLONE_VERSION}
-ARG PLONE_VERSION=6.0.6
+ARG PLONE_VERSION=6.0.9
 
 LABEL maintainer="Plone Foundation <[email protected]>" \
       org.label-schema.name="plone.org-backend" \

backend/Makefile

@@ -88,13 +88,20 @@ clean-test: ## remove test and coverage artifacts
 bin/pip:
 	@echo "$(GREEN)==> Setup Virtual Env$(RESET)"
 	$(PYTHON) -m venv .
-	bin/pip install -U "pip" "wheel" "cookiecutter" "mxdev"
+	bin/pip install -U -r requirements-bootstrap.txt -c constraints-mxdev.txt
 
 .PHONY: config
 config: bin/pip  ## Create instance configuration
 	@echo "$(GREEN)==> Create instance configuration$(RESET)"
 	bin/cookiecutter -f --no-input --config-file instance.yaml gh:plone/cookiecutter-zope-instance
 
+.PHONY: config-fs
+config-fs: bin/pip  ## Create instance configuration
+	@echo "$(GREEN)==> Create instance configuration$(RESET)"
+	bin/cookiecutter -f --no-input --config-file instance-filestorage.yaml gh:plone/cookiecutter-zope-instance
+
+
+
 # i18n
 bin/i18ndude: bin/pip
 	@echo "$(GREEN)==> Install translation tools$(RESET)"
@@ -115,6 +122,13 @@ build-dev: config ## pip install Plone packages
 	@bin/mxdev -c mx.ini
 	bin/pip install -r requirements-mxdev.txt
 
+.PHONY: build-dev-fs
+build-dev-fs: config-fs ## pip install Plone packages
+	@echo "$(GREEN)==> Setup Build$(RESET)"
+	sed "s/PLONE_VERSION/$(PLONE_VERSION)/g" constraints-template.txt > constraints.txt
+	@bin/mxdev -c mx.ini
+	bin/pip install -r requirements-mxdev.txt
+
 .PHONY: format
 format: ## Format the codebase according to our standards
 	@echo "$(GREEN)==> Format codebase$(RESET)"

backend/constraints.txt

@@ -1 +1 @@
--c https://dist.plone.org/release/6.0.6/constraints.txt
+-c https://dist.plone.org/release/6.0.9/constraints.txt

backend/instance-filestorage.yaml

@@ -0,0 +1,13 @@
+default_context:
+    initial_user_name: 'admin'
+    initial_user_password: 'admin'
+
+    load_zcml:
+        package_includes: ['ploneorg']
+
+    dos_protection_available: true
+
+    db_storage: direct
+
+    db_filestorage_location: ../data/filestorage/Data.fs
+    db_blobs_location: ../data/blobs

backend/instance.yaml

@@ -5,7 +5,9 @@ default_context:
     load_zcml:
         package_includes: ['ploneorg']
 
+    dos_protection_available: true
+
     db_storage: relstorage
     db_relstorage_postgresql_driver: psycopg2
-    db_relstorage_postgresql_dsn: dbname='ploneorg' user='ploneorg' host='localhost' password='ploneorg'
+    db_relstorage_postgresql_dsn: dbname='ploneorg' user='ploneorg' host='127.0.0.1' password='ploneorg'
     db_blobs_mode: cache

backend/mx.ini

@@ -8,6 +8,10 @@
 ; version-overrides =
 ;     example.package==2.1.0a2
 
+version-overrides = 
+     pip==23.3.2
+     mxdev==3.1.0
+
 [plone.app.vulnerabilities]
 url = https://github.com/plone/plone.app.vulnerabilities.git
 pushurl = [email protected]:plone/plone.app.vulnerabilities.git

backend/requirements-bootstrap.txt

@@ -0,0 +1,7 @@
+# These packages are installed after creating the virtualenv and are needed to further configure and
+# install the main backend application. Keep these versions pinned and add possible subdependencies
+pip==23.3.2
+wheel==0.42.0
+cookiecutter==2.5.0
+mxdev==3.1.0
+prettyconf==2.2.1

backend/version.txt

@@ -1 +1 @@
-6.0.6
+6.0.9

devops/zodbconvert/README.txt

@@ -0,0 +1,23 @@
+Convert postgresql to filestorage
+=================================
+
+In test and production the plone.org website stores the content data in a
+Postgresql database using the relstorage driver. If you want to create
+a local filestorage copy of the data to use for local development, you
+can use the zodbconvert utility.
+
+First make sure you are running a local postgresql server with the Plone
+content database. The zodbconvert config file assume you use the (datbase)
+settings as configured in the docker-compose.yml in the project root. This
+compose file starts a local postgresql container on 127.0.0.1:5432 with
+ploneorg/ploneorg/ploneorg as the database/role/password . 
+
+zodbconvert writes the data to the ./data directory in the project root,
+inside ./data/filestorage and ./data/blobs .  
+
+Now run zodbconvert from the project root: 
+
+> ./backend/bin/zodbconvert ./devops/zodbconvert/relstorage.cfg
+
+This can take 5-15 minutes, depending on your machine specs. 
+

devops/zodbconvert/relstorage.cfg

@@ -0,0 +1,11 @@
+<relstorage source>
+  blob-dir /tmp/blobcache
+  shared-blob-dir false 
+  <postgresql>
+    dsn dbname='ploneorg' user='ploneorg' host='127.0.0.1' port='5432' password='ploneorg'
+  </postgresql>
+</relstorage>
+<filestorage destination>
+  path ./data/filestorage/Data.fs
+  blob-dir ./data/blobs
+</filestorage>

docker-compose.override.yml.orig

@@ -1,16 +1,5 @@
-version: "3"
+version: "3.8"
 
 # To use this override, add a data directory
-# rename override.yml.orig to override.yml
-
-services:
-  backend:
-    environment:
-      - RELSTORAGE_DSN: ""
-    volumes:
-      - ${PROJECT_DIR}/data:/data
-
-services:
-  db: 
-    profiles:
-      - relstorage
+# rename override.yml.orig to override.yml and fill it with overrides
+# 

docker-compose.yml

@@ -3,17 +3,13 @@ version: "3.8"
 services:
 
   db:
-    image: postgres:14.2
+    image: postgres:14.10
     environment:
       POSTGRES_USER: ploneorg
       POSTGRES_PASSWORD: ploneorg
       POSTGRES_DB: ploneorg
-    command: postgres -c shared_buffers=4GB -c effective_cache_size=12GB -c maintenance_work_mem=1GB -c wal_buffers=16MB -c random_page_cost=1.1 -c effective_io_concurrency=200 -c work_mem=20971kB -c min_wal_size=2GB -c max_wal_size=8GB -c wal_keep_size=1GB -c max_locks_per_transaction=512
+    command: postgres -c shared_buffers=2GB -c effective_cache_size=4GB -c maintenance_work_mem=1GB -c wal_buffers=16MB -c random_page_cost=1.1 -c effective_io_concurrency=200 -c work_mem=20971kB -c min_wal_size=2GB -c max_wal_size=8GB -c wal_keep_size=1GB -c max_locks_per_transaction=512
     ports:
-      - 5432:5432
+      - "127.0.0.1:5432:5432"
     volumes:
-      - data:/var/lib/postgresql/data
-
-
-volumes:
-  data: {}
+      - ./data/postgresql:/var/lib/postgresql/data

frontend/package.json

@@ -150,16 +150,16 @@
   },
   "dependencies": {
     "@eeacms/volto-accordion-block": "^6.0.0",
-    "@eeacms/volto-matomo": "4.1.3",
+    "@eeacms/volto-matomo": "4.2.1",
     "@fontsource/assistant": "4.5.11",
     "@fortawesome/fontawesome-svg-core": "1.2.36",
     "@fortawesome/free-regular-svg-icons": "5.15.4",
     "@fortawesome/free-solid-svg-icons": "5.15.4",
     "@fortawesome/react-fontawesome": "0.1.15",
     "@kitconcept/volto-blocks-grid": "^5.2.0",
-    "@kitconcept/volto-slider-block": "^4.0.0",
-    "@plone-collective/volto-authomatic": "1.3.0",
-    "@plone/volto": "16.25.0",
+    "@kitconcept/volto-slider-block": "^4.3.0",
+    "@plone-collective/volto-authomatic": "^2.0.0",
+    "@plone/volto": "16.30.1",
     "react-slick": "0.28.1",
     "slick-carousel": "1.8.1",
     "volto-dropdownmenu": "2.4.3",