diff --git a/.github/workflows/backend-testing.yaml b/.github/workflows/backend-testing.yaml index 6ed264c..3d6e086 100644 --- a/.github/workflows/backend-testing.yaml +++ b/.github/workflows/backend-testing.yaml @@ -69,7 +69,7 @@ jobs: python: - 3.11 plone: - - "6.0.6" + - "6.0.9" defaults: run: diff --git a/CHANGELOG.md b/CHANGELOG.md index fc496c2..3eb1fce 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,8 +2,23 @@ --- +- Use mxdev 3.1.0 and pip 23.3.2 by overriding them in mx.ini. [fredvd] + +- List and pin backend bootstrap packages in backend/requirements-bootstrap.txt and install them in the 'bin/pip' Makefile target while passing in the full project constraints. [fredvd] + +- Add zodbconvert example and instructions in devops/zodbconvert to create filestorage version of the content database from a locally running postgresql server. [fredvd] + +- Add alternative filestorage based configuration profile for the backend with build-dev-fs (and config-fs) targets in backend/Makefile. [fredvd] + +- Update docker-compose example with latest postgresql 14 and local bind mount for the postgesql data. [fredvd] + +- Update to Volto 16.30.1. [fredvd] + +- Update to Plone backend 6.0.9 [fredvd] + - Update to Volto 16.25.0 [davisagli] + ## 1.0.13 (2023-10-09) --- diff --git a/README.md b/README.md index 8a4138d..b30677a 100644 --- a/README.md +++ b/README.md @@ -82,6 +82,16 @@ There are `Makefile` commands in place: `test-acceptance`: Start Core Cypress Acceptance Tests in dev mode +## Filestorage based backend alternative + +The default setup assumes for local development you also have a PostgreSQL server running. As a convenience an example docker compose file is provided in the project root for such a service. If you prefer to use filestorage, you can look in the backend directory its Makefile and +look at the alternative 'build-dev-fs' target, which depends on 'config-fs'. The difference with the normal 'config' target is the usages of +instance-filestorage.yaml instead of instanc.yaml. These files are used as input for the application server scaffolding and switch the storage +layer to either relstorage/postgresql or direct/filestorage. + +For both storages, if you want to develop for plone.org and need a copy of the data, you will need to as another community member (preferably from the website-team or the AI-team) for a database dump. + + ## Credits **This was generated by [cookiecutter-plone-starter](https://github.com/collective/cookiecutter-plone-starter) on 2022-10-14 12:14:12** diff --git a/backend/Dockerfile b/backend/Dockerfile index ad15f60..5cac8cc 100644 --- a/backend/Dockerfile +++ b/backend/Dockerfile @@ -1,6 +1,6 @@ -ARG PLONE_VERSION=6.0.6 +ARG PLONE_VERSION=6.0.9 FROM plone/plone-backend:${PLONE_VERSION} -ARG PLONE_VERSION=6.0.6 +ARG PLONE_VERSION=6.0.9 LABEL maintainer="Plone Foundation " \ org.label-schema.name="plone.org-backend" \ diff --git a/backend/Makefile b/backend/Makefile index 82ff654..5aba2f7 100644 --- a/backend/Makefile +++ b/backend/Makefile @@ -88,13 +88,20 @@ clean-test: ## remove test and coverage artifacts bin/pip: @echo "$(GREEN)==> Setup Virtual Env$(RESET)" $(PYTHON) -m venv . - bin/pip install -U "pip" "wheel" "cookiecutter" "mxdev" + bin/pip install -U -r requirements-bootstrap.txt -c constraints-mxdev.txt .PHONY: config config: bin/pip ## Create instance configuration @echo "$(GREEN)==> Create instance configuration$(RESET)" bin/cookiecutter -f --no-input --config-file instance.yaml gh:plone/cookiecutter-zope-instance +.PHONY: config-fs +config-fs: bin/pip ## Create instance configuration + @echo "$(GREEN)==> Create instance configuration$(RESET)" + bin/cookiecutter -f --no-input --config-file instance-filestorage.yaml gh:plone/cookiecutter-zope-instance + + + # i18n bin/i18ndude: bin/pip @echo "$(GREEN)==> Install translation tools$(RESET)" @@ -115,6 +122,13 @@ build-dev: config ## pip install Plone packages @bin/mxdev -c mx.ini bin/pip install -r requirements-mxdev.txt +.PHONY: build-dev-fs +build-dev-fs: config-fs ## pip install Plone packages + @echo "$(GREEN)==> Setup Build$(RESET)" + sed "s/PLONE_VERSION/$(PLONE_VERSION)/g" constraints-template.txt > constraints.txt + @bin/mxdev -c mx.ini + bin/pip install -r requirements-mxdev.txt + .PHONY: format format: ## Format the codebase according to our standards @echo "$(GREEN)==> Format codebase$(RESET)" diff --git a/backend/constraints.txt b/backend/constraints.txt index 9120178..cac72d1 100644 --- a/backend/constraints.txt +++ b/backend/constraints.txt @@ -1 +1 @@ --c https://dist.plone.org/release/6.0.6/constraints.txt +-c https://dist.plone.org/release/6.0.9/constraints.txt diff --git a/backend/instance-filestorage.yaml b/backend/instance-filestorage.yaml new file mode 100644 index 0000000..99c749d --- /dev/null +++ b/backend/instance-filestorage.yaml @@ -0,0 +1,13 @@ +default_context: + initial_user_name: 'admin' + initial_user_password: 'admin' + + load_zcml: + package_includes: ['ploneorg'] + + dos_protection_available: true + + db_storage: direct + + db_filestorage_location: ../data/filestorage/Data.fs + db_blobs_location: ../data/blobs \ No newline at end of file diff --git a/backend/instance.yaml b/backend/instance.yaml index 285c516..879ba1d 100644 --- a/backend/instance.yaml +++ b/backend/instance.yaml @@ -5,7 +5,9 @@ default_context: load_zcml: package_includes: ['ploneorg'] + dos_protection_available: true + db_storage: relstorage db_relstorage_postgresql_driver: psycopg2 - db_relstorage_postgresql_dsn: dbname='ploneorg' user='ploneorg' host='localhost' password='ploneorg' + db_relstorage_postgresql_dsn: dbname='ploneorg' user='ploneorg' host='127.0.0.1' password='ploneorg' db_blobs_mode: cache diff --git a/backend/mx.ini b/backend/mx.ini index efc2276..73a7184 100644 --- a/backend/mx.ini +++ b/backend/mx.ini @@ -8,6 +8,10 @@ ; version-overrides = ; example.package==2.1.0a2 +version-overrides = + pip==23.3.2 + mxdev==3.1.0 + [plone.app.vulnerabilities] url = https://github.com/plone/plone.app.vulnerabilities.git pushurl = git@github.com:plone/plone.app.vulnerabilities.git diff --git a/backend/requirements-bootstrap.txt b/backend/requirements-bootstrap.txt new file mode 100644 index 0000000..5789843 --- /dev/null +++ b/backend/requirements-bootstrap.txt @@ -0,0 +1,7 @@ +# These packages are installed after creating the virtualenv and are needed to further configure and +# install the main backend application. Keep these versions pinned and add possible subdependencies +pip==23.3.2 +wheel==0.42.0 +cookiecutter==2.5.0 +mxdev==3.1.0 +prettyconf==2.2.1 diff --git a/backend/version.txt b/backend/version.txt index b7ff151..f1bb5eb 100644 --- a/backend/version.txt +++ b/backend/version.txt @@ -1 +1 @@ -6.0.6 +6.0.9 diff --git a/devops/zodbconvert/README.txt b/devops/zodbconvert/README.txt new file mode 100644 index 0000000..1f9db88 --- /dev/null +++ b/devops/zodbconvert/README.txt @@ -0,0 +1,23 @@ +Convert postgresql to filestorage +================================= + +In test and production the plone.org website stores the content data in a +Postgresql database using the relstorage driver. If you want to create +a local filestorage copy of the data to use for local development, you +can use the zodbconvert utility. + +First make sure you are running a local postgresql server with the Plone +content database. The zodbconvert config file assume you use the (datbase) +settings as configured in the docker-compose.yml in the project root. This +compose file starts a local postgresql container on 127.0.0.1:5432 with +ploneorg/ploneorg/ploneorg as the database/role/password . + +zodbconvert writes the data to the ./data directory in the project root, +inside ./data/filestorage and ./data/blobs . + +Now run zodbconvert from the project root: + +> ./backend/bin/zodbconvert ./devops/zodbconvert/relstorage.cfg + +This can take 5-15 minutes, depending on your machine specs. + diff --git a/devops/zodbconvert/relstorage.cfg b/devops/zodbconvert/relstorage.cfg new file mode 100644 index 0000000..cc2ce2c --- /dev/null +++ b/devops/zodbconvert/relstorage.cfg @@ -0,0 +1,11 @@ + + blob-dir /tmp/blobcache + shared-blob-dir false + + dsn dbname='ploneorg' user='ploneorg' host='127.0.0.1' port='5432' password='ploneorg' + + + + path ./data/filestorage/Data.fs + blob-dir ./data/blobs + \ No newline at end of file diff --git a/docker-compose.override.yml.orig b/docker-compose.override.yml.orig index 66a5a29..83670a5 100644 --- a/docker-compose.override.yml.orig +++ b/docker-compose.override.yml.orig @@ -1,16 +1,5 @@ -version: "3" +version: "3.8" # To use this override, add a data directory -# rename override.yml.orig to override.yml - -services: - backend: - environment: - - RELSTORAGE_DSN: "" - volumes: - - ${PROJECT_DIR}/data:/data - -services: - db: - profiles: - - relstorage \ No newline at end of file +# rename override.yml.orig to override.yml and fill it with overrides +# diff --git a/docker-compose.yml b/docker-compose.yml index 7e0388e..80f6e74 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -3,17 +3,13 @@ version: "3.8" services: db: - image: postgres:14.2 + image: postgres:14.10 environment: POSTGRES_USER: ploneorg POSTGRES_PASSWORD: ploneorg POSTGRES_DB: ploneorg - command: postgres -c shared_buffers=4GB -c effective_cache_size=12GB -c maintenance_work_mem=1GB -c wal_buffers=16MB -c random_page_cost=1.1 -c effective_io_concurrency=200 -c work_mem=20971kB -c min_wal_size=2GB -c max_wal_size=8GB -c wal_keep_size=1GB -c max_locks_per_transaction=512 + command: postgres -c shared_buffers=2GB -c effective_cache_size=4GB -c maintenance_work_mem=1GB -c wal_buffers=16MB -c random_page_cost=1.1 -c effective_io_concurrency=200 -c work_mem=20971kB -c min_wal_size=2GB -c max_wal_size=8GB -c wal_keep_size=1GB -c max_locks_per_transaction=512 ports: - - 5432:5432 + - "127.0.0.1:5432:5432" volumes: - - data:/var/lib/postgresql/data - - -volumes: - data: {} + - ./data/postgresql:/var/lib/postgresql/data diff --git a/frontend/package.json b/frontend/package.json index b7bed6a..e57add9 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -150,16 +150,16 @@ }, "dependencies": { "@eeacms/volto-accordion-block": "^6.0.0", - "@eeacms/volto-matomo": "4.1.3", + "@eeacms/volto-matomo": "4.2.1", "@fontsource/assistant": "4.5.11", "@fortawesome/fontawesome-svg-core": "1.2.36", "@fortawesome/free-regular-svg-icons": "5.15.4", "@fortawesome/free-solid-svg-icons": "5.15.4", "@fortawesome/react-fontawesome": "0.1.15", "@kitconcept/volto-blocks-grid": "^5.2.0", - "@kitconcept/volto-slider-block": "^4.0.0", - "@plone-collective/volto-authomatic": "1.3.0", - "@plone/volto": "16.25.0", + "@kitconcept/volto-slider-block": "^4.3.0", + "@plone-collective/volto-authomatic": "^2.0.0", + "@plone/volto": "16.30.1", "react-slick": "0.28.1", "slick-carousel": "1.8.1", "volto-dropdownmenu": "2.4.3", diff --git a/frontend/yarn.lock b/frontend/yarn.lock index 45800ae..fac6303 100644 --- a/frontend/yarn.lock +++ b/frontend/yarn.lock @@ -1765,12 +1765,12 @@ __metadata: languageName: node linkType: hard -"@eeacms/volto-matomo@npm:4.1.3": - version: 4.1.3 - resolution: "@eeacms/volto-matomo@npm:4.1.3" +"@eeacms/volto-matomo@npm:4.2.1": + version: 4.2.1 + resolution: "@eeacms/volto-matomo@npm:4.2.1" dependencies: "@datapunt/matomo-tracker-react": 0.3.1 - checksum: 6c84a246215ebadd935e3fd8c528823d85a6eca932a0f2c3c50cc6284f921b8d0ecb87ee20442f02ef405c5dbd8f3b647f38cf692722db5d42c58d6832341d6a + checksum: 0cc007e342cb4cc78c003a3b52cd200f6e2a3b52f022721544564fed0bc5b4d35705e4461867c533c581de17aec10a84d44a4cf657a2aed1dbb787651c8129f5 languageName: node linkType: hard @@ -2517,6 +2517,13 @@ __metadata: languageName: node linkType: hard +"@juggle/resize-observer@npm:^3.4.0": + version: 3.4.0 + resolution: "@juggle/resize-observer@npm:3.4.0" + checksum: 2505028c05cc2e17639fcad06218b1c4b60f932a4ebb4b41ab546ef8c157031ae377e3f560903801f6d01706dbefd4943b6c4704bf19ed86dfa1c62f1473a570 + languageName: node + linkType: hard + "@kitconcept/volto-blocks-grid@npm:^5.2.0": version: 5.2.0 resolution: "@kitconcept/volto-blocks-grid@npm:5.2.0" @@ -2526,16 +2533,16 @@ __metadata: languageName: node linkType: hard -"@kitconcept/volto-slider-block@npm:^4.0.0": - version: 4.2.0 - resolution: "@kitconcept/volto-slider-block@npm:4.2.0" +"@kitconcept/volto-slider-block@npm:^4.3.0": + version: 4.3.0 + resolution: "@kitconcept/volto-slider-block@npm:4.3.0" dependencies: deepmerge: 4.2.2 react-slick: 0.29.0 slick-carousel: 1.8.1 peerDependencies: "@plone/volto": ^16.1.0 - checksum: d193e93dc1262016251da547b160639e2f555541eaafcae1d452a64840975ad81b0ac3510ecf12307bacafbf1f7c5f8217aee99e576422f310c518f488e1d965 + checksum: 16ebddcc3cd5c85d59e06123123a4738f80ba7f92ca7351a3270c41a1c7f8a4b21365ac7a0d9ebbda729dc539377dc215c8ab2423788578529c86d106f1cb91e languageName: node linkType: hard @@ -2729,12 +2736,12 @@ __metadata: languageName: node linkType: hard -"@plone-collective/volto-authomatic@npm:1.3.0": - version: 1.3.0 - resolution: "@plone-collective/volto-authomatic@npm:1.3.0" +"@plone-collective/volto-authomatic@npm:^2.0.0": + version: 2.0.0 + resolution: "@plone-collective/volto-authomatic@npm:2.0.0" peerDependencies: "@plone/volto": ">=13.8.0" - checksum: d882f18c267ea8ec9c41bcd263a3f99d14bd0a592384416a763b7276cc8229ea89d32b16a41c6b0d8b1396328f808ad0d595220260c60bc7351f2134b3912d6d + checksum: 16a91f0cc930f04959b1e6b1495a3a751edae8488e94a061481975c7e3a84d496431aff75086a8a4bf7931a51ee7a72b057e63ef62019980003a8872fa537f80 languageName: node linkType: hard @@ -2759,9 +2766,9 @@ __metadata: languageName: node linkType: hard -"@plone/volto@npm:16.25.0": - version: 16.25.0 - resolution: "@plone/volto@npm:16.25.0" +"@plone/volto@npm:16.30.1": + version: 16.30.1 + resolution: "@plone/volto@npm:16.30.1" dependencies: "@babel/core": ^7.0.0 "@babel/plugin-proposal-export-default-from": 7.18.9 @@ -2919,9 +2926,9 @@ __metadata: semantic-ui-react: 2.0.3 semver: 5.6.0 serialize-javascript: 3.1.0 - slate: 0.84.0 - slate-hyperscript: 0.81.3 - slate-react: 0.83.2 + slate: 0.100.0 + slate-hyperscript: 0.100.0 + slate-react: 0.98.4 start-server-and-test: 1.14.0 style-loader: 2 stylelint: 14.0.1 @@ -2943,7 +2950,7 @@ __metadata: webpack-node-externals: 3.0.0 xmlrpc: 1.3.2 yarnhook: 0.5.1 - checksum: 3046c1a2a8f225ff5e3ff2f7ef110d1df244f05e8984d1189b685572a6613ade20cb29a11b72e98077af82279281a60ab183b63f1e7eb6b18352c19c4bb66428 + checksum: 1e660f387aa8e883079a9adfc1814a876a70a51a193f816355ccd804c23282f76d7b1183939de0a5988ffe6c2b0589048a62ac632694e78635133c5df435be63 languageName: node linkType: hard @@ -11768,17 +11775,17 @@ __metadata: resolution: "frontend@workspace:." dependencies: "@eeacms/volto-accordion-block": ^6.0.0 - "@eeacms/volto-matomo": 4.1.3 + "@eeacms/volto-matomo": 4.2.1 "@fontsource/assistant": 4.5.11 "@fortawesome/fontawesome-svg-core": 1.2.36 "@fortawesome/free-regular-svg-icons": 5.15.4 "@fortawesome/free-solid-svg-icons": 5.15.4 "@fortawesome/react-fontawesome": 0.1.15 "@kitconcept/volto-blocks-grid": ^5.2.0 - "@kitconcept/volto-slider-block": ^4.0.0 - "@plone-collective/volto-authomatic": 1.3.0 + "@kitconcept/volto-slider-block": ^4.3.0 + "@plone-collective/volto-authomatic": ^2.0.0 "@plone/scripts": ^2.2.1 - "@plone/volto": 16.25.0 + "@plone/volto": 16.30.1 "@storybook/addon-actions": ^6.3.0 "@storybook/addon-controls": 6.3.0 "@storybook/addon-essentials": ^6.3.0 @@ -13184,10 +13191,10 @@ __metadata: languageName: node linkType: hard -"immer@npm:^9.0.6": - version: 9.0.21 - resolution: "immer@npm:9.0.21" - checksum: 70e3c274165995352f6936695f0ef4723c52c92c92dd0e9afdfe008175af39fa28e76aafb3a2ca9d57d1fb8f796efc4dd1e1cc36f18d33fa5b74f3dfb0375432 +"immer@npm:^10.0.3": + version: 10.0.3 + resolution: "immer@npm:10.0.3" + checksum: 76acabe6f40e752028313762ba477a5d901e57b669f3b8fb406b87b9bb9b14e663a6fbbf5a6d1ab323737dd38f4b2494a4e28002045b88948da8dbf482309f28 languageName: node linkType: hard @@ -21719,21 +21726,22 @@ __metadata: languageName: node linkType: hard -"slate-hyperscript@npm:0.81.3": - version: 0.81.3 - resolution: "slate-hyperscript@npm:0.81.3" +"slate-hyperscript@npm:0.100.0": + version: 0.100.0 + resolution: "slate-hyperscript@npm:0.100.0" dependencies: is-plain-object: ^5.0.0 peerDependencies: slate: ">=0.65.3" - checksum: cc3ba04d14e20c66eee5e5b0ca2d60fe92c0d38caf9b412153453d42f32d90bce61d57e3a00b2a4c30a7b15408bf8b14132871fd3c00f75de6ca4b49387637fe + checksum: 25c88fea4b9c6a8a57b2b6d5b1151190af815e84ebed76f43cc3c13b4831f26f55196a2a85a832692f3b6c9937aa9e75ee5db92fa603522c3ebdeecc692ff908 languageName: node linkType: hard -"slate-react@npm:0.83.2": - version: 0.83.2 - resolution: "slate-react@npm:0.83.2" +"slate-react@npm:0.98.4": + version: 0.98.4 + resolution: "slate-react@npm:0.98.4" dependencies: + "@juggle/resize-observer": ^3.4.0 "@types/is-hotkey": ^0.1.1 "@types/lodash": ^4.14.149 direction: ^1.0.3 @@ -21746,18 +21754,18 @@ __metadata: react: ">=16.8.0" react-dom: ">=16.8.0" slate: ">=0.65.3" - checksum: 0c18b9dfeab9d0e3baf5b22efb2df3216be326da886bdddab5ccce3d5291cc44229664f582a86936ba51d61efcdb80d13f75dbe5fe4b903f27b3bf5baa6fff48 + checksum: d47c01e92595870585879fd0a6fcbda7825e5ea3a4a22eca6704a2b6b91a2db4fb45c9f0ba6be648dceb5fca71211163c2fb0901614754feb448da583fee584c languageName: node linkType: hard -"slate@npm:0.84.0": - version: 0.84.0 - resolution: "slate@npm:0.84.0" +"slate@npm:0.100.0": + version: 0.100.0 + resolution: "slate@npm:0.100.0" dependencies: - immer: ^9.0.6 + immer: ^10.0.3 is-plain-object: ^5.0.0 tiny-warning: ^1.0.3 - checksum: d74de16a40571b2513dbd96efee8fe891909357a7011131f8fae738a0eace5b3c7c2da1c0ebfb1ffe66c8c933507ca618772fb607fbc9e44aa5158d3d874c5a2 + checksum: 2e12f46e5147c4d000995769041d51fa18735d28b4c734d43c48931daf01c164e820631e54ec355fc4d5d09f9fff6687782412d01f6841b6078c753fd61e60e4 languageName: node linkType: hard