Merge remote-tracking branch 'origin/develop' into erikj/py312_asyncio

.github/workflows/docs-pr-netlify.yaml

@@ -22,7 +22,7 @@ jobs:
           path: book
 
       - name: 📤 Deploy to Netlify
-        uses: matrix-org/netlify-pr-preview@v2
+        uses: matrix-org/netlify-pr-preview@v3
         with:
           path: book
           owner: ${{ github.event.workflow_run.head_repository.owner.login }}

.github/workflows/docs-pr.yaml

@@ -6,19 +6,30 @@ on:
       - docs/**
       - book.toml
       - .github/workflows/docs-pr.yaml
+      - scripts-dev/schema_versions.py
 
 jobs:
   pages:
     name: GitHub Pages
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          # Fetch all history so that the schema_versions script works.
+          fetch-depth: 0
 
       - name: Setup mdbook
         uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
         with:
           mdbook-version: '0.4.17'
 
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+
+      - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'"
+
       - name: Build the documentation
         # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
         # However, we're using docs/README.md for other purposes and need to pick a new page

.github/workflows/docs.yaml

@@ -51,12 +51,22 @@ jobs:
       - pre
     steps:
       - uses: actions/checkout@v4
+        with:
+          # Fetch all history so that the schema_versions script works.
+          fetch-depth: 0
 
       - name: Setup mdbook
         uses: peaceiris/actions-mdbook@adeb05db28a0c0004681db83893d56c0388ea9ea # v1.2.0
         with:
           mdbook-version: '0.4.17'
 
+      - name: Setup python
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+
+      - run: "pip install 'packaging>=20.0' 'GitPython>=3.1.20'"
+
       - name: Build the documentation
         # mdbook will only create an index.html if we're including docs/README.md in SUMMARY.md.
         # However, we're using docs/README.md for other purposes and need to pick a new page

.github/workflows/fix_lint.yaml

@@ -0,0 +1,52 @@
+# A helper workflow to automatically fixup any linting errors on a PR. Must be
+# triggered manually.
+
+name: Attempt to automatically fix linting errors
+
+on:
+  workflow_dispatch:
+
+jobs:
+  fixup:
+    name: Fix up
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install Rust
+        uses: dtolnay/rust-toolchain@master
+        with:
+          # We use nightly so that `fmt` correctly groups together imports, and
+          # clippy correctly fixes up the benchmarks.
+          toolchain: nightly-2022-12-01
+          components: rustfmt
+      - uses: Swatinem/rust-cache@v2
+
+      - name: Setup Poetry
+        uses: matrix-org/setup-python-poetry@v1
+        with:
+          install-project: "false"
+
+      - name: Import order (isort)
+        continue-on-error: true
+        run: poetry run isort .
+
+      - name: Code style (black)
+        continue-on-error: true
+        run: poetry run black .
+
+      - name: Semantic checks (ruff)
+        continue-on-error: true
+        run: poetry run ruff --fix .
+
+      - run: cargo clippy --all-features --fix -- -D warnings
+        continue-on-error: true
+
+      - run: cargo fmt
+        continue-on-error: true
+
+      - uses: stefanzweifel/git-auto-commit-action@v5
+        with:
+            commit_message: "Attempt to fix linting"

CHANGES.md

@@ -1,4 +1,4 @@
-# Synapse 1.97.0rc1 (2023-11-21)
+# Synapse 1.97.0 (2023-11-28)
 
 Synapse will soon be forked by Element under an AGPLv3.0 licence (with CLA, for
 proprietary dual licensing). You can read more about this here:
@@ -10,6 +10,12 @@ The Matrix.org Foundation copy of the project will be archived. Any changes need
 by server administrators will be communicated via our usual announcements channels, 
 but we are striving to make this as seamless as possible.
 
+
+No significant changes since 1.97.0rc1.
+
+
+# Synapse 1.97.0rc1 (2023-11-21)
+
 ### Features
 
 - Add support for asynchronous uploads as defined by [MSC2246](https://github.com/matrix-org/matrix-spec-proposals/pull/2246). Contributed by @sumnerevans at @beeper. ([\#15503](https://github.com/matrix-org/synapse/issues/15503))

book.toml

@@ -36,4 +36,7 @@ additional-css = [
     "docs/website_files/indent-section-headers.css",
 ]
 additional-js = ["docs/website_files/table-of-contents.js"]
-theme = "docs/website_files/theme"
+theme = "docs/website_files/theme"
+
+[preprocessor.schema_versions]
+command = "./scripts-dev/schema_versions.py"

changelog.d/15207.feature

@@ -0,0 +1 @@
+Adds on_user_login ModuleAPI callback allowing to execute custom code after (on) Auth.

changelog.d/16522.misc

@@ -0,0 +1 @@
+Clean-up unused tables.

changelog.d/16636.feature

@@ -0,0 +1 @@
+Support MSC4069: Inhibit profile propagation.

changelog.d/16661.doc

@@ -0,0 +1 @@
+Add schema rollback information to documentation.

changelog.d/16667.misc

@@ -0,0 +1 @@
+Reduce database load of pruning old `user_ips`.

changelog.d/16668.misc

@@ -0,0 +1 @@
+Reduce DB load when forget on leave setting is disabled.

changelog.d/16677.misc

@@ -0,0 +1 @@
+Ignore `encryption_enabled_by_default_for_room_type` setting when creating server notices room, since the notices will be send unencrypted anyway.

changelog.d/16695.doc

@@ -0,0 +1 @@
+Fix poetry version typo in [contributors' guide](https://matrix-org.github.io/synapse/latest/development/contributing_guide.html).

changelog.d/16697.misc

@@ -0,0 +1 @@
+Remove old full schema dumps which are no longer used.

changelog.d/16699.feature

@@ -0,0 +1 @@
+Add an autojoin setting for the notices room so users get joined directly instead of receiving an invite.

changelog.d/16700.doc

@@ -0,0 +1 @@
+Switch the example UNIX socket paths to /run. Add HAProxy example configuration for UNIX sockets.

changelog.d/16701.feature

@@ -0,0 +1 @@
+Follow redirects when downloading media over federation (per [MSC3860](https://github.com/matrix-org/matrix-spec-proposals/pull/3860)).

changelog.d/16702.misc

@@ -0,0 +1 @@
+Raise poetry-core upper bound to <=1.8.1. This allows contributors to import Synapse after `poetry install`ing with Poetry 1.6 and above. Contributed by Mo Balaa.

changelog.d/16704.misc

@@ -0,0 +1 @@
+Add a workflow to try and automatically fixup linting in a PR.

changelog.d/16707.feature

@@ -0,0 +1 @@
+Synapse now declares support for Matrix v1.7, v1.8, and v1.9.

debian/changelog

@@ -1,3 +1,9 @@
+matrix-synapse-py3 (1.97.0) stable; urgency=medium
+
+  * New Synapse release 1.97.0.
+
+ -- Synapse Packaging team <[email protected]>  Tue, 28 Nov 2023 14:08:58 +0000
+
 matrix-synapse-py3 (1.97.0~rc1) stable; urgency=medium
 
   * New Synapse release 1.97.0rc1.

docs/development/contributing_guide.md

@@ -66,7 +66,7 @@ Of their installation methods, we recommend
 
 ```shell
 pip install --user pipx
-pipx install poetry==1.5.2  # Problems with Poetry 1.6, see https://github.com/matrix-org/synapse/issues/16147
+pipx install poetry==1.5.1  # Problems with Poetry 1.6, see https://github.com/matrix-org/synapse/issues/16147
 ```
 
 but see poetry's [installation instructions](https://python-poetry.org/docs/#installation)

docs/modules/account_validity_callbacks.md

@@ -42,3 +42,16 @@ operations to keep track of them. (e.g. add them to a database table). The user
 represented by their Matrix user ID.
 
 If multiple modules implement this callback, Synapse runs them all in order.
+
+### `on_user_login`
+
+_First introduced in Synapse v1.98.0_
+
+```python
+async def on_user_login(user_id: str, auth_provider_type: str, auth_provider_id: str) -> None
+```
+
+Called after successfully login or registration of a user for cases when module needs to perform extra operations after auth.
+represented by their Matrix user ID.
+
+If multiple modules implement this callback, Synapse runs them all in order.

docs/reverse_proxy.md

@@ -181,7 +181,11 @@ frontend matrix-federation
 backend matrix
   server matrix 127.0.0.1:8008
 ```
-
+Example configuration, if using a UNIX socket. The configuration lines regarding the frontends do not need to be modified.
+```
+backend matrix
+  server matrix unix@/run/synapse/main_public.sock
+```
 
 [Delegation](delegate.md) example:
 ```

docs/server_notices.md

@@ -46,6 +46,7 @@ server_notices:
    system_mxid_display_name: "Server Notices"
    system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
    room_name: "Server Notices"
+   auto_join: true
 ```
 
 The only compulsory setting is `system_mxid_localpart`, which defines the user
@@ -55,6 +56,8 @@ room which will be created.
 `system_mxid_display_name` and `system_mxid_avatar_url` can be used to set the
 displayname and avatar of the Server Notices user.
 
+`auto_join` will autojoin users to the notices room instead of sending an invite.
+
 ## Sending notices
 
 To send server notices to users you can use the

docs/upgrade.md

@@ -88,6 +88,15 @@ process, for example:
     dpkg -i matrix-synapse-py3_1.3.0+stretch1_amd64.deb
     ```
 
+Generally Synapse database schemas are compatible across multiple versions, once
+a version of Synapse is deployed you may not be able to rollback automatically.
+The following table gives the version ranges and the earliest version they can
+be rolled back to. E.g. Synapse versions v1.58.0 through v1.61.1 can be rolled
+back safely to v1.57.0, but starting with v1.62.0 it is only safe to rollback to
+v1.61.0.
+
+<!-- REPLACE_WITH_SCHEMA_VERSIONS -->
+
 # Upgrading to v1.93.0
 
 ## Minimum supported Rust version

docs/usage/configuration/config_documentation.md

@@ -566,7 +566,7 @@ listeners:
   # Note that x_forwarded will default to true, when using a UNIX socket. Please see
   # https://matrix-org.github.io/synapse/latest/reverse_proxy.html.
   #
-  - path: /var/run/synapse/main_public.sock
+  - path: /run/synapse/main_public.sock
     type: http
     resources:
       - names: [client, federation]
@@ -3815,6 +3815,8 @@ Sub-options for this setting include:
 * `system_mxid_display_name`: set the display name of the "notices" user
 * `system_mxid_avatar_url`: set the avatar for the "notices" user
 * `room_name`: set the room name of the server notices room
+* `auto_join`: boolean. If true, the user will be automatically joined to the room instead of being invited.
+  Defaults to false. _Added in Synapse 1.98.0._
 
 Example configuration:
 ```yaml
@@ -3823,6 +3825,7 @@ server_notices:
   system_mxid_display_name: "Server Notices"
   system_mxid_avatar_url: "mxc://server.com/oumMVlgDnLYFaPVkExemNVVZ"
   room_name: "Server Notices"
+  auto_join: true
 ```
 ---
 ### `enable_room_list_search`
@@ -4215,9 +4218,9 @@ Example configuration(#2, for UNIX sockets):
 ```yaml
 instance_map:
   main:
-    path: /var/run/synapse/main_replication.sock
+    path: /run/synapse/main_replication.sock
   worker1:
-    path: /var/run/synapse/worker1_replication.sock
+    path: /run/synapse/worker1_replication.sock
 ```
 ---
 ### `stream_writers`
@@ -4403,13 +4406,13 @@ Example configuration(#2, using UNIX sockets with a `replication` listener):
 ```yaml
 worker_listeners:
   - type: http
-    path: /var/run/synapse/worker_public.sock
+    path: /run/synapse/worker_replication.sock
     resources:
-      - names: [client, federation]
+      - names: [replication]
   - type: http
-    path: /var/run/synapse/worker_replication.sock
+    path: /run/synapse/worker_public.sock
     resources:
-      - names: [replication]
+      - names: [client, federation]
 ```
 ---
 ### `worker_manhole`