Merge pull request #22 from slusarz/vitepress_doc

Vitepress doc

.github/workflows/documentation.yml

@@ -0,0 +1,49 @@
+name: Test Building (and Deploy Documentation to GitHub Pages)
+
+on:
+  workflow_dispatch:
+  push:
+
+permissions:
+  contents: read
+  pages: write
+  id-token: write
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+      - name: Setup Node
+        uses: actions/setup-node@v3
+        with:
+          node-version: 18
+          cache: npm
+      - name: Setup Pages
+        if: ${{ github.repository == 'dovecot/imaptest' && github.ref == 'refs/heads/main' }}
+        uses: actions/configure-pages@v3
+      - name: Install Dependencies
+        run: npm ci
+      - name: Build with VitePress
+        run: |
+          npm run docs:build
+          touch docs/.vitepress/dist/.nojekyll
+      - name: Upload Artifact
+        if: ${{ github.repository == 'dovecot/imaptest' && github.ref == 'refs/heads/main' }}
+        uses: actions/upload-pages-artifact@v2
+        with:
+          path: docs/.vitepress/dist
+
+  deploy:
+    if: ${{ github.repository == 'dovecot/imaptest' && github.ref == 'refs/heads/main' }}
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+    name: Deploy
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v2

.gitignore

@@ -53,3 +53,7 @@ Makefile.in
 
 src/imaptest
 src/rawlog.*
+
+docs/.vitepress/cache
+docs/.vitepress/dist
+node_modules/

README.md

@@ -3,8 +3,8 @@ IMAP Server Tester
 
 ImapTest is a generic IMAP server compliancy tester that works with all IMAP servers. It supports:
 
- - Stress testing with state tracking. ImapTest sends random commands to the server and verifies that server's output looks correct.
- - Scripted testing where it runs a list of predefined scripted tests and verifies that server returns expected output.
+ - Stress testing with state tracking
+ - Scripted testing
  - Benchmarking
 
-Please see https://imapwiki.org/ImapTest/ for more information.
+Please see https://dovecot.github.io/imaptest/ for more information.

docs/.vitepress/config.mts

@@ -0,0 +1,66 @@
+import { defineConfig } from 'vitepress'
+
+// https://vitepress.dev/reference/site-config
+export default defineConfig({
+  base: '/imaptest/',
+  title: "IMAP Server Tester",
+  lang: 'en-US',
+  description: "Documentation for the IMAP Server Tester",
+  srcExclude: [ '/DOCS.md' ],
+
+  sitemap: {
+    hostname: 'https://dovecot.github.io/imaptest/',
+  },
+
+  themeConfig: {
+    // https://vitepress.dev/reference/default-theme-config
+    nav: [
+      { text: 'Home', link: '/' },
+      { text: 'Configuration', link: '/configuration' }
+    ],
+
+    sidebar: [
+      {
+        text: 'Introduction',
+        items: [
+          { text: 'Features', link: '/features' },
+          { text: 'Benchmarking', link: '/benchmarking' },
+          { text: 'Compliancy', link: '/compliancy' },
+        ]
+      },
+      {
+        text: 'Installation',
+        items: [
+          { text: 'Installation', link: '/installation' },
+        ]
+      },
+      {
+        text: 'Operation',
+        items: [
+          { text: 'Configuration', link: '/configuration' },
+          { text: 'Scripted Testing', link: '/scripted_test' },
+          { text: 'States', link: '/states' },
+          { text: 'Profile', link: '/profile' },
+          { text: 'Examples', link: '/examples' },
+        ]
+      },
+      {
+        items: [
+          { text: 'Authors', link: '/authors' },
+          { text: 'License', link: '/license' },
+        ]
+      },
+    ],
+
+    socialLinks: [
+      { icon: 'github', link: 'https://github.com/dovecot/imaptest/' }
+    ],
+
+    outline: 'deep',
+    externalLinkIcon: true,
+
+    search: {
+      provider: 'local'
+    },
+  }
+})

docs/authors.md

@@ -0,0 +1,9 @@
+---
+layout: doc
+---
+
+# ImapTest Authors
+
+ImapTest was originally written by [Timo Sirianen](mailto:[email protected]).
+
+It is now maintained by the Dovecot Team at [Open-Xchange](http://www.open-xchange.com/).

docs/benchmarking.md

@@ -0,0 +1,61 @@
+---
+layout: doc
+---
+
+# IMAP Server Benchmarking
+
+## Overview
+
+"Which IMAP server is the fastest?" is a difficult question to answer, because the answer completely depends on what IMAP clients are used.
+
+There are basically two types of IMAP clients:
+
+1. Caching IMAP clients which fetch new messages only once from the server. After that the only metadata the client is interested of is the message's flags. Most real IMAP clients belong to this category (e.g. Outlook, Thunderbird, Apple Mail).
+1. Non-caching IMAP clients keep fetching the same messages over and over again. Webmails typically belong to this category.
+
+Some IMAP servers cache commonly used metadata for non-caching clients to avoid reading and parsing the message multiple times. This has however some problems:
+
+* If it's done for users who use only caching IMAP clients, it adds extra disk I/O (and consume disk space unneededly).
+* What fields should be cached? If too many are cached, it again adds extra unnecessary disk I/O. If too few are cached, all the caching was for nothing because the server has to now parse the message anyway.
+
+If clients can be placed into two categories, servers can be placed into three:
+
+1. Non-caching (or very little / non-permanently caching) servers parse the message whenever a client requests for metadata (e.g. UW-IMAP, Courier).
+1. Statically caching servers keep a predefined list of metadata permanently cached (e.g. Cyrus).
+1. Dynamically caching servers change the list of permanently cached metadata based on what clients actually use (e.g. Dovecot).
+
+We can describe the client <-> server performance as a table (assumes theoretically optimal implementations):
+
+|                        | **Non-Caching Server** | **Statically Caching Server**   | **Dynamically Caching Server** |
+| ---------------------- | ---------------------- | ------------------------------- | ------------------------------ |
+| **Caching Client**     | Optimal performance    | Wastes disk I/O and disk space. | Near-optimal performance       |
+| **Non-Caching Client** | Worst performance      | Optimal performance if cached fields match exactly the wanted fields. Near-optimal if all fields are cached. Bad performance if some fields are missing | Optimal performance |
+
+So when you read about IMAP benchmarks, make sure you know what caching models the benchmark tests for.
+
+## Other Issues
+
+### Mail Delivery
+
+With caching IMAP servers the metadata caching is often offloaded to MDA. This means that while the mail delivery may be slower than for non-caching IMAP servers, the IMAP performance may be better as a result. Then again if only caching clients are used, the extra disk I/O still slows down the server, even if it was done outside the actual IMAP session.
+
+### Disk I/O
+
+The performance bottleneck with most IMAP servers is disk I/O. The benchmark should reflect that by using realistic amount of disk I/O instead of getting everything cached in memory after running a while.
+
+### fsync()
+
+Some servers call `fsync()` after saving messages and possibly performing some other operations to guarantee (if sysadmin so configured) that changes have been written to disk before replying OK to client.
+
+Other servers don't use `fsync()` at all and some can be configured either way.
+
+Using `fsync()` should guarantee that mails don't get lost when the server unexpectedly dies, but it can make the performance somewhat worse, especially in benchmarks.
+
+### Dynamic Caching
+
+With dynamically caching servers, if the benchmark switches between different client behaviors for the same user the result may be bad, because the server assumes the user is using multiple clients and tries to optimize the performance for all of them.
+
+## Software
+
+* [MStone](http://mstone.sourceforge.net/) - Benchmarks for caching client performance.
+* ImapTest - Benchmarks for non-caching client performance.

docs/compliancy.md

@@ -0,0 +1,103 @@
+---
+layout: doc
+---
+
+# IMAP Server Compliancy Status
+
+## Categories
+
+### Checkpoint
+
+* Checkpoint parameter works. When issuing a CHECK command in all sessions, their state looks identical.
+   * It would actually be better to be consistent across all IMAP commands, but especially when NOOP or IDLE is used. Some servers have inconsistent IMAP session states because the client connections end up in, e.g., different servers that don't synchronize immediately. This can cause problems with clients that rely on the different sessions seeing the same state.
+
+### Recent Flag
+
+* Exactly one session sees a new message as \Recent - no more and no less.
+   * ImapTest prints an error if it notices multiple sessions having the same \Recent flag ("Message seq=.. UID=.. has \Recent flag in multiple sessions"). This happens automatically when running ImapTest with multiple clients for the same user, e.g., `imaptest user=testuser clients=10`.
+   * But missing \Recent flags isn't detected with any tests.
+
+### Atomic Flags
+
+* Flags and keywords can be added/removed in multiple sessions without one session dropping changes made by other
+   * Test by running stress test with [`own_flags`](/configuration#own-flags) parameter and multiple clients for the same user, e.g., `imaptest user=testuser clients=10 own_flags`
+      * If no errors are printed then it's likely ok.
+
+### Expunge Fetch
+
+* If message is expunged in one session, can another session that still sees the message fetch its contents?
+   * Yes: Yes, everything can be fetched
+   * Cached: Only cached data can be fetched. Message header or body can't be.
+   * No: Nothing but flags can be fetched.
+   * This needs to be tested manually to see how it behaves - there is no ready imaptest test for this. Manual testing with IMAP:
+      * Open two IMAP sessions and "SELECT INBOX" in both
+      * Session 1: "STORE 1 +FLAGS \Deleted", EXPUNGE
+      * Session 2: "FETCH 1 BODYSTRUCTURE" -> Success = Cached
+      * Session 2: "FETCH 1 BODY.PEEK[]" -> Success = Yes
+
+### Expunge Store
+
+* If message is expunged in one session, can another session update its flags?
+   * Yes: Yes, and a 3rd session also sees the change
+   * Session: Yes, but a 3rd session won't see the change
+   * Delayed: Yes, but a 3rd session won't see *any* changes until EXPUNGEs are reported
+   * Ignored: STORE command replies OK, but no change is made
+   * No: STORE command replies NO
+   * This needs to be tested manually to see how it behaves - there is no ready ImapTest test for this. Manual testing with IMAP:
+      * Open three IMAP sessions and "SELECT INBOX" in all
+      * Session 1: "STORE 1 +FLAGS \Deleted", EXPUNGE
+      * Session 2: "STORE 1,2 +FLAGS \Seen"
+      * Session 3: "FETCH 1,2 FLAGS"
+         * If neither mail shows \Seen flag, run in Session 3 NOOP and "FETCH 1 FLAGS". This should result in EXPUNGE notification and mail 2 showing \Seen flag. This result is delayed.
+
+### Failures (failures/total)
+* Number of failures using [scripted tests](/scripted_test). These numbers may not be exact all the time, because the tests are still changing.
+   * Failure groups: Each test belongs to a wider group of tests, typically testing a command or part of a command. If this count is low but individual command failure count is high, it probably means that the server has failed to implement wrong only a couple of commands.
+   * Base fails: Number of individual base IMAP4rev1 protocol commands that failed.
+   * Ext fails: Number of individual IMAP extension commands that failed. Extensions not supported aren't included in the numbers.
+   * Test using, e.g., `imaptest user=testuser test=tests/`.
+
+
+## Server Compliance
+
+::: warning
+These results are out-of-date!
+
+### Fully Compliant Servers
+
+| Server | Checkpoint | \Recent | Atomic flags | Expunge fetch | Expunge store | Failure groups | Base fails | Ext fails |
+| ------ | ---------- | ------- | ------------ | ------------- | ------------- | -------------- | ---------- | --------- |
+| [Dovecot](https://www.dovecot.org/) | Yes | Yes | Yes | Yes / Cached (depends on storage) | Yes | 0/40 | 0/403 | 0/100 |
+| [Panda IMAP](https://github.com/jonabbey/panda-imap/) 2008, mix format | Yes | Yes | Yes | Yes | Session | 0/34 | 0/328 | 0/97 |
+| [SurgeMail](http://netwinsite.com/surgemail/) 5.0h3 | Yes | Yes | Yes | Yes | Yes | 0/35 | 0/342 | 0/26 |
+
+
+### Non-Compliant Servers
+
+| Server | Checkpoint | \Recent | Atomic flags | Expunge fetch | Expunge store | Failure groups | Base fails | Ext fails |
+| ------ | ---------- | ------- | ------------ | ------------- | ------------- | -------------- | ---------- | --------- |
+| [UW-IMAP](http://www.washington.edu/imap/) 2007b, mix format | Yes | Yes | Yes | Yes | Session | 2/34 | 0/328 | 6/53 |
+| [Isode M-Box](http://www.isode.com/products/m-box.html) 14.3a0 | No | Unreliable | Yes | Cached | Ignored | 4/40 | 1/408 | 8/112 |
+| [CommuniGate Pro](http://www.communigate.com/community/) 5.2.1 | Yes | Yes | Yes | Cached (some) | No | 8/34 | 8/328 | 0/0 |
+| [Cyrus](https://www.cyrusimap.org/) 3.0.10 | No | Unreliable | Bugs | Cached | Delayed | 0/35 | 0/366 | 0/100 |
+| [Sun Java Messaging Server](http://www.sun.com/software/products/messaging_srvr/index.xml) 6.3-0.15 | No | Unreliable | Bugs | Yes | Delayed | 9/34 | 17/328 | 9/21 |
+| [Archiveopteryx](http://www.archiveopteryx.org/) 3.0.3 | No | Unreliable | No | No | No | 13/38 | 25/346 | 6/26 |
+| [Courier](http://www.courier-mta.org/imap/) 4.3.1 | Yes | Unreliable | Yes | No | No | 18/34 | 33/328 | 20/53 |
+| [GMail](http://www.gmail.com/) 2012-08-12 | No | Not implemented | Bugs | Cached (some) | Ignored | 12/49 | 66/376 | 0/0 |
+| [Zarafa Collaboration Platform](http://www.zarafa.com/) 7.1.10 | No | Unreliable | Yes | No | No | 6/35 | 14/340 | 0/0 |
+| [Zimbra](http://www.zimbra.com/) 5.0.5 | Yes | Yes | Yes | Cached | No | 8/34 | 50/328 | 2/33 |
+| [MS Exchange](http://www.microsoft.com/exchange/default.mspx) | No | Unreliable | ? | No | No | 16/40 | 52/287 | 0/0 |
+| [Citadel](http://www.citadel.org/) 7.36 | No | Unreliable | Yes | Yes | Session (flag changes are never seen by other sessions) | 19/34 | 98/328 | 0/0 |
+| [hMailServer](http://www.hmailserver.com/) 5.3.3 b1879 | ? | ? | ? | ? | ? | Fails hardcoded OK response format test. Other tests don't work, if OK response test fails. | | |
+
+### Major Problems with Multiple Connections
+
+Makes further testing difficult and **MAY CAUSE ACCIDENTAL MAIL LOSS!**
+
+| Server | Checkpoint | \Recent | Atomic flags | Expunge fetch | Expunge store | Failure groups | Base fails | Ext fails |
+| ------ | ---------- | ------- | ------------ | ------------- | ------------- | -------------- | ---------- | --------- |
+| [dbmail](http://www.dbmail.org/) 3.0.2 | ? | Unreliable | ? | Yes | Yes |  9/361 - UID/sequence mapping becomes wrong |
+| [IBM Domino](http://www.ibm.com/software/lotus/products/notes/) 8.0 | No | Unreliable | ? | ? | No | 16/34 - Too many EXPUNGEs are sent, EXISTS is dropped before sending EXPUNGEs, FETCHing with valid messagesets produce errors |
+| [Kerio Mail Server](http://www.kerio.eu/kms_home.html) 6.5.1 | ? | ? | ? | ? | ? | 18/34 - EXPUNGEs are sent wrong |
+| [Axigen](http://www.axigen.com/) 7.1.2 | ? | Unreliable | Yes | Broken | Broken | 19/312 - FETCH/STORE sends EXPUNGEs immediately |
+:::