Add benchmarks and tune up the benchmark app

.gitignore

@@ -25,3 +25,4 @@ Gemfile.lock
 /bench/public/assets
 /bench/.env*
 !/bench/.env*.erb
+/bench/k6

README.md

@@ -81,14 +81,130 @@ The options are:
 
 Messages are autotrimmed based upon the `message_retention` setting to determine how long messages are to be kept around. If no `message_retention` is given or parsing fails, it defaults to `1.day`. Messages are trimmed when a messsage is broadcast.
 
-Autotrimming can negatively impact performance depending on your workload because it is potentially doing a delete on broadcast. If
+Autotrimming can negatively impact performance slightly depending on your workload because it is potentially doing a delete on broadcast. If
 you would prefer, you can disable autotrimming by setting `autotrim: false` and you can manually enqueue the job later, `SolidCable::TrimJob.perform_later`, or run it on a recurring interval out of band.
 
+
 ## Upgrading
 
 If you have already installed Solid Cable < 3 and are upgrading to version 3,
 run `solid_cable:update` to install a new migration.
 
+
+## Benchmarks
+
+Inside the `bench` directory there is a minimal Rails app that is used to benchmark.
+You are welcome to update the config/deploy.yml file to point to your own server
+if you want to deploy the app to your own server and run benchmarks.
+
+To benchmark we use [k6](https://k6.io). Most of the setup was gotten from this
+[article](https://evilmartians.com/chronicles/real-time-stress-anycable-k6-websockets-and-yabeda).
+1. Install k6
+1. Install xk6-cable by running `xk6 build --with
+   github.com/anycable/xk6-cable`. This will output a custom k6 binary.
+1. Run the load test with `./k6 run loadtest.js`
+    - This script takes a variety of ENV variables:
+        - WS_URL: The url to send websocket connections
+        - MAX: The number of virtual users to hit the server with
+        - TIME: The duration of the load test
+        - MESSAGES_NUM: The number of messages each VU will send to the server
+
+
+#### Results
+
+Our loadtest is run on a Hetzner CCX13, with a MESSAGES_NUM of 5, and a TIME of 90.
+
+###### SQLite
+
+With a polling interval of 0.1 seconds and autotrimming enabled.
+
+100 VUs
+```
+rtt..................: avg=135.82ms min=50ms     med=138ms    max=357ms    p(90)=174ms    p(95)=195ms
+ws_connecting........: avg=205.81ms min=149.35ms med=199.01ms max=509.48ms p(90)=254.04ms p(95)=261.77ms
+```
+250 VUs
+```
+rtt..................: avg=146.24ms min=50ms     med=144ms    max=435ms p(90)=209ms   p(95)=234.04ms
+ws_connecting........: avg=222.15ms min=146.47ms med=208.57ms max=1.3s  p(90)=263.6ms p(95)=284.18ms
+```
+500 VUs
+```
+rtt..................: avg=271.79ms min=48ms     med=205ms    max=1.15s p(90)=558ms    p(95)=660ms
+ws_connecting........: avg=248.81ms min=145.89ms med=221.89ms max=1.38s p(90)=290.41ms p(95)=322.2ms
+```
+750 VUs
+```
+rtt..................: avg=548.27ms min=51ms     med=438ms    max=5.19s  p(90)=1.18s  p(95)=1.29s
+ws_connecting........: avg=266.37ms min=144.06ms med=224.93ms max=2.33s  p(90)=298ms  p(95)=342.87ms
+```
+
+With trimming disabled
+250 VUs
+```
+rtt..................: avg=139.47ms min=48ms     med=142ms    max=807ms p(90)=189ms    p(95)=214ms
+ws_connecting........: avg=212.58ms min=146.19ms med=196.25ms max=1.25s p(90)=255.74ms p(95)=272.44ms
+```
+
+With a polling interval of 0.01 seconds it becomes comparable to Redis
+250 VUs
+```
+rtt..................: avg=84.22ms  min=43ms     med=69ms     max=416ms p(90)=137ms    p(95)=150ms
+ws_connecting........: avg=219.37ms min=144.71ms med=200.77ms max=2.17s p(90)=265.23ms p(95)=290.83ms
+```
+
+###### Redis
+
+This instance was hosted on the same machine.
+
+100 VUs
+```
+rtt..................: avg=68.95ms  min=41ms     med=56ms     max=6.23s  p(90)=114ms   p(95)=129ms
+ws_connecting........: avg=211.09ms min=153.23ms med=195.69ms max=1.44s  p(90)=258.1ms p(95)=272.23ms
+```
+250 VUs
+```
+rtt..................: avg=69.32ms  min=40ms     med=56ms     max=645ms p(90)=119ms    p(95)=135ms
+ws_connecting........: avg=212.95ms min=142.92ms med=196.31ms max=1.25s p(90)=260.25ms p(95)=273.49ms
+```
+500 VUs
+```
+rtt..................: avg=87.5ms   min=40ms     med=67ms     max=839ms p(90)=149ms    p(95)=176ms
+ws_connecting........: avg=242.62ms min=142.03ms med=213.76ms max=2.34s p(90)=291.25ms p(95)=324.04ms
+```
+750 VUs
+```
+rtt..................: avg=162.54ms min=39ms  med=123ms    max=2.26s p(90)=343.1ms  p(95)=438ms
+ws_connecting........: avg=353.08ms min=143ms med=264.15ms max=2.73s p(90)=541.36ms p(95)=1.15s
+```
+
+
+###### MySQL
+
+With a polling interval of 0.1 seconds and autotrimming enabled. This instance
+was also hosted on the same machine.
+
+100 VUs
+```
+rtt..................: avg=136.02ms min=51ms     med=137ms    max=877ms p(90)=168.1ms  p(95)=198ms
+ws_connecting........: avg=207.76ms min=151.93ms med=196.74ms max=1.21s p(90)=249.91ms p(95)=260.37ms
+```
+250 VUs
+```
+rtt..................: avg=159.33ms min=51ms    med=149ms    max=559ms p(90)=236ms    p(95)=263ms
+ws_connecting........: avg=232.38ms min=151.6ms med=218.09ms max=1.38s p(90)=287.99ms p(95)=324.6ms
+```
+500 VUs
+```
+rtt..................: avg=441.07ms min=51ms     med=312ms    max=2.29s  p(90)=931ms    p(95)=1.07s
+ws_connecting........: avg=256.73ms min=152.23ms med=231.02ms max=2.31s  p(90)=305.69ms p(95)=340.83ms
+```
+750 VUs
+```
+rtt..................: avg=822.08ms min=51ms     med=732ms    max=5.05s  p(90)=1.76s    p(95)=1.97s
+ws_connecting........: avg=278.08ms min=146.66ms med=236.35ms max=2.37s  p(90)=318.17ms p(95)=374.98ms
+```
+
 ## License
 
 The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT).

bench/.dockerignore

@@ -0,0 +1,52 @@
+# See https://docs.docker.com/engine/reference/builder/#dockerignore-file for more about ignoring files.
+
+# Ignore git directory.
+/.git/
+
+# Ignore bundler config.
+/.bundle
+
+# Ignore all default key files.
+/config/master.key
+/config/credentials/*.key
+
+# Ignore all environment files.
+/.env*
+!/.env.example
+
+# Ignore all logfiles and tempfiles.
+/log/*
+/tmp/*
+!/log/.keep
+!/tmp/.keep
+tmp/*
+log/*
+tmp/cache/assets/*
+
+# Ignore pidfiles, but keep the directory.
+/tmp/pids/*
+!/tmp/pids/
+!/tmp/pids/.keep
+
+# Ignore storage (uploaded files in development and any SQLite databases).
+/storage/*
+!/storage/.keep
+/tmp/storage/*
+/tmp/cache/*
+tmp/storage/*
+tmp/cache/*
+/coverage/*
+coverage/*
+!/tmp/storage/.keep
+
+# Ignore assets.
+/node_modules/
+/app/assets/builds/*
+!/app/assets/builds/.keep
+/public/assets
+/vendor/bundle
+test/*
+/test/*
+tags
+/tags
+k6

bench/.ruby-version

@@ -0,0 +1 @@
+3.3.5

bench/Dockerfile

@@ -7,30 +7,31 @@
 # For a containerized dev environment, see Dev Containers: https://guides.rubyonrails.org/getting_started_with_devcontainer.html
 
 # Make sure RUBY_VERSION matches the Ruby version in .ruby-version
-ARG RUBY_VERSION=3.3.1
-FROM docker.io/library/ruby:$RUBY_VERSION-slim as base
+ARG RUBY_VERSION=3.3.5
+FROM docker.io/library/ruby:$RUBY_VERSION-slim AS base
 
 # Rails app lives here
 WORKDIR /rails
 
 # Install base packages
 RUN apt-get update -qq && \
-    apt-get install --no-install-recommends -y curl libjemalloc2 libsqlite3-0 libvips && \
-    rm -rf /var/lib/apt/lists /var/cache/apt/archives
+    apt-get install --no-install-recommends -y curl libjemalloc2 && \
+    apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Set production environment
 ENV RAILS_ENV="production" \
+    BUNDLE_WITHOUT="development:test:linters:deploy" \
     BUNDLE_DEPLOYMENT="1" \
     BUNDLE_PATH="/usr/local/bundle" \
     BUNDLE_WITHOUT="development"
 
 # Throw-away build stage to reduce size of final image
-FROM base as build
+FROM base AS build
 
 # Install packages needed to build gems
 RUN apt-get update -qq && \
     apt-get install --no-install-recommends -y build-essential git pkg-config && \
-    rm -rf /var/lib/apt/lists /var/cache/apt/archives
+    apt-get clean && rm -rf /var/cache/apt/archives /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
 # Install application gems
 COPY Gemfile Gemfile.lock ./
@@ -41,14 +42,9 @@ RUN bundle install && \
 # Copy application code
 COPY . .
 
-# Precompile bootsnap code for faster boot times
-RUN bundle exec bootsnap precompile app/ lib/
-
-# Precompiling assets for production without requiring secret RAILS_MASTER_KEY
-RUN SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:precompile
-
-
-
+RUN bundle exec bootsnap precompile app/ lib/ && \
+    SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:precompile && \
+    rm -rf vendor/ruby/3.3.0/cache
 
 # Final stage for app image
 FROM base

bench/Gemfile.lock

@@ -99,7 +99,7 @@ GIT
 
 GIT
   remote: https://github.com/rails/solid_cable.git
-  revision: 9403858cbe5eb9866fb9a86395e8f2182aa3ed46
+  revision: 2971cf3983ee0ed9359f69889f9a93f473fec387
   branch: main
   specs:
     solid_cable (2.0.2)

bench/config/cable.yml

@@ -10,9 +10,9 @@ test:
 
 production:
   # adapter: redis
-  # url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %>
+  # url: <%= ENV.fetch("REDIS_URL") { "redis://#{ENV["HOST"]}:6379/1" } %>
   # channel_prefix: cablebench_production
 
   adapter: solid_cable
   message_retention: 1.day
-  polling_interval: 0.2.seconds
+  polling_interval: 0.1.seconds

bench/config/credentials.yml.enc

@@ -1 +1 @@
-AbJCrZ0ZzNVVCpxO79c33dE6uL0RbYK1VQy1Ve5uu88wtzO5V5mwh/deUHzvsIjEu8qwVCRvYdfmozEePohQJKfu79m6YqauOSGLBEAWYC44BmZMn07VeAnSmIp/23l6HCaCsiSxlBPVgc0yAgk2wugl/utI+87L0cqJ/IJX1xIXlr6w5JmFl+BwSm+c68+y27GsajRbnqXYvqglLhvBVPNq3lniLG9mRG5GZGuSymitX4b9POvxpXMrSeKObtD5vanP/m48Tn02xPNHrGBuKyObCQPJBYgyzHTVFsDp+2bARDfW3OwOTHfa3xmKKWoo6WkmibwLRvToSByeCkjCKimuVCEg2/0kpTd+foBi5om5cymRlP4S0em8wH5LSBaybPg5MwTdTaSmCMhAz8NYW+9ECEZFYVwukJ6Bj5iI8tWFqVpf4MS7qYmGKaIkPpDIaw+dmKI7CMqxhFo/VtBS7e3OzIZ9SLulHTzfFCZVN7tF94HZFX/HwnG2ZkXJ39LAcPQRs0oEZcwqURCCj9LXYudnTTDnimKelcZ44IEqq1Ijy4hogYzCE3Fb3DQhV7EKVsS/AX80dPKtTKR1TVraGrAMZPJUMRk=--akhG+9kRFAj1PeyQ--hOH/OfBg3ixM7g3FbC3laA==
+7lxKveoOh/YSo6BSUdrNHtg29iVtfI9AinJgpSiiQkSWpoVPHluT1PNVQpleQhs3LqM0WP4aMBX12xzUP843RI2LYQmtUmbjkJtfkMs/dCijfslkwfrwSI8ApQINOTwVG3fbYdFKe5N8C2Qhqbch+vMDktSL/eTXw7x1dSaUUjRF1zfEzi+xtn8D4nbDSv3m5tbUAzG2D3NU2bYIBaMtSNlnu0orY6RFPi8HRnGArsoMd41cBxUUJP6a23jatCGTl7BRrwRncBpoMKPbimiqE2YpE1XD+A25qcjA/1kxneSVvRl+0S0lUqVbUO9L84tsj9prt8YWKTeDWP9MWu9wGvyVoUb9eDMf15k3soBrKvG2++oZ6t/2S7pIg9gGEMBcRZwq9nHRnaS5SEhUFBmb4yr40/qWdpKGFKGAA/9al3sD3ChAlYXc5zQ81V4+o3hCxapvWbUr--2KQ9UYDapBokyl5i--Me6kHSK8rQVQyCeZM2w1lw==

bench/config/database.yml

@@ -27,13 +27,8 @@ production:
   # adapter: trilogy
   # pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 50 } %>
   # timeout: 5000
-  # database: cablebench
+  # database: solid_cable
   # user: root
-  # password: password
-  # host: <%= Rails.application.credentials.trilogy.host %>
-  #
-  # username: cablebench
-  # password: <%= Rails.application.credentials.mysql.password %>
-  # host: <%= Rails.application.credentials.mysql.host %>
-  # port: 25060
+  # password: <%= ENV["MYSQL_ROOT_PASSWORD"] %>
+  # host: <%= ENV["HOST"] %>
   # ssl: true

bench/config/deploy.yml

@@ -1,9 +1,9 @@
-service: cablebench
-image: npezza/cablebench
+service: solidcable
+image: npezza/solid_cable
 asset_path: /rails/public/assets
 servers:
   web:
-    - 178.156.138.21
+    - <%= ENV["HOST"] %>
 registry:
   username: npezza
   password:
@@ -12,9 +12,11 @@ registry:
 env:
   secret:
     - RAILS_MASTER_KEY
+    - MYSQL_ROOT_PASSWORD
+    - HOST
 
 volumes:
-  - "cablebench:/data"
+  - "solidcable:/data"
 builder:
   context: "."
   local:
@@ -23,7 +25,7 @@ builder:
 accessories:
   mysql:
     image: mysql:8.3
-    host: 178.156.138.21
+    host: "<%= ENV['HOST'] %>"
     port: 3306
     env:
       clear:
@@ -34,3 +36,10 @@ accessories:
       - config/init.sql:/docker-entrypoint-initdb.d/setup.sql
     directories:
       - data:/var/lib/mysql
+  redis:
+    image: redis:latest
+    host: "<%= ENV['HOST'] %>"
+    port: 6379
+    cmd: "redis-server"
+    volumes:
+      - /var/lib/redis:/data

bench/loadtest.js

@@ -5,7 +5,7 @@ import { Trend } from "k6/metrics";
 
 let rttTrend = new Trend("rtt", true);
 
-const WS_URL = __ENV.WS_URL || "wss://felling.app/cable";
+const WS_URL = __ENV.WS_URL || "wss://solid-cable.dev/cable";
 const WS_COOKIE = __ENV.WS_COOKIE; // we need a valid cookie to authorize request
 const MAX = parseInt(__ENV.MAX || "20");
 // Total test duration