diff --git a/.gitignore b/.gitignore index ba9636d..afb9b2b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ Gemfile.lock /bench/public/assets /bench/.env* !/bench/.env*.erb +/bench/k6 diff --git a/README.md b/README.md index 88bb3c0..d2e439e 100644 --- a/README.md +++ b/README.md @@ -81,14 +81,130 @@ The options are: Messages are autotrimmed based upon the `message_retention` setting to determine how long messages are to be kept around. If no `message_retention` is given or parsing fails, it defaults to `1.day`. Messages are trimmed when a messsage is broadcast. -Autotrimming can negatively impact performance depending on your workload because it is potentially doing a delete on broadcast. If +Autotrimming can negatively impact performance slightly depending on your workload because it is potentially doing a delete on broadcast. If you would prefer, you can disable autotrimming by setting `autotrim: false` and you can manually enqueue the job later, `SolidCable::TrimJob.perform_later`, or run it on a recurring interval out of band. + ## Upgrading If you have already installed Solid Cable < 3 and are upgrading to version 3, run `solid_cable:update` to install a new migration. + +## Benchmarks + +Inside the `bench` directory there is a minimal Rails app that is used to benchmark. +You are welcome to update the config/deploy.yml file to point to your own server +if you want to deploy the app to your own server and run benchmarks. + +To benchmark we use [k6](https://k6.io). Most of the setup was gotten from this +[article](https://evilmartians.com/chronicles/real-time-stress-anycable-k6-websockets-and-yabeda). +1. Install k6 +1. Install xk6-cable by running `xk6 build --with + github.com/anycable/xk6-cable`. This will output a custom k6 binary. +1. Run the load test with `./k6 run loadtest.js` + - This script takes a variety of ENV variables: + - WS_URL: The url to send websocket connections + - MAX: The number of virtual users to hit the server with + - TIME: The duration of the load test + - MESSAGES_NUM: The number of messages each VU will send to the server + + +#### Results + +Our loadtest is run on a Hetzner CCX13, with a MESSAGES_NUM of 5, and a TIME of 90. + +###### SQLite + +With a polling interval of 0.1 seconds and autotrimming enabled. + +100 VUs +``` +rtt..................: avg=135.82ms min=50ms med=138ms max=357ms p(90)=174ms p(95)=195ms +ws_connecting........: avg=205.81ms min=149.35ms med=199.01ms max=509.48ms p(90)=254.04ms p(95)=261.77ms +``` +250 VUs +``` +rtt..................: avg=146.24ms min=50ms med=144ms max=435ms p(90)=209ms p(95)=234.04ms +ws_connecting........: avg=222.15ms min=146.47ms med=208.57ms max=1.3s p(90)=263.6ms p(95)=284.18ms +``` +500 VUs +``` +rtt..................: avg=271.79ms min=48ms med=205ms max=1.15s p(90)=558ms p(95)=660ms +ws_connecting........: avg=248.81ms min=145.89ms med=221.89ms max=1.38s p(90)=290.41ms p(95)=322.2ms +``` +750 VUs +``` +rtt..................: avg=548.27ms min=51ms med=438ms max=5.19s p(90)=1.18s p(95)=1.29s +ws_connecting........: avg=266.37ms min=144.06ms med=224.93ms max=2.33s p(90)=298ms p(95)=342.87ms +``` + +With trimming disabled +250 VUs +``` +rtt..................: avg=139.47ms min=48ms med=142ms max=807ms p(90)=189ms p(95)=214ms +ws_connecting........: avg=212.58ms min=146.19ms med=196.25ms max=1.25s p(90)=255.74ms p(95)=272.44ms +``` + +With a polling interval of 0.01 seconds it becomes comparable to Redis +250 VUs +``` +rtt..................: avg=84.22ms min=43ms med=69ms max=416ms p(90)=137ms p(95)=150ms +ws_connecting........: avg=219.37ms min=144.71ms med=200.77ms max=2.17s p(90)=265.23ms p(95)=290.83ms +``` + +###### Redis + +This instance was hosted on the same machine. + +100 VUs +``` +rtt..................: avg=68.95ms min=41ms med=56ms max=6.23s p(90)=114ms p(95)=129ms +ws_connecting........: avg=211.09ms min=153.23ms med=195.69ms max=1.44s p(90)=258.1ms p(95)=272.23ms +``` +250 VUs +``` +rtt..................: avg=69.32ms min=40ms med=56ms max=645ms p(90)=119ms p(95)=135ms +ws_connecting........: avg=212.95ms min=142.92ms med=196.31ms max=1.25s p(90)=260.25ms p(95)=273.49ms +``` +500 VUs +``` +rtt..................: avg=87.5ms min=40ms med=67ms max=839ms p(90)=149ms p(95)=176ms +ws_connecting........: avg=242.62ms min=142.03ms med=213.76ms max=2.34s p(90)=291.25ms p(95)=324.04ms +``` +750 VUs +``` +rtt..................: avg=162.54ms min=39ms med=123ms max=2.26s p(90)=343.1ms p(95)=438ms +ws_connecting........: avg=353.08ms min=143ms med=264.15ms max=2.73s p(90)=541.36ms p(95)=1.15s +``` + + +###### MySQL + +With a polling interval of 0.1 seconds and autotrimming enabled. This instance +was also hosted on the same machine. + +100 VUs +``` +rtt..................: avg=136.02ms min=51ms med=137ms max=877ms p(90)=168.1ms p(95)=198ms +ws_connecting........: avg=207.76ms min=151.93ms med=196.74ms max=1.21s p(90)=249.91ms p(95)=260.37ms +``` +250 VUs +``` +rtt..................: avg=159.33ms min=51ms med=149ms max=559ms p(90)=236ms p(95)=263ms +ws_connecting........: avg=232.38ms min=151.6ms med=218.09ms max=1.38s p(90)=287.99ms p(95)=324.6ms +``` +500 VUs +``` +rtt..................: avg=441.07ms min=51ms med=312ms max=2.29s p(90)=931ms p(95)=1.07s +ws_connecting........: avg=256.73ms min=152.23ms med=231.02ms max=2.31s p(90)=305.69ms p(95)=340.83ms +``` +750 VUs +``` +rtt..................: avg=822.08ms min=51ms med=732ms max=5.05s p(90)=1.76s p(95)=1.97s +ws_connecting........: avg=278.08ms min=146.66ms med=236.35ms max=2.37s p(90)=318.17ms p(95)=374.98ms +``` + ## License The gem is available as open source under the terms of the [MIT License](https://opensource.org/licenses/MIT). diff --git a/bench/.dockerignore b/bench/.dockerignore new file mode 100644 index 0000000..6d09db1 --- /dev/null +++ b/bench/.dockerignore @@ -0,0 +1,52 @@ +# See https://docs.docker.com/engine/reference/builder/#dockerignore-file for more about ignoring files. + +# Ignore git directory. +/.git/ + +# Ignore bundler config. +/.bundle + +# Ignore all default key files. +/config/master.key +/config/credentials/*.key + +# Ignore all environment files. +/.env* +!/.env.example + +# Ignore all logfiles and tempfiles. +/log/* +/tmp/* +!/log/.keep +!/tmp/.keep +tmp/* +log/* +tmp/cache/assets/* + +# Ignore pidfiles, but keep the directory. +/tmp/pids/* +!/tmp/pids/ +!/tmp/pids/.keep + +# Ignore storage (uploaded files in development and any SQLite databases). +/storage/* +!/storage/.keep +/tmp/storage/* +/tmp/cache/* +tmp/storage/* +tmp/cache/* +/coverage/* +coverage/* +!/tmp/storage/.keep + +# Ignore assets. +/node_modules/ +/app/assets/builds/* +!/app/assets/builds/.keep +/public/assets +/vendor/bundle +test/* +/test/* +tags +/tags +k6 diff --git a/bench/.ruby-version b/bench/.ruby-version new file mode 100644 index 0000000..fa7adc7 --- /dev/null +++ b/bench/.ruby-version @@ -0,0 +1 @@ +3.3.5 diff --git a/bench/Dockerfile b/bench/Dockerfile index 374924c..bb6f1c1 100644 --- a/bench/Dockerfile +++ b/bench/Dockerfile @@ -7,30 +7,31 @@ # For a containerized dev environment, see Dev Containers: https://guides.rubyonrails.org/getting_started_with_devcontainer.html # Make sure RUBY_VERSION matches the Ruby version in .ruby-version -ARG RUBY_VERSION=3.3.1 -FROM docker.io/library/ruby:$RUBY_VERSION-slim as base +ARG RUBY_VERSION=3.3.5 +FROM docker.io/library/ruby:$RUBY_VERSION-slim AS base # Rails app lives here WORKDIR /rails # Install base packages RUN apt-get update -qq && \ - apt-get install --no-install-recommends -y curl libjemalloc2 libsqlite3-0 libvips && \ - rm -rf /var/lib/apt/lists /var/cache/apt/archives + apt-get install --no-install-recommends -y curl libjemalloc2 && \ + apt-get clean && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* # Set production environment ENV RAILS_ENV="production" \ + BUNDLE_WITHOUT="development:test:linters:deploy" \ BUNDLE_DEPLOYMENT="1" \ BUNDLE_PATH="/usr/local/bundle" \ BUNDLE_WITHOUT="development" # Throw-away build stage to reduce size of final image -FROM base as build +FROM base AS build # Install packages needed to build gems RUN apt-get update -qq && \ apt-get install --no-install-recommends -y build-essential git pkg-config && \ - rm -rf /var/lib/apt/lists /var/cache/apt/archives + apt-get clean && rm -rf /var/cache/apt/archives /var/lib/apt/lists/* /tmp/* /var/tmp/* # Install application gems COPY Gemfile Gemfile.lock ./ @@ -41,14 +42,9 @@ RUN bundle install && \ # Copy application code COPY . . -# Precompile bootsnap code for faster boot times -RUN bundle exec bootsnap precompile app/ lib/ - -# Precompiling assets for production without requiring secret RAILS_MASTER_KEY -RUN SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:precompile - - - +RUN bundle exec bootsnap precompile app/ lib/ && \ + SECRET_KEY_BASE_DUMMY=1 ./bin/rails assets:precompile && \ + rm -rf vendor/ruby/3.3.0/cache # Final stage for app image FROM base diff --git a/bench/Gemfile.lock b/bench/Gemfile.lock index 219e7c6..54ece96 100644 --- a/bench/Gemfile.lock +++ b/bench/Gemfile.lock @@ -99,7 +99,7 @@ GIT GIT remote: https://github.com/rails/solid_cable.git - revision: 9403858cbe5eb9866fb9a86395e8f2182aa3ed46 + revision: 2971cf3983ee0ed9359f69889f9a93f473fec387 branch: main specs: solid_cable (2.0.2) diff --git a/bench/config/cable.yml b/bench/config/cable.yml index 902e5db..dbae202 100644 --- a/bench/config/cable.yml +++ b/bench/config/cable.yml @@ -10,9 +10,9 @@ test: production: # adapter: redis - # url: <%= ENV.fetch("REDIS_URL") { "redis://localhost:6379/1" } %> + # url: <%= ENV.fetch("REDIS_URL") { "redis://#{ENV["HOST"]}:6379/1" } %> # channel_prefix: cablebench_production adapter: solid_cable message_retention: 1.day - polling_interval: 0.2.seconds + polling_interval: 0.1.seconds diff --git a/bench/config/credentials.yml.enc b/bench/config/credentials.yml.enc index a122bca..724a4fa 100644 --- a/bench/config/credentials.yml.enc +++ b/bench/config/credentials.yml.enc @@ -1 +1 @@ -AbJCrZ0ZzNVVCpxO79c33dE6uL0RbYK1VQy1Ve5uu88wtzO5V5mwh/deUHzvsIjEu8qwVCRvYdfmozEePohQJKfu79m6YqauOSGLBEAWYC44BmZMn07VeAnSmIp/23l6HCaCsiSxlBPVgc0yAgk2wugl/utI+87L0cqJ/IJX1xIXlr6w5JmFl+BwSm+c68+y27GsajRbnqXYvqglLhvBVPNq3lniLG9mRG5GZGuSymitX4b9POvxpXMrSeKObtD5vanP/m48Tn02xPNHrGBuKyObCQPJBYgyzHTVFsDp+2bARDfW3OwOTHfa3xmKKWoo6WkmibwLRvToSByeCkjCKimuVCEg2/0kpTd+foBi5om5cymRlP4S0em8wH5LSBaybPg5MwTdTaSmCMhAz8NYW+9ECEZFYVwukJ6Bj5iI8tWFqVpf4MS7qYmGKaIkPpDIaw+dmKI7CMqxhFo/VtBS7e3OzIZ9SLulHTzfFCZVN7tF94HZFX/HwnG2ZkXJ39LAcPQRs0oEZcwqURCCj9LXYudnTTDnimKelcZ44IEqq1Ijy4hogYzCE3Fb3DQhV7EKVsS/AX80dPKtTKR1TVraGrAMZPJUMRk=--akhG+9kRFAj1PeyQ--hOH/OfBg3ixM7g3FbC3laA== \ No newline at end of file +7lxKveoOh/YSo6BSUdrNHtg29iVtfI9AinJgpSiiQkSWpoVPHluT1PNVQpleQhs3LqM0WP4aMBX12xzUP843RI2LYQmtUmbjkJtfkMs/dCijfslkwfrwSI8ApQINOTwVG3fbYdFKe5N8C2Qhqbch+vMDktSL/eTXw7x1dSaUUjRF1zfEzi+xtn8D4nbDSv3m5tbUAzG2D3NU2bYIBaMtSNlnu0orY6RFPi8HRnGArsoMd41cBxUUJP6a23jatCGTl7BRrwRncBpoMKPbimiqE2YpE1XD+A25qcjA/1kxneSVvRl+0S0lUqVbUO9L84tsj9prt8YWKTeDWP9MWu9wGvyVoUb9eDMf15k3soBrKvG2++oZ6t/2S7pIg9gGEMBcRZwq9nHRnaS5SEhUFBmb4yr40/qWdpKGFKGAA/9al3sD3ChAlYXc5zQ81V4+o3hCxapvWbUr--2KQ9UYDapBokyl5i--Me6kHSK8rQVQyCeZM2w1lw== \ No newline at end of file diff --git a/bench/config/database.yml b/bench/config/database.yml index aebd991..9e0ff5a 100644 --- a/bench/config/database.yml +++ b/bench/config/database.yml @@ -27,13 +27,8 @@ production: # adapter: trilogy # pool: <%= ENV.fetch("RAILS_MAX_THREADS") { 50 } %> # timeout: 5000 - # database: cablebench + # database: solid_cable # user: root - # password: password - # host: <%= Rails.application.credentials.trilogy.host %> - # - # username: cablebench - # password: <%= Rails.application.credentials.mysql.password %> - # host: <%= Rails.application.credentials.mysql.host %> - # port: 25060 + # password: <%= ENV["MYSQL_ROOT_PASSWORD"] %> + # host: <%= ENV["HOST"] %> # ssl: true diff --git a/bench/config/deploy.yml b/bench/config/deploy.yml index 8cad121..c74bbf9 100644 --- a/bench/config/deploy.yml +++ b/bench/config/deploy.yml @@ -1,9 +1,9 @@ -service: cablebench -image: npezza/cablebench +service: solidcable +image: npezza/solid_cable asset_path: /rails/public/assets servers: web: - - 178.156.138.21 + - <%= ENV["HOST"] %> registry: username: npezza password: @@ -12,9 +12,11 @@ registry: env: secret: - RAILS_MASTER_KEY + - MYSQL_ROOT_PASSWORD + - HOST volumes: - - "cablebench:/data" + - "solidcable:/data" builder: context: "." local: @@ -23,7 +25,7 @@ builder: accessories: mysql: image: mysql:8.3 - host: 178.156.138.21 + host: "<%= ENV['HOST'] %>" port: 3306 env: clear: @@ -34,3 +36,10 @@ accessories: - config/init.sql:/docker-entrypoint-initdb.d/setup.sql directories: - data:/var/lib/mysql + redis: + image: redis:latest + host: "<%= ENV['HOST'] %>" + port: 6379 + cmd: "redis-server" + volumes: + - /var/lib/redis:/data diff --git a/bench/k6 b/bench/k6 deleted file mode 100755 index 09faa34..0000000 Binary files a/bench/k6 and /dev/null differ diff --git a/bench/loadtest.js b/bench/loadtest.js index d9c3032..d3738d7 100644 --- a/bench/loadtest.js +++ b/bench/loadtest.js @@ -5,7 +5,7 @@ import { Trend } from "k6/metrics"; let rttTrend = new Trend("rtt", true); -const WS_URL = __ENV.WS_URL || "wss://felling.app/cable"; +const WS_URL = __ENV.WS_URL || "wss://solid-cable.dev/cable"; const WS_COOKIE = __ENV.WS_COOKIE; // we need a valid cookie to authorize request const MAX = parseInt(__ENV.MAX || "20"); // Total test duration