From d518c0bcdff93a820fa621d2f133a017259c44de Mon Sep 17 00:00:00 2001 From: David Backeus Date: Fri, 7 Jun 2024 14:12:32 +0200 Subject: [PATCH] Add caching + don't push :latest tag on feature branches --- .env.test | 2 + .github/workflows/depot-build-and-push.yaml | 20 +++- .gitignore | 6 + .rspec | 1 + Gemfile | 6 + Gemfile.lock | 36 ++++++ app/api/cloudflare.rb | 42 +++++++ app/controllers/posts_controller.rb | 39 ++++-- app/models/cached_url.rb | 16 +++ cloudflare-csrf-worker/index.js | 111 ++++++++++++++++++ .../20240607180302_create_cached_urls.rb | 13 ++ db/schema.rb | 11 +- spec/models/cached_url_spec.rb | 26 ++++ spec/rails_helper.rb | 66 +++++++++++ spec/spec_helper.rb | 93 +++++++++++++++ 15 files changed, 473 insertions(+), 15 deletions(-) create mode 100644 .env.test create mode 100644 .rspec create mode 100644 app/api/cloudflare.rb create mode 100644 app/models/cached_url.rb create mode 100644 cloudflare-csrf-worker/index.js create mode 100644 db/migrate/20240607180302_create_cached_urls.rb create mode 100644 spec/models/cached_url_spec.rb create mode 100644 spec/rails_helper.rb create mode 100644 spec/spec_helper.rb diff --git a/.env.test b/.env.test new file mode 100644 index 0000000..6d5b9b4 --- /dev/null +++ b/.env.test @@ -0,0 +1,2 @@ +CLOUDFLARE_API_TOKEN=cloudflare-token +CLOUDFLARE_ZONE_ID=zone-id diff --git a/.github/workflows/depot-build-and-push.yaml b/.github/workflows/depot-build-and-push.yaml index 07acfaa..fe52a48 100644 --- a/.github/workflows/depot-build-and-push.yaml +++ b/.github/workflows/depot-build-and-push.yaml @@ -4,21 +4,23 @@ on: push jobs: docker: - runs-on: ubuntu-20.04 + runs-on: ubuntu-24.04 permissions: contents: read pages: write id-token: write steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: depot/setup-action@v1 - - uses: docker/login-action@v2 + - uses: docker/login-action@v3 with: username: ${{ secrets.DOCKERHUB_USERNAME }} password: ${{ secrets.DOCKERHUB_TOKEN }} - - uses: depot/build-push-action@v1 + - name: Build and push with sha + latest tag on master + if: github.ref == 'refs/heads/master' + uses: depot/build-push-action@v1 with: project: b4qlt63xvg platforms: linux/amd64,linux/arm64 @@ -26,3 +28,13 @@ jobs: tags: | reclaimthestack/rails-example:latest reclaimthestack/rails-example:sha-${{ github.sha }} + + - name: Build and push with sha tag on feature branches + if: github.ref != 'refs/heads/master' + uses: depot/build-push-action@v1 + with: + project: b4qlt63xvg + platforms: linux/amd64,linux/arm64 + push: true + tags: | + reclaimthestack/rails-example:sha-${{ github.sha }} diff --git a/.gitignore b/.gitignore index 2322ba5..ce8d2dd 100644 --- a/.gitignore +++ b/.gitignore @@ -29,3 +29,9 @@ # Ignore master key for decrypting credentials and more. /config/master.key + +# RSpec +spec/examples.txt + +# Dotenv +.env.development.local \ No newline at end of file diff --git a/.rspec b/.rspec new file mode 100644 index 0000000..9f9f2d7 --- /dev/null +++ b/.rspec @@ -0,0 +1 @@ +--require rails_helper --color diff --git a/Gemfile b/Gemfile index 958c867..d01097f 100644 --- a/Gemfile +++ b/Gemfile @@ -6,6 +6,7 @@ ruby File.read(".ruby-version") gem "rails", github: "rails/rails", branch: "main" gem "bootsnap", require: false +gem "httpx" gem "importmap-rails" gem "opengraph_parser" gem "pg" @@ -22,4 +23,9 @@ end group :development, :test do gem "dotenv-rails" + gem "rspec-rails" +end + +group :test do + gem "webmock" end diff --git a/Gemfile.lock b/Gemfile.lock index a22d4ab..44a1478 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -99,14 +99,19 @@ GEM specs: addressable (2.8.2) public_suffix (>= 2.0.2, < 6.0) + bigdecimal (3.1.8) bindex (0.8.1) bootsnap (1.16.0) msgpack (~> 1.2) builder (3.2.4) concurrent-ruby (1.2.2) connection_pool (2.4.0) + crack (1.0.0) + bigdecimal + rexml crass (1.0.6) date (3.3.3) + diff-lcs (1.5.0) dotenv (2.8.1) dotenv-rails (2.8.1) dotenv (= 2.8.1) @@ -114,6 +119,10 @@ GEM erubi (1.12.0) globalid (1.1.0) activesupport (>= 5.0) + hashdiff (1.1.0) + http-2-next (1.0.3) + httpx (1.2.5) + http-2-next (>= 1.0.3) i18n (1.12.0) concurrent-ruby (~> 1.0) importmap-rails (1.1.5) @@ -183,6 +192,25 @@ GEM connection_pool reline (0.3.3) io-console (~> 0.5) + rexml (3.2.8) + strscan (>= 3.0.9) + rspec-core (3.12.1) + rspec-support (~> 3.12.0) + rspec-expectations (3.12.2) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-mocks (3.12.4) + diff-lcs (>= 1.2.0, < 2.0) + rspec-support (~> 3.12.0) + rspec-rails (6.0.1) + actionpack (>= 6.1) + activesupport (>= 6.1) + railties (>= 6.1) + rspec-core (~> 3.11) + rspec-expectations (~> 3.11) + rspec-mocks (~> 3.11) + rspec-support (~> 3.11) + rspec-support (3.12.0) sidekiq (7.0.7) concurrent-ruby (< 2) connection_pool (>= 2.3.0) @@ -190,6 +218,7 @@ GEM redis-client (>= 0.11.0) stimulus-rails (1.2.1) railties (>= 6.0.0) + strscan (3.1.0) thor (1.2.1) timeout (0.3.2) turbo-rails (1.4.0) @@ -203,6 +232,10 @@ GEM activemodel (>= 6.0.0) bindex (>= 0.4.0) railties (>= 6.0.0) + webmock (3.23.1) + addressable (>= 2.8.0) + crack (>= 0.3.2) + hashdiff (>= 0.4.0, < 2.0.0) webrick (1.8.1) websocket-driver (0.7.5) websocket-extensions (>= 0.1.0) @@ -217,6 +250,7 @@ PLATFORMS DEPENDENCIES bootsnap dotenv-rails + httpx importmap-rails opengraph_parser pg @@ -224,10 +258,12 @@ DEPENDENCIES puma rails! redis + rspec-rails sidekiq stimulus-rails turbo-rails web-console + webmock RUBY VERSION ruby 3.2.1p31 diff --git a/app/api/cloudflare.rb b/app/api/cloudflare.rb new file mode 100644 index 0000000..5ce6931 --- /dev/null +++ b/app/api/cloudflare.rb @@ -0,0 +1,42 @@ +module Cloudflare + BASE_URL = "https://api.cloudflare.com/client/v4".freeze + + # https://developers.cloudflare.com/api/operations/zone-purge#purge-cached-content-by-tag-host-or-prefix + # + # Rate-limiting: Cache-Tag, host and prefix purging each have a rate limit + # of 30,000 purge API calls in every 24 hour period. You may purge up to + # 30 tags, hosts, or prefixes in one API call. This rate limit can be + # raised for customers who need to purge at higher volume. + # + # Provide tags as an Array of Strings, eg: ["mnd-assets-id-xxx", ...] or a single String + def self.purge_by_tags(tags, zone_id: ENV.fetch("CLOUDFLARE_ZONE_ID")) + tags = Array.wrap(tags) + + post("zones/#{zone_id}/purge_cache", tags:) + end + + # https://developers.cloudflare.com/api/operations/zone-purge#purge-cached-content-by-url + def self.purge_by_urls(urls, zone_id: ENV.fetch("CLOUDFLARE_ZONE_ID")) + urls = Array.wrap(urls) + + post("zones/#{zone_id}/purge_cache", files: urls) + end + + %w[get post delete patch].each do |verb| + define_singleton_method(verb) do |path, params = {}| + request(verb.upcase, path, params) + end + end + + def self.request(verb, path, params) + HTTPX.send( + verb.downcase, + "#{BASE_URL}/#{path}", + headers: { + "Authorization" => "Bearer #{ENV.fetch('CLOUDFLARE_API_TOKEN')}", + "Accept" => "application/json", + }, + json: params, + ).raise_for_status + end +end diff --git a/app/controllers/posts_controller.rb b/app/controllers/posts_controller.rb index 4a75017..79ecbbb 100644 --- a/app/controllers/posts_controller.rb +++ b/app/controllers/posts_controller.rb @@ -1,5 +1,6 @@ class PostsController < ApplicationController - before_action :set_post, only: %i[ show edit update destroy ] + before_action :enable_caching, only: %i[index show new edit] + skip_before_action :verify_authenticity_token # GET /posts def index @@ -8,6 +9,7 @@ def index # GET /posts/1 def show + @post = Post.find(params[:id]) end # GET /posts/new @@ -17,6 +19,7 @@ def new # GET /posts/1/edit def edit + @post = Post.find(params[:id]) end # POST /posts @@ -24,6 +27,7 @@ def create @post = Post.new(post_params) if @post.save + CachedUrl.expire_by_tags(["posts"]) redirect_to @post, notice: "Post was successfully created." else render :new, status: :unprocessable_entity @@ -32,7 +36,10 @@ def create # PATCH/PUT /posts/1 def update + @post = Post.find(params[:id]) + if @post.update(post_params) + CachedUrl.expire_by_tags(["posts", "post-#{@post.id}"]) redirect_to @post, notice: "Post was successfully updated." else render :edit, status: :unprocessable_entity @@ -41,18 +48,30 @@ def update # DELETE /posts/1 def destroy - @post.destroy! + post = Post.find(params[:id]) + + post.destroy! + + CachedUrl.expire_by_tags(["posts"]) + redirect_to posts_url, notice: "Post was successfully destroyed.", status: :see_other end private - # Use callbacks to share common setup or constraints between actions. - def set_post - @post = Post.find(params[:id]) - end - # Only allow a list of trusted parameters through. - def post_params - params.require(:post).permit(:title, :body) - end + def post_params + params.require(:post).permit(:title, :body) + end + + def enable_caching + # don't cache cookies (note: Cloudflare won't cache responses with cookies) + request.session_options[:skip] = true + + return if params[:nocache] + + tags = action_name == "index" ? ["section:posts", "posts:all"] : ["section:posts", "posts:#{params[:id]}"] + + CachedUrl.upsert({ url: request.url, tags:, expires_at: 1.hour.from_now }, unique_by: :url) + expires_in 1.hour, public: true + end end diff --git a/app/models/cached_url.rb b/app/models/cached_url.rb new file mode 100644 index 0000000..b80fa70 --- /dev/null +++ b/app/models/cached_url.rb @@ -0,0 +1,16 @@ +class CachedUrl < ApplicationRecord + scope :tagged_one_of, -> (tags) { where("tags && ARRAY[?]::varchar[]", tags) } + + def self.expire_by_tags(tags) + transaction do + cached_urls = tagged_one_of(tags) + + now = Time.now + urls_to_purge = cached_urls.map { |cu| cu.url unless cu.expires_at < now }.compact + + Cloudflare.purge_by_urls(urls_to_purge) + + cached_urls.delete_all + end + end +end diff --git a/cloudflare-csrf-worker/index.js b/cloudflare-csrf-worker/index.js new file mode 100644 index 0000000..2e08c1d --- /dev/null +++ b/cloudflare-csrf-worker/index.js @@ -0,0 +1,111 @@ +// In a production setting these would be set as environment variables in the Cloudflare dashboard +const SECRET = "0c55b6b18a5072a6ba83773679c6a114234798c1be4d8591f628023e9475f11300d97ccc14fd4293cfb038b1253937704e9311677610a15875a48899bc70be91" +const ORIGIN_HOST = "rails-example.staging.mynewsdesk.dev" + +addEventListener('fetch', event => { + event.respondWith(handleRequest(event.request)) +}) + +encodeText = text => new TextEncoder().encode(text) // text to ArrayBuffer +decodeText = buffer => new TextDecoder().decode(buffer) // ArrayBuffer to text + +// Function to generate a crypto key from a secret key +async function generateCryptoKey(secretKey) { + return await crypto.subtle.importKey( + 'raw', + encodeText(secretKey), + { name: 'AES-GCM' }, + false, + ['encrypt', 'decrypt'] + ) +} + +// SHA256 the secret to ensure it's the correct length +async function hashSecretKey(secretKey) { + return await crypto.subtle.digest('SHA-256', encodeText(secretKey)) +} + +function generateRandomToken() { + return Array.from(crypto.getRandomValues(new Uint8Array(32))) + .map(int => int.toString(16).padStart(2, '0')) + .join('') +} + +function getCookieValue(cookieString, name) { + const cookies = cookieString.split('; ') + for (const cookie of cookies) { + const [cookieName, cookieValue] = cookie.split('=') + if (cookieName === name) { + return cookieValue + } + } + return null +} + +async function encryptMessage(secretKey, message) { + const iv = crypto.getRandomValues(new Uint8Array(12)); // Initialization vector + const key = await generateCryptoKey(secretKey) + const encrypted = await crypto.subtle.encrypt({ name: 'AES-GCM', iv }, key, encodeText(message)) + + // Combine iv and encrypted data + const encryptedArray = new Uint8Array(encrypted) + const combinedArray = new Uint8Array(iv.length + encryptedArray.length) + combinedArray.set(iv, 0) + combinedArray.set(encryptedArray, iv.length) + + return btoa(String.fromCharCode(...combinedArray)) +} + +async function decryptMessage(secretKey, encryptedMessage) { + const combinedArray = new Uint8Array(atob(encryptedMessage).split('').map(char => char.charCodeAt(0))) + const iv = combinedArray.slice(0, 12) + const encryptedArray = combinedArray.slice(12) + + const key = await generateCryptoKey(secretKey) + const decrypted = await crypto.subtle.decrypt({ name: 'AES-GCM', iv }, key, encryptedArray) + + return decodeText(decrypted) +} + +async function handleRequest(request, env, ctx) { + const requestUrl = new URL(request.url) + requestUrl.hostname = ORIGIN_HOST + + const secretKey = hashSecretKey(SECRET) + + if (request.method == 'POST' || request.method == 'PUT' || request.method == 'DELETE' || request.method == 'PATCH') { + // Read form data from a clone to avoid corrupting the original request before we forward it. + const clonedRequest = request.clone() + const form = await clonedRequest.formData() + const csrfToken = form.get('authenticity_token') + if (!csrfToken) return new Response(`CSRF token not found!\n${form}`, { status: 403 }) + + const cookieToken = getCookieValue(request.headers.get('Cookie'), 'csrf_token') + const decryptedToken = await decryptMessage(secretKey, cookieToken) + + // console.log('csrfToken:', csrfToken) + // console.log('cookieToken:', cookieToken) + // console.log('decryptedToken:', decryptedToken) + + if (csrfToken != decryptedToken) return new Response('CSRF validation failed!', { status: 403 }) + } + + const response = await fetch(requestUrl, request) + let html = await response.text() + + // If the response doesn't contain a CSRF token, we don't need to do anything + if(!html.includes(' e + abort e.to_s.strip +end + +RSpec.configure do |config| + # Remove this line if you're not using ActiveRecord or ActiveRecord fixtures + config.fixture_path = "#{Rails.root}/spec/fixtures" + + # If you're not using ActiveRecord, or you'd prefer not to run each of your + # examples within a transaction, remove the following line or assign false + # instead of true. + config.use_transactional_fixtures = true + + # You can uncomment this line to turn off ActiveRecord support entirely. + # config.use_active_record = false + + # RSpec Rails can automatically mix in different behaviours to your tests + # based on their file location, for example enabling you to call `get` and + # `post` in specs under `spec/controllers`. + # + # You can disable this behaviour by removing the line below, and instead + # explicitly tag your specs with their type, e.g.: + # + # RSpec.describe UsersController, type: :controller do + # # ... + # end + # + # The different available types are documented in the features, such as in + # https://relishapp.com/rspec/rspec-rails/docs + config.infer_spec_type_from_file_location! + + # Filter lines from Rails gems in backtraces. + config.filter_rails_from_backtrace! + # arbitrary gems may also be filtered via: + # config.filter_gems_from_backtrace("gem name") +end diff --git a/spec/spec_helper.rb b/spec/spec_helper.rb new file mode 100644 index 0000000..46d6a66 --- /dev/null +++ b/spec/spec_helper.rb @@ -0,0 +1,93 @@ +# This file was generated by the `rails generate rspec:install` command. Conventionally, all +# specs live under a `spec` directory, which RSpec adds to the `$LOAD_PATH`. +# The generated `.rspec` file contains `--require spec_helper` which will cause +# this file to always be loaded, without a need to explicitly require it in any +# files. +# +# Given that it is always loaded, you are encouraged to keep this file as +# light-weight as possible. Requiring heavyweight dependencies from this file +# will add to the boot time of your test suite on EVERY test run, even for an +# individual file that may not need all of that loaded. Instead, consider making +# a separate helper file that requires the additional dependencies and performs +# the additional setup, and require it from the spec files that actually need +# it. +# +# See https://rubydoc.info/gems/rspec-core/RSpec/Core/Configuration +RSpec.configure do |config| + # rspec-expectations config goes here. You can use an alternate + # assertion/expectation library such as wrong or the stdlib/minitest + # assertions if you prefer. + config.expect_with :rspec do |expectations| + # This option will default to `true` in RSpec 4. It makes the `description` + # and `failure_message` of custom matchers include text for helper methods + # defined using `chain`, e.g.: + # be_bigger_than(2).and_smaller_than(4).description + # # => "be bigger than 2 and smaller than 4" + # ...rather than: + # # => "be bigger than 2" + expectations.include_chain_clauses_in_custom_matcher_descriptions = true + end + + # rspec-mocks config goes here. You can use an alternate test double + # library (such as bogus or mocha) by changing the `mock_with` option here. + config.mock_with :rspec do |mocks| + # Prevents you from mocking or stubbing a method that does not exist on + # a real object. This is generally recommended, and will default to + # `true` in RSpec 4. + mocks.verify_partial_doubles = true + end + + # This option will default to `:apply_to_host_groups` in RSpec 4 (and will + # have no way to turn it off -- the option exists only for backwards + # compatibility in RSpec 3). It causes shared context metadata to be + # inherited by the metadata hash of host groups and examples, rather than + # triggering implicit auto-inclusion in groups with matching metadata. + config.shared_context_metadata_behavior = :apply_to_host_groups + + # The settings below are suggested to provide a good initial experience + # with RSpec, but feel free to customize to your heart's content. + + # This allows you to limit a spec run to individual examples or groups + # you care about by tagging them with `:focus` metadata. When nothing + # is tagged with `:focus`, all examples get run. RSpec also provides + # aliases for `it`, `describe`, and `context` that include `:focus` + # metadata: `fit`, `fdescribe` and `fcontext`, respectively. + config.filter_run_when_matching :focus + + # Allows RSpec to persist some state between runs in order to support + # the `--only-failures` and `--next-failure` CLI options. We recommend + # you configure your source control system to ignore this file. + config.example_status_persistence_file_path = "spec/examples.txt" + + # Limits the available syntax to the non-monkey patched syntax that is + # recommended. For more details, see: + # https://relishapp.com/rspec/rspec-core/docs/configuration/zero-monkey-patching-mode + config.disable_monkey_patching! + + # Many RSpec users commonly either run the entire suite or an individual + # file, and it's useful to allow more verbose output when running an + # individual spec file. + if config.files_to_run.one? + # Use the documentation formatter for detailed output, + # unless a formatter has already been configured + # (e.g. via a command-line flag). + config.default_formatter = "doc" + end + + # Print the 10 slowest examples and example groups at the + # end of the spec run, to help surface which specs are running + # particularly slow. + config.profile_examples = 5 + + # Run specs in random order to surface order dependencies. If you find an + # order dependency and want to debug it, you can fix the order by providing + # the seed, which is printed after each run. + # --seed 1234 + config.order = :random + + # Seed global randomization in this process using the `--seed` CLI option. + # Setting this allows you to use `--seed` to deterministically reproduce + # test failures related to randomization by passing the same `--seed` value + # as the one that triggered the failure. + Kernel.srand config.seed +end