diff --git a/Gemfile b/Gemfile index bb5039a67..052943732 100644 --- a/Gemfile +++ b/Gemfile @@ -24,7 +24,7 @@ gem "raygun4ruby" gem "redis" gem "sassc-rails" gem "sidekiq" -gem "sidekiq-batch", path: "vendor/gems/sidekiq-batch" +gem "sidekiq-batch", github: "breamware/sidekiq-batch" # required for sidekiq v7, as changes have not yet been released gem "sqlite3" gem "turbolinks" gem "webpacker", "~> 5.0" diff --git a/Gemfile.lock b/Gemfile.lock index a56dac7bc..d56f75e0f 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,8 +1,9 @@ -PATH - remote: vendor/gems/sidekiq-batch +GIT + remote: https://github.com/breamware/sidekiq-batch.git + revision: f2ef0289ea23e652797741de96506957535099ff specs: sidekiq-batch (0.1.9) - sidekiq (>= 3, < 8) + sidekiq (>= 7, < 8) GEM remote: https://rubygems.org/ diff --git a/vendor/gems/sidekiq-batch/.rspec b/vendor/gems/sidekiq-batch/.rspec deleted file mode 100644 index 8c18f1abd..000000000 --- a/vendor/gems/sidekiq-batch/.rspec +++ /dev/null @@ -1,2 +0,0 @@ ---format documentation ---color diff --git a/vendor/gems/sidekiq-batch/.travis.yml b/vendor/gems/sidekiq-batch/.travis.yml deleted file mode 100644 index bbeaf2cb4..000000000 --- a/vendor/gems/sidekiq-batch/.travis.yml +++ /dev/null @@ -1,7 +0,0 @@ -sudo: false -language: ruby -rvm: - - 2.3.1 -before_install: gem install bundler -v 1.12.5 -after_success: - - bundle exec codeclimate-test-reporter diff --git a/vendor/gems/sidekiq-batch/Gemfile b/vendor/gems/sidekiq-batch/Gemfile deleted file mode 100644 index f95a3bb37..000000000 --- a/vendor/gems/sidekiq-batch/Gemfile +++ /dev/null @@ -1,8 +0,0 @@ -source 'https://rubygems.org' - -gemspec - -group :test do - gem "simplecov" - gem "codeclimate-test-reporter", "~> 1.0.0" -end diff --git a/vendor/gems/sidekiq-batch/Guardfile b/vendor/gems/sidekiq-batch/Guardfile deleted file mode 100644 index ac65cab1e..000000000 --- a/vendor/gems/sidekiq-batch/Guardfile +++ /dev/null @@ -1,4 +0,0 @@ -guard 'rspec', cmd: 'rspec --color' do - watch(%r{^lib/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" } - watch(%r|^spec/(.*)_spec\.rb|) -end diff --git a/vendor/gems/sidekiq-batch/LICENSE.txt b/vendor/gems/sidekiq-batch/LICENSE.txt deleted file mode 100644 index 622384f58..000000000 --- a/vendor/gems/sidekiq-batch/LICENSE.txt +++ /dev/null @@ -1,21 +0,0 @@ -The MIT License (MIT) - -Copyright (c) 2016 Breamware - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in -all copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -THE SOFTWARE. diff --git a/vendor/gems/sidekiq-batch/README.md b/vendor/gems/sidekiq-batch/README.md deleted file mode 100644 index 31649bac8..000000000 --- a/vendor/gems/sidekiq-batch/README.md +++ /dev/null @@ -1,78 +0,0 @@ -[gem]: https://rubygems.org/gems/sidekiq-batch -[travis]: https://travis-ci.org/breamware/sidekiq-batch -[codeclimate]: https://codeclimate.com/github/breamware/sidekiq-batch - -# Sidekiq::Batch - -[![Join the chat at https://gitter.im/breamware/sidekiq-batch](https://badges.gitter.im/breamware/sidekiq-batch.svg)](https://gitter.im/breamware/sidekiq-batch?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) - -[![Gem Version](https://badge.fury.io/rb/sidekiq-batch.svg)][gem] -[![Build Status](https://travis-ci.org/breamware/sidekiq-batch.svg?branch=master)][travis] -[![Code Climate](https://codeclimate.com/github/breamware/sidekiq-batch/badges/gpa.svg)][codeclimate] -[![Code Climate](https://codeclimate.com/github/breamware/sidekiq-batch/badges/coverage.svg)][codeclimate] -[![Code Climate](https://codeclimate.com/github/breamware/sidekiq-batch/badges/issue_count.svg)][codeclimate] - -Simple Sidekiq Batch Job implementation. - -## Installation - -Add this line to your application's Gemfile: - -```ruby -gem 'sidekiq-batch' -``` - -And then execute: - - $ bundle - -Or install it yourself as: - - $ gem install sidekiq-batch - -## Usage - -Sidekiq Batch is MOSTLY a drop-in replacement for the API from Sidekiq PRO. See https://github.com/mperham/sidekiq/wiki/Batches for usage. - -## Caveats/Gotchas - -Consider the following workflow: - - * Batch Z created - * Worker A queued in batch Z - * Worker A starts Worker B in batch Z - * Worker B completes *before* worker A does - * Worker A completes - -In the standard configuration, the `on(:success)` and `on(:complete)` callbacks will be triggered when Worker B completes. -This configuration is the default, simply for legacy reasons. This gem adds the following option to the sidekiq.yml options: - -```yaml -:batch_push_interval: 0 -``` - -When this value is *absent* (aka legacy), Worker A will only push the increment of batch jobs (aka Worker B) *when it completes* - -When this value is set to `0`, Worker A will increment the count as soon as `WorkerB.perform_async` is called - -When this value is a positive number, Worker A will wait a maximum of value-seconds before pushing the increment to redis, or until it's done, whichever comes first. - -This comes into play if Worker A is queueing thousands of WorkerB jobs, or has some other reason for WorkerB to complete beforehand. - -If you are queueing many WorkerB jobs, it is recommended to set this value to something like `3` to avoid thousands of calls to redis, and call WorkerB like so: -```ruby -WorkerB.perform_in(4.seconds, some, args) -``` -this will ensure that the batch callback does not get triggered until WorkerA *and* the last WorkerB job complete. - -If WorkerA is just slow for whatever reason, setting to `0` will update the batch status immediately so that the callbacks don't fire. - - -## Contributing - -Bug reports and pull requests are welcome on GitHub at https://github.com/breamware/sidekiq-batch. - - -## License - -The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT). diff --git a/vendor/gems/sidekiq-batch/Rakefile b/vendor/gems/sidekiq-batch/Rakefile deleted file mode 100644 index b7e9ed549..000000000 --- a/vendor/gems/sidekiq-batch/Rakefile +++ /dev/null @@ -1,6 +0,0 @@ -require "bundler/gem_tasks" -require "rspec/core/rake_task" - -RSpec::Core::RakeTask.new(:spec) - -task :default => :spec diff --git a/vendor/gems/sidekiq-batch/lib/sidekiq/batch.rb b/vendor/gems/sidekiq-batch/lib/sidekiq/batch.rb deleted file mode 100644 index b392d1e54..000000000 --- a/vendor/gems/sidekiq-batch/lib/sidekiq/batch.rb +++ /dev/null @@ -1,331 +0,0 @@ -require 'securerandom' -require 'sidekiq' - -require 'sidekiq/batch/callback' -require 'sidekiq/batch/middleware' -require 'sidekiq/batch/status' -require 'sidekiq/batch/version' - -module Sidekiq - class Batch - class NoBlockGivenError < StandardError; end - - BID_EXPIRE_TTL = 2_592_000 - - attr_reader :bid, :description, :callback_queue, :created_at - - def initialize(existing_bid = nil) - @bid = existing_bid || SecureRandom.urlsafe_base64(10) - @existing = !(!existing_bid || existing_bid.empty?) # Basically existing_bid.present? - @initialized = false - @created_at = Time.now.utc.to_f - @bidkey = "BID-" + @bid.to_s - @queued_jids = [] - @pending_jids = [] - - @incremental_push = !Sidekiq.default_configuration[:batch_push_interval].nil? - @batch_push_interval = Sidekiq.default_configuration[:batch_push_interval] - end - - def description=(description) - @description = description - persist_bid_attr('description', description) - end - - def callback_queue=(callback_queue) - @callback_queue = callback_queue - persist_bid_attr('callback_queue', callback_queue) - end - - def callback_batch=(callback_batch) - @callback_batch = callback_batch - persist_bid_attr('callback_batch', callback_batch) - end - - def on(event, callback, options = {}) - return unless %w(success complete).include?(event.to_s) - callback_key = "#{@bidkey}-callbacks-#{event}" - Sidekiq.redis do |r| - r.multi do |pipeline| - pipeline.sadd(callback_key, [JSON.unparse({ - callback: callback, - opts: options - })]) - pipeline.expire(callback_key, BID_EXPIRE_TTL) - end - end - end - - def jobs - raise NoBlockGivenError unless block_given? - - bid_data, Thread.current[:bid_data] = Thread.current[:bid_data], [] - - begin - if !@existing && !@initialized - parent_bid = Thread.current[:batch].bid if Thread.current[:batch] - - Sidekiq.redis do |r| - r.multi do |pipeline| - pipeline.hset(@bidkey, "created_at", @created_at) - pipeline.expire(@bidkey, BID_EXPIRE_TTL) - if parent_bid - pipeline.hset(@bidkey, "parent_bid", parent_bid.to_s) - pipeline.hincrby("BID-#{parent_bid}", "children", 1) - end - end - end - - @initialized = true - end - - @queued_jids = [] - @pending_jids = [] - - begin - parent = Thread.current[:batch] - Thread.current[:batch] = self - Thread.current[:parent_bid] = parent_bid - yield - ensure - Thread.current[:batch] = parent - Thread.current[:parent_bid] = nil - end - - return [] if @queued_jids.size == 0 - conditional_redis_increment!(true) - - Sidekiq.redis do |r| - r.multi do |pipeline| - if parent_bid - pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL) - end - - pipeline.expire(@bidkey, BID_EXPIRE_TTL) - - pipeline.sadd(@bidkey + "-jids", @queued_jids) - pipeline.expire(@bidkey + "-jids", BID_EXPIRE_TTL) - end - end - - @queued_jids - ensure - Thread.current[:bid_data] = bid_data - end - end - - def increment_job_queue(jid) - @queued_jids << jid - @pending_jids << jid - conditional_redis_increment! - end - - def conditional_redis_increment!(force=false) - if should_increment? || force - parent_bid = Thread.current[:parent_bid] - Sidekiq.redis do |r| - r.multi do |pipeline| - if parent_bid - pipeline.hincrby("BID-#{parent_bid}", "total", @pending_jids.length) - pipeline.expire("BID-#{parent_bid}", BID_EXPIRE_TTL) - end - - pipeline.hincrby(@bidkey, "pending", @pending_jids.length) - pipeline.hincrby(@bidkey, "total", @pending_jids.length) - pipeline.expire(@bidkey, BID_EXPIRE_TTL) - end - end - @pending_jids = [] - end - end - - def should_increment? - return false unless @incremental_push - return true if @batch_push_interval == 0 || @queued_jids.length == 1 - now = Time.now.to_f - @last_increment ||= now - if @last_increment + @batch_push_interval > now - @last_increment = now - return true - end - end - - def invalidate_all - Sidekiq.redis do |r| - r.setex("invalidated-bid-#{bid}", BID_EXPIRE_TTL, 1) - end - end - - def parent_bid - Sidekiq.redis do |r| - r.hget(@bidkey, "parent_bid") - end - end - - def parent - if parent_bid - Sidekiq::Batch.new(parent_bid) - end - end - - def valid?(batch = self) - valid = !Sidekiq.redis { |r| r.exists("invalidated-bid-#{batch.bid}") } - batch.parent ? valid && valid?(batch.parent) : valid - end - - private - - def persist_bid_attr(attribute, value) - Sidekiq.redis do |r| - r.multi do |pipeline| - pipeline.hset(@bidkey, attribute, value) - pipeline.expire(@bidkey, BID_EXPIRE_TTL) - end - end - end - - class << self - def process_failed_job(bid, jid) - _, pending, failed, children, complete, parent_bid = Sidekiq.redis do |r| - r.multi do |pipeline| - pipeline.sadd("BID-#{bid}-failed", [jid]) - - pipeline.hincrby("BID-#{bid}", "pending", 0) - pipeline.scard("BID-#{bid}-failed") - pipeline.hincrby("BID-#{bid}", "children", 0) - pipeline.scard("BID-#{bid}-complete") - pipeline.hget("BID-#{bid}", "parent_bid") - - pipeline.expire("BID-#{bid}-failed", BID_EXPIRE_TTL) - end - end - - # if the batch failed, and has a parent, update the parent to show one pending and failed job - if parent_bid - Sidekiq.redis do |r| - r.multi do |pipeline| - pipeline.hincrby("BID-#{parent_bid}", "pending", 1) - pipeline.sadd("BID-#{parent_bid}-failed", [jid]) - pipeline.expire("BID-#{parent_bid}-failed", BID_EXPIRE_TTL) - end - end - end - - if pending.to_i == failed.to_i && children == complete - enqueue_callbacks(:complete, bid) - end - end - - def process_successful_job(bid, jid) - failed, pending, children, complete, success, total, parent_bid = Sidekiq.redis do |r| - r.multi do |pipeline| - pipeline.scard("BID-#{bid}-failed") - pipeline.hincrby("BID-#{bid}", "pending", -1) - pipeline.hincrby("BID-#{bid}", "children", 0) - pipeline.scard("BID-#{bid}-complete") - pipeline.scard("BID-#{bid}-success") - pipeline.hget("BID-#{bid}", "total") - pipeline.hget("BID-#{bid}", "parent_bid") - - pipeline.srem("BID-#{bid}-failed", [jid]) - pipeline.srem("BID-#{bid}-jids", [jid]) - pipeline.expire("BID-#{bid}", BID_EXPIRE_TTL) - end - end - - all_success = pending.to_i.zero? && children == success - # if complete or successfull call complete callback (the complete callback may then call successful) - if (pending.to_i == failed.to_i && children == complete) || all_success - enqueue_callbacks(:complete, bid) - enqueue_callbacks(:success, bid) if all_success - end - end - - def enqueue_callbacks(event, bid) - event_name = event.to_s - batch_key = "BID-#{bid}" - callback_key = "#{batch_key}-callbacks-#{event_name}" - already_processed, _, callbacks, queue, parent_bid, callback_batch = Sidekiq.redis do |r| - r.multi do |pipeline| - pipeline.hget(batch_key, event_name) - pipeline.hset(batch_key, event_name, 'true') - pipeline.smembers(callback_key) - pipeline.hget(batch_key, "callback_queue") - pipeline.hget(batch_key, "parent_bid") - pipeline.hget(batch_key, "callback_batch") - end - end - - return if already_processed == 'true' - - queue ||= "default" - parent_bid = !parent_bid || parent_bid.empty? ? nil : parent_bid # Basically parent_bid.blank? - callback_args = callbacks.reduce([]) do |memo, jcb| - cb = Sidekiq.load_json(jcb) - memo << [cb['callback'], event_name, cb['opts'], bid, parent_bid] - end - - opts = {"bid" => bid, "event" => event_name} - - # Run callback batch finalize synchronously - if callback_batch - # Extract opts from cb_args or use current - # Pass in stored event as callback finalize is processed on complete event - cb_opts = callback_args.first&.at(2) || opts - - Sidekiq.logger.debug {"Run callback batch bid: #{bid} event: #{event_name} args: #{callback_args.inspect}"} - # Finalize now - finalizer = Sidekiq::Batch::Callback::Finalize.new - status = Status.new bid - finalizer.dispatch(status, cb_opts) - - return - end - - Sidekiq.logger.debug {"Enqueue callback bid: #{bid} event: #{event_name} args: #{callback_args.inspect}"} - - if callback_args.empty? - # Finalize now - finalizer = Sidekiq::Batch::Callback::Finalize.new - status = Status.new bid - finalizer.dispatch(status, opts) - else - # Otherwise finalize in sub batch complete callback - cb_batch = self.new - cb_batch.callback_batch = 'true' - Sidekiq.logger.debug {"Adding callback batch: #{cb_batch.bid} for batch: #{bid}"} - cb_batch.on(:complete, "Sidekiq::Batch::Callback::Finalize#dispatch", opts) - cb_batch.jobs do - push_callbacks callback_args, queue - end - end - end - - def cleanup_redis(bid) - Sidekiq.logger.debug {"Cleaning redis of batch #{bid}"} - Sidekiq.redis do |r| - r.del( - "BID-#{bid}", - "BID-#{bid}-callbacks-complete", - "BID-#{bid}-callbacks-success", - "BID-#{bid}-failed", - - "BID-#{bid}-success", - "BID-#{bid}-complete", - "BID-#{bid}-jids", - ) - end - end - - private - - def push_callbacks args, queue - Sidekiq::Client.push_bulk( - 'class' => Sidekiq::Batch::Callback::Worker, - 'args' => args, - 'queue' => queue - ) unless args.empty? - end - end - end -end diff --git a/vendor/gems/sidekiq-batch/lib/sidekiq/batch/callback.rb b/vendor/gems/sidekiq-batch/lib/sidekiq/batch/callback.rb deleted file mode 100644 index 7186665bd..000000000 --- a/vendor/gems/sidekiq-batch/lib/sidekiq/batch/callback.rb +++ /dev/null @@ -1,104 +0,0 @@ -module Sidekiq - class Batch - module Callback - class Worker - include Sidekiq::Worker - - def perform(clazz, event, opts, bid, parent_bid) - return unless %w(success complete).include?(event) - clazz, method = clazz.split("#") if (clazz && clazz.class == String && clazz.include?("#")) - method = "on_#{event}" if method.nil? - status = Sidekiq::Batch::Status.new(bid) - - if clazz && object = Object.const_get(clazz) - instance = object.new - instance.send(method, status, opts) if instance.respond_to?(method) - end - end - end - - class Finalize - def dispatch status, opts - bid = opts["bid"] - callback_bid = status.bid - event = opts["event"].to_sym - callback_batch = bid != callback_bid - - Sidekiq.logger.debug {"Finalize #{event} batch id: #{opts["bid"]}, callback batch id: #{callback_bid} callback_batch #{callback_batch}"} - - batch_status = Status.new bid - send(event, bid, batch_status, batch_status.parent_bid) - - - # Different events are run in different callback batches - Sidekiq::Batch.cleanup_redis callback_bid if callback_batch - Sidekiq::Batch.cleanup_redis bid if event == :success - end - - def success(bid, status, parent_bid) - return unless parent_bid - - _, _, success, _, complete, pending, children, failure = Sidekiq.redis do |r| - r.multi do |pipeline| - pipeline.sadd("BID-#{parent_bid}-success", [bid]) - pipeline.expire("BID-#{parent_bid}-success", Sidekiq::Batch::BID_EXPIRE_TTL) - pipeline.scard("BID-#{parent_bid}-success") - pipeline.sadd("BID-#{parent_bid}-complete", [bid]) - pipeline.scard("BID-#{parent_bid}-complete") - pipeline.hincrby("BID-#{parent_bid}", "pending", 0) - pipeline.hincrby("BID-#{parent_bid}", "children", 0) - pipeline.scard("BID-#{parent_bid}-failed") - end - end - # if job finished successfully and parent batch completed call parent complete callback - # Success callback is called after complete callback - if complete == children && pending == failure - Sidekiq.logger.debug {"Finalize parent complete bid: #{parent_bid}"} - Batch.enqueue_callbacks(:complete, parent_bid) - end - - end - - def complete(bid, status, parent_bid) - pending, children, success = Sidekiq.redis do |r| - r.multi do |pipeline| - pipeline.hincrby("BID-#{bid}", "pending", 0) - pipeline.hincrby("BID-#{bid}", "children", 0) - pipeline.scard("BID-#{bid}-success") - end - end - - # if we batch was successful run success callback - if pending.to_i.zero? && children == success - Batch.enqueue_callbacks(:success, bid) - - elsif parent_bid - # if batch was not successfull check and see if its parent is complete - # if the parent is complete we trigger the complete callback - # We don't want to run this if the batch was successfull because the success - # callback may add more jobs to the parent batch - - Sidekiq.logger.debug {"Finalize parent complete bid: #{parent_bid}"} - _, complete, pending, children, failure = Sidekiq.redis do |r| - r.multi do |pipeline| - pipeline.sadd("BID-#{parent_bid}-complete", [bid]) - pipeline.scard("BID-#{parent_bid}-complete") - pipeline.hincrby("BID-#{parent_bid}", "pending", 0) - pipeline.hincrby("BID-#{parent_bid}", "children", 0) - pipeline.scard("BID-#{parent_bid}-failed") - end - end - if complete == children && pending == failure - Batch.enqueue_callbacks(:complete, parent_bid) - end - end - end - - def cleanup_redis bid, callback_bid=nil - Sidekiq::Batch.cleanup_redis bid - Sidekiq::Batch.cleanup_redis callback_bid if callback_bid - end - end - end - end -end diff --git a/vendor/gems/sidekiq-batch/lib/sidekiq/batch/extension/worker.rb b/vendor/gems/sidekiq-batch/lib/sidekiq/batch/extension/worker.rb deleted file mode 100644 index c5da7324b..000000000 --- a/vendor/gems/sidekiq-batch/lib/sidekiq/batch/extension/worker.rb +++ /dev/null @@ -1,15 +0,0 @@ -module Sidekiq::Batch::Extension - module Worker - def bid - Thread.current[:batch].bid - end - - def batch - Thread.current[:batch] - end - - def valid_within_batch? - batch.valid? - end - end -end diff --git a/vendor/gems/sidekiq-batch/lib/sidekiq/batch/middleware.rb b/vendor/gems/sidekiq-batch/lib/sidekiq/batch/middleware.rb deleted file mode 100644 index 04f7bd0c0..000000000 --- a/vendor/gems/sidekiq-batch/lib/sidekiq/batch/middleware.rb +++ /dev/null @@ -1,55 +0,0 @@ -require_relative 'extension/worker' - -module Sidekiq - class Batch - module Middleware - class ClientMiddleware - def call(_worker, msg, _queue, _redis_pool = nil) - if (batch = Thread.current[:batch]) - batch.increment_job_queue(msg['jid']) if (msg[:bid] = batch.bid) - end - yield - end - end - - class ServerMiddleware - def call(_worker, msg, _queue) - if (bid = msg['bid']) - begin - Thread.current[:batch] = Sidekiq::Batch.new(bid) - yield - Thread.current[:batch] = nil - Batch.process_successful_job(bid, msg['jid']) - rescue - Batch.process_failed_job(bid, msg['jid']) - raise - ensure - Thread.current[:batch] = nil - end - else - yield - end - end - end - - def self.configure - Sidekiq.configure_client do |config| - config.client_middleware do |chain| - chain.add Sidekiq::Batch::Middleware::ClientMiddleware - end - end - Sidekiq.configure_server do |config| - config.client_middleware do |chain| - chain.add Sidekiq::Batch::Middleware::ClientMiddleware - end - config.server_middleware do |chain| - chain.add Sidekiq::Batch::Middleware::ServerMiddleware - end - end - Sidekiq::Worker.send(:include, Sidekiq::Batch::Extension::Worker) - end - end - end -end - -Sidekiq::Batch::Middleware.configure diff --git a/vendor/gems/sidekiq-batch/lib/sidekiq/batch/status.rb b/vendor/gems/sidekiq-batch/lib/sidekiq/batch/status.rb deleted file mode 100644 index 93a2bbc00..000000000 --- a/vendor/gems/sidekiq-batch/lib/sidekiq/batch/status.rb +++ /dev/null @@ -1,61 +0,0 @@ -module Sidekiq - class Batch - class Status - attr_reader :bid - - def initialize(bid) - @bid = bid - end - - def join - raise "Not supported" - end - - def pending - Sidekiq.redis { |r| r.hget("BID-#{bid}", 'pending') }.to_i - end - - def failures - Sidekiq.redis { |r| r.scard("BID-#{bid}-failed") }.to_i - end - - def created_at - Sidekiq.redis { |r| r.hget("BID-#{bid}", 'created_at') } - end - - def total - Sidekiq.redis { |r| r.hget("BID-#{bid}", 'total') }.to_i - end - - def parent_bid - Sidekiq.redis { |r| r.hget("BID-#{bid}", "parent_bid") } - end - - def failure_info - Sidekiq.redis { |r| r.smembers("BID-#{bid}-failed") } || [] - end - - def complete? - 'true' == Sidekiq.redis { |r| r.hget("BID-#{bid}", 'complete') } - end - - def child_count - Sidekiq.redis { |r| r.hget("BID-#{bid}", 'children') }.to_i - end - - def data - { - bid: bid, - total: total, - failures: failures, - pending: pending, - created_at: created_at, - complete: complete?, - failure_info: failure_info, - parent_bid: parent_bid, - child_count: child_count - } - end - end - end -end diff --git a/vendor/gems/sidekiq-batch/lib/sidekiq/batch/version.rb b/vendor/gems/sidekiq-batch/lib/sidekiq/batch/version.rb deleted file mode 100644 index ac7af42d8..000000000 --- a/vendor/gems/sidekiq-batch/lib/sidekiq/batch/version.rb +++ /dev/null @@ -1,5 +0,0 @@ -module Sidekiq - class Batch - VERSION = '0.1.9'.freeze - end -end diff --git a/vendor/gems/sidekiq-batch/sidekiq-batch.gemspec b/vendor/gems/sidekiq-batch/sidekiq-batch.gemspec deleted file mode 100644 index decf80ec3..000000000 --- a/vendor/gems/sidekiq-batch/sidekiq-batch.gemspec +++ /dev/null @@ -1,28 +0,0 @@ -# coding: utf-8 -lib = File.expand_path('../lib', __FILE__) -$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib) -require 'sidekiq/batch/version' - -Gem::Specification.new do |spec| - spec.name = "sidekiq-batch" - spec.version = Sidekiq::Batch::VERSION - spec.authors = ["Marcin Naglik"] - spec.email = ["marcin.naglik@gmail.com"] - - spec.summary = "Sidekiq Batch Jobs" - spec.description = "Sidekiq Batch Jobs Implementation" - spec.homepage = "http://github.com/breamware/sidekiq-batch" - spec.license = "MIT" - - spec.files = `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) } - spec.bindir = "exe" - spec.executables = spec.files.grep(%r{^exe/}) { |f| File.basename(f) } - spec.require_paths = ["lib"] - - spec.add_dependency "sidekiq", ">= 3", "<8" - - spec.add_development_dependency "bundler", "~> 2.1" - spec.add_development_dependency "rake", "~> 13.0" - spec.add_development_dependency "rspec", "~> 3.0" - spec.add_development_dependency "fakeredis", "~> 0.8.0" -end diff --git a/vendor/gems/sidekiq-batch/spec/integration/integration.rb b/vendor/gems/sidekiq-batch/spec/integration/integration.rb deleted file mode 100644 index 6b6e31553..000000000 --- a/vendor/gems/sidekiq-batch/spec/integration/integration.rb +++ /dev/null @@ -1,57 +0,0 @@ -require 'integration_helper' - -# Simple test of adding jobs to the current batch -# Batches: -# - Overall (TestWoker) + Another worker - -class AnotherWorker - include Sidekiq::Worker - - def perform - Sidekiq.logger.info "Another Worker" - end -end - -class TestWorker - include Sidekiq::Worker - - def perform - Sidekiq.logger.info "Test Worker" - if bid - batch.jobs do - AnotherWorker.perform_async - end - end - end -end - -class MyCallback - def on_success(status, options) - Sidekiq.logger.info "Success #{options} #{status.data}" - end - alias_method :multi, :on_success - - def on_complete(status, options) - Sidekiq.logger.info "Complete #{options} #{status.data}" - end -end - -batch = Sidekiq::Batch.new -batch.description = 'Test batch' -batch.callback_queue = :default -batch.on(:success, 'MyCallback#on_success', to: 'success@gmail.com') -batch.on(:success, 'MyCallback#multi', to: 'success@gmail.com') -batch.on(:complete, MyCallback, to: 'complete@gmail.com') - -batch.jobs do - 10.times do - TestWorker.perform_async - end -end -puts Sidekiq::Batch::Status.new(batch.bid).data - -dump_redis_keys - -Sidekiq::Worker.drain_all - -dump_redis_keys diff --git a/vendor/gems/sidekiq-batch/spec/integration/nested.rb b/vendor/gems/sidekiq-batch/spec/integration/nested.rb deleted file mode 100644 index 8cc84837e..000000000 --- a/vendor/gems/sidekiq-batch/spec/integration/nested.rb +++ /dev/null @@ -1,88 +0,0 @@ -require 'integration_helper' - -# Tests deep nesting of batches -# Batches: -# - Overall (Worker 1) -# - Worker 2 -# - Worker 3 -# - Worker 4 - -class Worker1 - include Sidekiq::Worker - - def perform - Sidekiq.logger.info "Work1" - batch = Sidekiq::Batch.new - batch.on(:success, Worker2) - batch.jobs do - Worker2.perform_async - end - end -end - -class Worker2 - include Sidekiq::Worker - - def perform - Sidekiq.logger.info "Work2" - batch = Sidekiq::Batch.new - batch.on(:success, Worker3) - batch.jobs do - Worker3.perform_async - end - end - - def on_success status, opts - Sidekiq.logger.info "Worker 2 Success" - end -end - -class Worker3 - include Sidekiq::Worker - - def perform - Sidekiq.logger.info "Work3" - batch = Sidekiq::Batch.new - batch.on(:success, Worker4) - batch.jobs do - Worker4.perform_async - end - end - - def on_success status, opts - Sidekiq.logger.info "Worker 3 Success" - end -end - -class Worker4 - include Sidekiq::Worker - - def perform - Sidekiq.logger.info "Work4" - end - - def on_success status, opts - Sidekiq.logger.info "Worker 4 Success" - end -end - - -class SomeClass - def on_complete(status, options) - Sidekiq.logger.info "Overall Complete #{options} #{status.data}" - end - def on_success(status, options) - Sidekiq.logger.info "Overall Success #{options} #{status.data}" - end -end -batch = Sidekiq::Batch.new -batch.on(:success, SomeClass, 'uid' => 3) -batch.on(:complete, SomeClass, 'uid' => 3) -batch.jobs do - Worker1.perform_async -end - -puts "Overall bid #{batch.bid}" - -output, keys = process_tests -overall_tests output, keys diff --git a/vendor/gems/sidekiq-batch/spec/integration/simple.rb b/vendor/gems/sidekiq-batch/spec/integration/simple.rb deleted file mode 100644 index e19796dbd..000000000 --- a/vendor/gems/sidekiq-batch/spec/integration/simple.rb +++ /dev/null @@ -1,47 +0,0 @@ -require 'integration_helper' - -# Simple nested batch without callbacks -# Batches: -# - Overall (Worker1) -# - Worker2 - -class Worker1 - include Sidekiq::Worker - - def perform - Sidekiq.logger.info "Work1" - batch = Sidekiq::Batch.new - batch.jobs do - Worker2.perform_async - end - end -end - -class Worker2 - include Sidekiq::Worker - - def perform - Sidekiq.logger.info "Work2" - end -end - -class SomeClass - def on_complete(status, options) - Sidekiq.logger.info "Overall Complete #{options} #{status.data}" - end - def on_success(status, options) - Sidekiq.logger.info "Overall Success #{options} #{status.data}" - end -end - -batch = Sidekiq::Batch.new -batch.on(:success, SomeClass) -batch.on(:complete, SomeClass) -batch.jobs do - Worker1.perform_async -end - -puts "Overall bid #{batch.bid}" - -output, keys = process_tests -overall_tests output, keys diff --git a/vendor/gems/sidekiq-batch/spec/integration/workflow.rb b/vendor/gems/sidekiq-batch/spec/integration/workflow.rb deleted file mode 100644 index 9c72015a5..000000000 --- a/vendor/gems/sidekiq-batch/spec/integration/workflow.rb +++ /dev/null @@ -1,134 +0,0 @@ -require 'integration_helper' - -# Complex workflow with sequential and nested -# Also test sub batches without callbacks -# Batches: -# - Overall -# - Worker1 -# - Worker3 -# - Worker2 + Worker3 -# - Worker1 -# - Worker3 -# - Worker4 -# - Worker5 - -class Callbacks - def worker1 status, opts - Sidekiq.logger.info "Success 1 #{status.data}" - - overall = Sidekiq::Batch.new status.parent_bid - overall.jobs do - batch = Sidekiq::Batch.new - batch.on(:success, "Callbacks#worker2") - batch.jobs do - Worker2.perform_async - end - end - end - - def worker2 status, opts - Sidekiq.logger.info "Success 2 #{status.data}" - overall = Sidekiq::Batch.new status.parent_bid - overall.jobs do - batch = Sidekiq::Batch.new - batch.on(:success, "Callbacks#worker4") - batch.jobs do - Worker4.perform_async - end - end - - end - - def worker4 status, opts - Sidekiq.logger.info "Success 4 #{status.data}" - overall = Sidekiq::Batch.new status.parent_bid - overall.jobs do - batch = Sidekiq::Batch.new - batch.on(:success, "Callbacks#worker5") - batch.jobs do - Worker5.perform_async - end - end - end - - def worker5 status, opts - Sidekiq.logger.info "Success 5 #{status.data}" - end -end - -class Worker1 - include Sidekiq::Worker - - def perform - Sidekiq.logger.info "Work 1" - batch = Sidekiq::Batch.new - batch.jobs do - Worker3.perform_async - end - end -end - -class Worker2 - include Sidekiq::Worker - - def perform - Sidekiq.logger.info "Work 2" - if bid - batch.jobs do - Worker3.perform_async - end - newb = Sidekiq::Batch.new - newb.jobs do - Worker1.perform_async - end - end - end -end - -class Worker3 - include Sidekiq::Worker - def perform - Sidekiq.logger.info "Work 3" - end -end - -class Worker4 - include Sidekiq::Worker - def perform - Sidekiq.logger.info "Work 4" - end -end - -class Worker5 - include Sidekiq::Worker - def perform - Sidekiq.logger.info "Work 5" - end -end - -class MyCallback - def on_success(status, options) - Sidekiq.logger.info "Overall Success #{options} #{status.data}" - end - alias_method :multi, :on_success - - def on_complete(status, options) - Sidekiq.logger.info "Overall Complete #{options} #{status.data}" - end -end - -overall = Sidekiq::Batch.new -overall.on(:success, MyCallback, to: 'success@gmail.com') -overall.on(:complete, MyCallback, to: 'success@gmail.com') -overall.jobs do - batch1 = Sidekiq::Batch.new - batch1.on(:success, "Callbacks#worker1") - batch1.jobs do - Worker1.perform_async - end -end - -puts "Overall bid #{overall.bid}" - -output, keys = process_tests -overall_tests output, keys diff --git a/vendor/gems/sidekiq-batch/spec/integration_helper.rb b/vendor/gems/sidekiq-batch/spec/integration_helper.rb deleted file mode 100644 index a4f7ba4b5..000000000 --- a/vendor/gems/sidekiq-batch/spec/integration_helper.rb +++ /dev/null @@ -1,48 +0,0 @@ -require 'spec_helper' -require 'sidekiq/batch' -require 'sidekiq/testing' - -Sidekiq::Testing.server_middleware do |chain| - chain.add Sidekiq::Batch::Middleware::ServerMiddleware -end - -Sidekiq.redis { |r| r.flushdb } - -def redis_keys - Sidekiq.redis { |r| r.keys('BID-*') } -end - -def dump_redis_keys - puts redis_keys.inspect -end - -def process_tests - out_buf = StringIO.new - Sidekiq.logger = Logger.new out_buf - - # Sidekiq.logger.level = :info - - Sidekiq::Worker.drain_all - - output = out_buf.string - keys = redis_keys - puts out_buf.string - - [output, keys] -end - -def overall_tests output, keys - describe "sidekiq batch" do - it "runs overall complete callback" do - expect(output).to include "Overall Complete" - end - - it "runs overall success callback" do - expect(output).to include "Overall Success" - end - - it "cleans redis keys" do - expect(keys).to eq([]) - end - end -end diff --git a/vendor/gems/sidekiq-batch/spec/sidekiq/batch/callback_spec.rb b/vendor/gems/sidekiq-batch/spec/sidekiq/batch/callback_spec.rb deleted file mode 100644 index 34231e033..000000000 --- a/vendor/gems/sidekiq-batch/spec/sidekiq/batch/callback_spec.rb +++ /dev/null @@ -1,38 +0,0 @@ -require 'spec_helper' - -describe Sidekiq::Batch::Callback::Worker do - describe '#perform' do - it 'does not do anything if it cannot find the callback class' do - subject.perform('SampleCallback', 'complete', {}, 'ABCD', 'EFGH') - end - - it 'does not do anything if event is different from complete or success' do - expect(SampleCallback).not_to receive(:new) - subject.perform('SampleCallback', 'ups', {}, 'ABCD', 'EFGH') - end - - it 'calls on_success if defined' do - callback_instance = double('SampleCallback', on_success: true) - expect(SampleCallback).to receive(:new).and_return(callback_instance) - expect(callback_instance).to receive(:on_success) - .with(instance_of(Sidekiq::Batch::Status), {}) - subject.perform('SampleCallback', 'success', {}, 'ABCD', 'EFGH') - end - - it 'calls on_complete if defined' do - callback_instance = double('SampleCallback') - expect(SampleCallback).to receive(:new).and_return(callback_instance) - expect(callback_instance).to receive(:on_complete) - .with(instance_of(Sidekiq::Batch::Status), {}) - subject.perform('SampleCallback', 'complete', {}, 'ABCD', 'EFGH') - end - - it 'calls specific callback if defined' do - callback_instance = double('SampleCallback') - expect(SampleCallback).to receive(:new).and_return(callback_instance) - expect(callback_instance).to receive(:sample_method) - .with(instance_of(Sidekiq::Batch::Status), {}) - subject.perform('SampleCallback#sample_method', 'complete', {}, 'ABCD', 'EFGH') - end - end -end diff --git a/vendor/gems/sidekiq-batch/spec/sidekiq/batch/flow_spec.rb b/vendor/gems/sidekiq-batch/spec/sidekiq/batch/flow_spec.rb deleted file mode 100644 index 46266ad5e..000000000 --- a/vendor/gems/sidekiq-batch/spec/sidekiq/batch/flow_spec.rb +++ /dev/null @@ -1,89 +0,0 @@ -require 'spec_helper' - -class WorkerA - include Sidekiq::Worker - def perform - end -end - -class WorkerB - include Sidekiq::Worker - def perform - end -end - -class WorkerC - include Sidekiq::Worker - def perform - end -end - -describe 'Batch flow' do - context 'when handling a batch' do - let(:batch) { Sidekiq::Batch.new } - before { batch.on(:complete, SampleCallback, :id => 42) } - before { batch.description = 'describing the batch' } - let(:status) { Sidekiq::Batch::Status.new(batch.bid) } - let(:jids) { batch.jobs do 3.times do TestWorker.perform_async end end } - let(:queue) { Sidekiq::Queue.new } - - it 'correctly initializes' do - expect(jids.size).to eq(3) - - expect(batch.bid).not_to be_nil - expect(batch.description).to eq('describing the batch') - - expect(status.total).to eq(3) - expect(status.pending).to eq(3) - expect(status.failures).to eq(0) - expect(status.complete?).to be false - expect(status.created_at).not_to be_nil - expect(status.bid).to eq(batch.bid) - end - - it 'handles an empty batch' do - batch = Sidekiq::Batch.new - jids = batch.jobs do nil end - expect(jids.size).to eq(0) - end - end - - context 'when handling a nested batch' do - let(:batchA) { Sidekiq::Batch.new } - let(:batchB) { Sidekiq::Batch.new } - let(:batchC) { Sidekiq::Batch.new(batchA.bid) } - let(:batchD) { Sidekiq::Batch.new } - let(:jids) { [] } - let(:parent) { batchA.bid } - let(:children) { [] } - - it 'handles a basic nested batch' do - batchA.jobs do - jids << WorkerA.perform_async - batchB.jobs do - jids << WorkerB.perform_async - end - jids << WorkerA.perform_async - children << batchB.bid - end - - batchC.jobs do - batchD.jobs do - jids << WorkerC.perform_async - end - children << batchD.bid - end - - expect(jids.size).to eq(4) - expect(Sidekiq::Batch::Status.new(parent).child_count).to eq(2) - children.each do |kid| - status = Sidekiq::Batch::Status.new(kid) - expect(status.child_count).to eq(0) - expect(status.pending).to eq(1) - expect(status.parent_bid).to eq(parent) - end - - end - - end -end diff --git a/vendor/gems/sidekiq-batch/spec/sidekiq/batch/middleware_spec.rb b/vendor/gems/sidekiq-batch/spec/sidekiq/batch/middleware_spec.rb deleted file mode 100644 index 776b19825..000000000 --- a/vendor/gems/sidekiq-batch/spec/sidekiq/batch/middleware_spec.rb +++ /dev/null @@ -1,120 +0,0 @@ -require 'spec_helper' - -describe Sidekiq::Batch::Middleware do - describe Sidekiq::Batch::Middleware::ServerMiddleware do - context 'when without batch' do - it 'just yields' do - yielded = false - expect(Sidekiq::Batch).not_to receive(:process_successful_job) - expect(Sidekiq::Batch).not_to receive(:process_failed_job) - subject.call(nil, {}, nil) { yielded = true } - expect(yielded).to be_truthy - end - end - - context 'when in batch' do - let(:bid) { 'SAMPLEBID' } - let(:jid) { 'SAMPLEJID' } - context 'when successful' do - it 'yields' do - yielded = false - subject.call(nil, { 'bid' => bid, 'jid' => jid }, nil) { yielded = true } - expect(yielded).to be_truthy - end - - it 'calls process_successful_job' do - expect(Sidekiq::Batch).to receive(:process_successful_job).with(bid, nil) - subject.call(nil, { 'bid' => bid }, nil) {} - end - end - - context 'when failed' do - it 'calls process_failed_job and reraises exception' do - reraised = false - expect(Sidekiq::Batch).to receive(:process_failed_job) - begin - subject.call(nil, { 'bid' => bid }, nil) { raise 'ERR' } - rescue - reraised = true - end - expect(reraised).to be_truthy - end - end - end - end - - describe Sidekiq::Batch::Middleware::ClientMiddleware do - context 'when without batch' do - it 'just yields' do - yielded = false - expect(Sidekiq::Batch).not_to receive(:increment_job_queue) - subject.call(nil, {}, nil) { yielded = true } - expect(yielded).to be_truthy - end - end - - context 'when in batch' do - let(:bid) { 'SAMPLEBID' } - let(:jid) { 'SAMPLEJID' } - before { Thread.current[:batch] = Sidekiq::Batch.new(bid) } - - it 'yields' do - yielded = false - subject.call(nil, { 'jid' => jid }, nil) { yielded = true } - expect(yielded).to be_truthy - end - - it 'increments job queue' do - # expect(Sidekiq::Batch).to receive(:increment_job_queue).with(bid) - # subject.call(nil, { 'jid' => jid }, nil) {} - end - - it 'assigns bid to msg' do - msg = { 'jid' => jid } - subject.call(nil, msg, nil) {} - expect(msg[:bid]).to eq(bid) - end - end - end -end - -describe Sidekiq::Batch::Middleware do - let(:config) { class_double(Sidekiq) } - let(:client_middleware) { double(Sidekiq::Middleware::Chain) } - - context 'client' do - it 'adds client middleware' do - expect(Sidekiq).to receive(:configure_client).and_yield(config) - expect(config).to receive(:client_middleware).and_yield(client_middleware) - expect(client_middleware).to receive(:add).with(Sidekiq::Batch::Middleware::ClientMiddleware) - Sidekiq::Batch::Middleware.configure - end - end - - context 'server' do - let(:server_middleware) { double(Sidekiq::Middleware::Chain) } - - it 'adds client and server middleware' do - expect(Sidekiq).to receive(:configure_server).and_yield(config) - expect(config).to receive(:client_middleware).and_yield(client_middleware) - expect(config).to receive(:server_middleware).and_yield(server_middleware) - expect(client_middleware).to receive(:add).with(Sidekiq::Batch::Middleware::ClientMiddleware) - expect(server_middleware).to receive(:add).with(Sidekiq::Batch::Middleware::ServerMiddleware) - Sidekiq::Batch::Middleware.configure - end - end - - context 'worker' do - it 'defines method bid' do - expect(Sidekiq::Worker.instance_methods).to include(:bid) - end - - it 'defines method batch' do - expect(Sidekiq::Worker.instance_methods).to include(:batch) - end - - it 'defines method valid_within_batch?' do - expect(Sidekiq::Worker.instance_methods).to include(:valid_within_batch?) - end - end -end diff --git a/vendor/gems/sidekiq-batch/spec/sidekiq/batch/status_spec.rb b/vendor/gems/sidekiq-batch/spec/sidekiq/batch/status_spec.rb deleted file mode 100644 index 3b4a85af9..000000000 --- a/vendor/gems/sidekiq-batch/spec/sidekiq/batch/status_spec.rb +++ /dev/null @@ -1,92 +0,0 @@ -require 'spec_helper' - -describe Sidekiq::Batch::Status do - let(:bid) { 'BID' } - let(:batch) { Sidekiq::Batch.new(bid) } - subject { described_class.new(bid) } - - describe '#join' do - it 'raises info' do - expect { subject.join }.to raise_error('Not supported') - end - end - - describe '#pending' do - context 'when not initalized' do - it 'returns 0 pending jobs' do - expect(subject.pending).to eq(0) - end - end - - context 'when more than 0' do - before { batch.jobs do TestWorker.perform_async end } - it 'returns pending jobs' do - expect(subject.pending).to eq(1) - end - end - end - - describe '#failures' do - context 'when not initalized' do - it 'returns 0 failed jobs' do - expect(subject.failures).to eq(0) - end - end - - context 'when more than 0' do - before { batch.increment_job_queue(bid) } - before { Sidekiq::Batch.process_failed_job(bid, 'FAILEDID') } - - it 'returns failed jobs' do - expect(subject.failures).to eq(1) - end - end - end - - describe '#failure_info' do - context 'when not initalized' do - it 'returns empty array' do - expect(subject.failure_info).to eq([]) - end - end - - context 'when with error' do - before { Sidekiq::Batch.process_failed_job(bid, 'jid123') } - - it 'returns array with failed jids' do - expect(subject.failure_info).to eq(['jid123']) - end - end - end - - describe '#total' do - context 'when not initalized' do - it 'returns 0 failed jobs' do - expect(subject.total).to eq(0) - end - end - - context 'when more than 0' do - before { batch.jobs do TestWorker.perform_async end } - - it 'returns failed jobs' do - expect(subject.total).to eq(1) - end - end - end - - describe '#data' do - it 'returns batch description' do - expect(subject.data).to include(total: 0, failures: 0, pending: 0, created_at: nil, complete: false, failure_info: [], parent_bid: nil) - end - end - - describe '#created_at' do - it 'returns time' do - batch = Sidekiq::Batch.new - batch.jobs do TestWorker.perform_async end - status = described_class.new(batch.bid) - expect(status.created_at).not_to be_nil - end - end -end diff --git a/vendor/gems/sidekiq-batch/spec/sidekiq/batch_spec.rb b/vendor/gems/sidekiq-batch/spec/sidekiq/batch_spec.rb deleted file mode 100644 index 4d6afaa92..000000000 --- a/vendor/gems/sidekiq-batch/spec/sidekiq/batch_spec.rb +++ /dev/null @@ -1,304 +0,0 @@ -require 'spec_helper' - -class TestWorker - include Sidekiq::Worker - def perform - end -end - -describe Sidekiq::Batch do - it 'has a version number' do - expect(Sidekiq::Batch::VERSION).not_to be nil - end - - describe '#initialize' do - subject { described_class } - - it 'creates bid when called without it' do - expect(subject.new.bid).not_to be_nil - end - - it 'reuses bid when called with it' do - batch = subject.new('dayPO5KxuRXXxw') - expect(batch.bid).to eq('dayPO5KxuRXXxw') - end - end - - describe '#description' do - let(:description) { 'custom description' } - before { subject.description = description } - - it 'sets descriptions' do - expect(subject.description).to eq(description) - end - - it 'persists description' do - expect(Sidekiq.redis { |r| r.hget("BID-#{subject.bid}", 'description') }) - .to eq(description) - end - end - - describe '#callback_queue' do - let(:callback_queue) { 'custom_queue' } - before { subject.callback_queue = callback_queue } - - it 'sets callback_queue' do - expect(subject.callback_queue).to eq(callback_queue) - end - - it 'persists callback_queue' do - expect(Sidekiq - .redis { |r| r.hget("BID-#{subject.bid}", 'callback_queue') }) - .to eq(callback_queue) - end - end - - describe '#jobs' do - it 'throws error if no block given' do - expect { subject.jobs }.to raise_error Sidekiq::Batch::NoBlockGivenError - end - - it 'increments to_process (when started)' - - it 'decrements to_process (when finished)' - # it 'calls process_successful_job to wait for block to finish' do - # batch = Sidekiq::Batch.new - # expect(Sidekiq::Batch).to receive(:process_successful_job).with(batch.bid) - # batch.jobs {} - # end - - it 'sets Thread.current bid' do - batch = Sidekiq::Batch.new - batch.jobs do - expect(Thread.current[:batch]).to eq(batch) - end - end - end - - describe '#invalidate_all' do - class InvalidatableJob - include Sidekiq::Worker - - def perform - return unless valid_within_batch? - was_performed - end - - def was_performed; end - end - - it 'marks batch in redis as invalidated' do - batch = Sidekiq::Batch.new - job = InvalidatableJob.new - allow(job).to receive(:was_performed) - - batch.invalidate_all - batch.jobs { job.perform } - - expect(job).not_to have_received(:was_performed) - end - - context 'nested batches' do - let(:batch_parent) { Sidekiq::Batch.new } - let(:batch_child_1) { Sidekiq::Batch.new } - let(:batch_child_2) { Sidekiq::Batch.new } - let(:job_of_parent) { InvalidatableJob.new } - let(:job_of_child_1) { InvalidatableJob.new } - let(:job_of_child_2) { InvalidatableJob.new } - - before do - allow(job_of_parent).to receive(:was_performed) - allow(job_of_child_1).to receive(:was_performed) - allow(job_of_child_2).to receive(:was_performed) - end - - it 'invalidates all job if parent batch is marked as invalidated' do - batch_parent.invalidate_all - batch_parent.jobs do - [ - job_of_parent.perform, - batch_child_1.jobs do - [ - job_of_child_1.perform, - batch_child_2.jobs { job_of_child_2.perform } - ] - end - ] - end - - expect(job_of_parent).not_to have_received(:was_performed) - expect(job_of_child_1).not_to have_received(:was_performed) - expect(job_of_child_2).not_to have_received(:was_performed) - end - - it 'invalidates only requested batch' do - batch_child_2.invalidate_all - batch_parent.jobs do - [ - job_of_parent.perform, - batch_child_1.jobs do - [ - job_of_child_1.perform, - batch_child_2.jobs { job_of_child_2.perform } - ] - end - ] - end - - expect(job_of_parent).to have_received(:was_performed) - expect(job_of_child_1).to have_received(:was_performed) - expect(job_of_child_2).not_to have_received(:was_performed) - end - end - end - - describe '#process_failed_job' do - let(:batch) { Sidekiq::Batch.new } - let(:bid) { batch.bid } - let(:jid) { 'ABCD' } - before { Sidekiq.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } } - - context 'complete' do - let(:failed_jid) { 'xxx' } - - it 'tries to call complete callback' do - expect(Sidekiq::Batch).to receive(:enqueue_callbacks).with(:complete, bid) - Sidekiq::Batch.process_failed_job(bid, failed_jid) - end - - it 'add job to failed list' do - Sidekiq::Batch.process_failed_job(bid, 'failed-job-id') - Sidekiq::Batch.process_failed_job(bid, failed_jid) - failed = Sidekiq.redis { |r| r.smembers("BID-#{bid}-failed") } - expect(failed).to eq(['xxx', 'failed-job-id']) - end - end - end - - describe '#process_successful_job' do - let(:batch) { Sidekiq::Batch.new } - let(:bid) { batch.bid } - let(:jid) { 'ABCD' } - before { Sidekiq.redis { |r| r.hset("BID-#{bid}", 'pending', 1) } } - - context 'complete' do - before { batch.on(:complete, Object) } - # before { batch.increment_job_queue(bid) } - before { batch.jobs do TestWorker.perform_async end } - before { Sidekiq::Batch.process_failed_job(bid, 'failed-job-id') } - - it 'tries to call complete callback' do - expect(Sidekiq::Batch).to receive(:enqueue_callbacks).with(:complete, bid) - Sidekiq::Batch.process_successful_job(bid, 'failed-job-id') - end - end - - context 'success' do - before { batch.on(:complete, Object) } - it 'tries to call complete callback' do - expect(Sidekiq::Batch).to receive(:enqueue_callbacks).with(:complete, bid).ordered - expect(Sidekiq::Batch).to receive(:enqueue_callbacks).with(:success, bid).ordered - Sidekiq::Batch.process_successful_job(bid, jid) - end - - it 'cleanups redis key' do - Sidekiq::Batch.process_successful_job(bid, jid) - expect(Sidekiq.redis { |r| r.get("BID-#{bid}-pending") }.to_i).to eq(0) - end - end - end - - describe '#increment_job_queue' do - let(:bid) { 'BID' } - let(:batch) { Sidekiq::Batch.new } - - it 'increments pending' do - batch.jobs do TestWorker.perform_async end - pending = Sidekiq.redis { |r| r.hget("BID-#{batch.bid}", 'pending') } - expect(pending).to eq('1') - end - - it 'increments total' do - batch.jobs do TestWorker.perform_async end - total = Sidekiq.redis { |r| r.hget("BID-#{batch.bid}", 'total') } - expect(total).to eq('1') - end - end - - describe '#enqueue_callbacks' do - let(:callback) { double('callback') } - let(:event) { :complete } - - context 'on :success' do - let(:event) { :success } - - context 'when no callbacks are defined' do - it 'clears redis keys' do - batch = Sidekiq::Batch.new - expect(Sidekiq::Batch).to receive(:cleanup_redis).with(batch.bid) - Sidekiq::Batch.enqueue_callbacks(event, batch.bid) - end - end - end - - context 'when already called' do - it 'returns and does not enqueue callbacks' do - batch = Sidekiq::Batch.new - batch.on(event, SampleCallback) - Sidekiq.redis { |r| r.hset("BID-#{batch.bid}", event, true) } - - expect(Sidekiq::Client).not_to receive(:push) - Sidekiq::Batch.enqueue_callbacks(event, batch.bid) - end - end - - context 'when not yet called' do - context 'when there is no callback' do - it 'it returns' do - batch = Sidekiq::Batch.new - - expect(Sidekiq::Client).not_to receive(:push) - Sidekiq::Batch.enqueue_callbacks(event, batch.bid) - end - end - - context 'when callback defined' do - let(:opts) { { 'a' => 'b' } } - - it 'calls it passing options' do - batch = Sidekiq::Batch.new - batch.on(event, SampleCallback, opts) - - expect(Sidekiq::Client).to receive(:push_bulk).with( - 'class' => Sidekiq::Batch::Callback::Worker, - 'args' => [['SampleCallback', event.to_s, opts, batch.bid, nil]], - 'queue' => 'default' - ) - Sidekiq::Batch.enqueue_callbacks(event, batch.bid) - end - end - - context 'when multiple callbacks are defined' do - let(:opts) { { 'a' => 'b' } } - let(:opts2) { { 'b' => 'a' } } - - it 'enqueues each callback passing their options' do - batch = Sidekiq::Batch.new - batch.on(event, SampleCallback, opts) - batch.on(event, SampleCallback2, opts2) - - expect(Sidekiq::Client).to receive(:push_bulk).with( - 'class' => Sidekiq::Batch::Callback::Worker, - 'args' => [ - ['SampleCallback2', event.to_s, opts2, batch.bid, nil], - ['SampleCallback', event.to_s, opts, batch.bid, nil] - ], - 'queue' => 'default' - ) - - Sidekiq::Batch.enqueue_callbacks(event, batch.bid) - end - end - end - end -end diff --git a/vendor/gems/sidekiq-batch/spec/spec_helper.rb b/vendor/gems/sidekiq-batch/spec/spec_helper.rb deleted file mode 100644 index ca90a0d67..000000000 --- a/vendor/gems/sidekiq-batch/spec/spec_helper.rb +++ /dev/null @@ -1,24 +0,0 @@ -require "simplecov" -SimpleCov.start - -$LOAD_PATH.unshift File.expand_path('../../lib', __FILE__) -require 'fakeredis/rspec' -require 'sidekiq/batch' - -redis_opts = { url: "redis://127.0.0.1:6379/1" } -redis_opts[:driver] = Redis::Connection::Memory if defined?(Redis::Connection::Memory) - -Sidekiq.configure_client do |config| - config.redis = redis_opts -end - -Sidekiq.configure_server do |config| - config.redis = redis_opts -end - -RSpec.configure do |config| - config.filter_run focus: true - config.run_all_when_everything_filtered = true -end - -Dir[File.dirname(__FILE__) + "/support/**/*.rb"].each {|f| require f } diff --git a/vendor/gems/sidekiq-batch/spec/support/sample_callback.rb b/vendor/gems/sidekiq-batch/spec/support/sample_callback.rb deleted file mode 100644 index 8ca78b349..000000000 --- a/vendor/gems/sidekiq-batch/spec/support/sample_callback.rb +++ /dev/null @@ -1,2 +0,0 @@ - class SampleCallback; end - class SampleCallback2; end