From 01eef8d6de4ea2cbc6766e67716bc65c915bd78f Mon Sep 17 00:00:00 2001 From: GCorbel Date: Tue, 20 Aug 2024 12:13:27 -0400 Subject: [PATCH 01/15] Gives the previous cursor in the scroll block --- lib/mongoid/criteria/scrollable.rb | 5 ++-- lib/mongoid/scroll/base_cursor.rb | 38 +++++++++++++++++------------- spec/mongoid/criteria_spec.rb | 19 +++++++++++++++ 3 files changed, 44 insertions(+), 18 deletions(-) diff --git a/lib/mongoid/criteria/scrollable.rb b/lib/mongoid/criteria/scrollable.rb index bd70ee1..b324a2d 100644 --- a/lib/mongoid/criteria/scrollable.rb +++ b/lib/mongoid/criteria/scrollable.rb @@ -11,11 +11,12 @@ def scroll(cursor_or_type = nil, &_block) criteria.merge!(default_sort) if no_sort_option? cursor_options = build_cursor_options(criteria) cursor = cursor.is_a?(cursor_type) ? cursor : new_cursor(cursor_type, cursor, cursor_options) + direction = scroll_direction(criteria) raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options) cursor_criteria = build_cursor_criteria(criteria, cursor) if block_given? - cursor_criteria.order_by(_id: scroll_direction(criteria)).each do |record| - yield record, cursor_from_record(cursor_type, record, cursor_options) + cursor_criteria.order_by(_id: direction).each do |record| + yield record, cursor_from_record(cursor_type, record, cursor_options), cursor_from_record(cursor_type, record, cursor_options.merge(previous: true)) end else cursor_criteria diff --git a/lib/mongoid/scroll/base_cursor.rb b/lib/mongoid/scroll/base_cursor.rb index b34e225..f18def1 100644 --- a/lib/mongoid/scroll/base_cursor.rb +++ b/lib/mongoid/scroll/base_cursor.rb @@ -1,7 +1,7 @@ module Mongoid module Scroll class BaseCursor - attr_accessor :value, :tiebreak_id, :field_type, :field_name, :direction, :include_current + attr_accessor :value, :tiebreak_id, :field_type, :field_name, :direction, :include_current, :previous def initialize(value, options = {}) @value = value @@ -10,6 +10,7 @@ def initialize(value, options = {}) @field_name = options[:field_name] @direction = options[:direction] || 1 @include_current = options[:include_current] || false + @previous = options[:previous] || false end def criteria @@ -81,25 +82,30 @@ def transform_field_value(field_type, field_name, value) end def extract_field_options(options) - if options && (field_name = options[:field_name]) && (field_type = options[:field_type]) - { - field_type: field_type.to_s, - field_name: field_name.to_s, - direction: options[:direction] || 1, - include_current: options[:include_current] || false - } - elsif options && (field = options[:field]) - { - field_type: field.type.to_s, - field_name: field.name.to_s, - direction: options[:direction] || 1, - include_current: options[:include_current] || false - } + return nil unless options + + { + direction: options[:direction] || 1, + include_current: options[:include_current] || false, + previous: options[:previous] || false + }.tap do |extracted_options| + if (field_name = options[:field_name]) && (field_type = options[:field_type]) + extracted_options.merge!({ + field_type: field_type.to_s, + field_name: field_name.to_s + }) + elsif (field = options[:field]) + extracted_options.merge!({ + field_type: field.type.to_s, + field_name: field.name.to_s + }) + end end end def compare_direction - direction == 1 ? '$gt' : '$lt' + dir = previous ? -direction : direction + dir == 1 ? '$gt' : '$lt' end def tiebreak_compare_direction diff --git a/spec/mongoid/criteria_spec.rb b/spec/mongoid/criteria_spec.rb index 7f61625..4d6bf07 100644 --- a/spec/mongoid/criteria_spec.rb +++ b/spec/mongoid/criteria_spec.rb @@ -160,6 +160,25 @@ break end end + it 'can scroll back with the previous cursor' do + first_cursor = nil + second_cursor = nil + third_cursor = nil + Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, cursor| + second_cursor = cursor + end + Feed::Item.asc(field_name).limit(2).scroll(second_cursor) do |_, next_cursor, previous_cursor| + first_cursor = previous_cursor + third_cursor = next_cursor + end + first_item = Feed::Item.asc(field_name).to_a[0] + from_item = Feed::Item.asc(field_name).scroll(first_cursor).to_a.first + expect(from_item).to eq first_item + + fifth_item = Feed::Item.asc(field_name).to_a[4] + from_item = Feed::Item.asc(field_name).scroll(third_cursor).to_a.first + expect(from_item).to eq fifth_item + end end end end From 772010b4e4c39a3ef13f9026150824e8cc588702 Mon Sep 17 00:00:00 2001 From: GCorbel Date: Tue, 20 Aug 2024 13:39:27 -0400 Subject: [PATCH 02/15] apply changes for mongo --- lib/mongo/scrollable.rb | 2 +- lib/mongoid/scroll/base_cursor.rb | 34 ++++++++++++++---------------- spec/mongo/collection_view_spec.rb | 22 +++++++++++++++++++ spec/mongoid/criteria_spec.rb | 1 + 4 files changed, 40 insertions(+), 19 deletions(-) diff --git a/lib/mongo/scrollable.rb b/lib/mongo/scrollable.rb index 74a80a2..0449cee 100644 --- a/lib/mongo/scrollable.rb +++ b/lib/mongo/scrollable.rb @@ -27,7 +27,7 @@ def scroll(cursor_or_type = nil, options = nil, &_block) # scroll if block_given? view.each do |record| - yield record, cursor_type.from_record(record, cursor_options) + yield record, cursor_type.from_record(record, cursor_options), cursor_type.from_record(record, cursor_options.merge(previous: true)) end else view diff --git a/lib/mongoid/scroll/base_cursor.rb b/lib/mongoid/scroll/base_cursor.rb index f18def1..41ac877 100644 --- a/lib/mongoid/scroll/base_cursor.rb +++ b/lib/mongoid/scroll/base_cursor.rb @@ -82,24 +82,22 @@ def transform_field_value(field_type, field_name, value) end def extract_field_options(options) - return nil unless options - - { - direction: options[:direction] || 1, - include_current: options[:include_current] || false, - previous: options[:previous] || false - }.tap do |extracted_options| - if (field_name = options[:field_name]) && (field_type = options[:field_type]) - extracted_options.merge!({ - field_type: field_type.to_s, - field_name: field_name.to_s - }) - elsif (field = options[:field]) - extracted_options.merge!({ - field_type: field.type.to_s, - field_name: field.name.to_s - }) - end + if options && (field_name = options[:field_name]) && (field_type = options[:field_type]) + { + field_type: field_type.to_s, + field_name: field_name.to_s, + direction: options[:direction] || 1, + include_current: options[:include_current] || false, + previous: options[:previous] || false + } + elsif options && (field = options[:field]) + { + field_type: field.type.to_s, + field_name: field.name.to_s, + direction: options[:direction] || 1, + include_current: options[:include_current] || false, + previous: options[:previous] || false + } end end diff --git a/spec/mongo/collection_view_spec.rb b/spec/mongo/collection_view_spec.rb index c8e938c..d2dd237 100644 --- a/spec/mongo/collection_view_spec.rb +++ b/spec/mongo/collection_view_spec.rb @@ -106,6 +106,28 @@ expect(cursor.value).to eq record[field_name.to_s] expect(cursor.tiebreak_id).to eq record['_id'] end + it 'can scroll back with the previous cursor' do + records = [] + first_cursor = nil + second_cursor = nil + third_cursor = nil + criteria = Mongoid.default_client['feed_items'].find.limit(2) + criteria.scroll(cursor_type) do |record, next_cursor| + second_cursor = next_cursor + end + criteria.scroll(second_cursor) do |record, next_cursor, previous_cursor| + first_cursor = previous_cursor + third_cursor = next_cursor + end + + first_item = Mongoid.default_client['feed_items'].find.to_a[0] + from_item = Mongoid.default_client['feed_items'].find.scroll(first_cursor).to_a.first + expect(from_item).to eq first_item + + fifth_item = Mongoid.default_client['feed_items'].find.to_a[4] + from_item = Mongoid.default_client['feed_items'].find.scroll(third_cursor).to_a.first + expect(from_item).to eq fifth_item + end end end end diff --git a/spec/mongoid/criteria_spec.rb b/spec/mongoid/criteria_spec.rb index 4d6bf07..d51eba8 100644 --- a/spec/mongoid/criteria_spec.rb +++ b/spec/mongoid/criteria_spec.rb @@ -171,6 +171,7 @@ first_cursor = previous_cursor third_cursor = next_cursor end + first_item = Feed::Item.asc(field_name).to_a[0] from_item = Feed::Item.asc(field_name).scroll(first_cursor).to_a.first expect(from_item).to eq first_item From b5824588d004287903d93548bb2930c3d6658efe Mon Sep 17 00:00:00 2001 From: GCorbel Date: Tue, 20 Aug 2024 15:12:04 -0400 Subject: [PATCH 03/15] encrypt with the previous option --- lib/mongoid/scroll/base64_encoded_cursor.rb | 6 ++-- spec/mongoid/base64_encoded_cursor_spec.rb | 38 +++++++++++++++++---- 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/lib/mongoid/scroll/base64_encoded_cursor.rb b/lib/mongoid/scroll/base64_encoded_cursor.rb index b0c2b47..a58a564 100644 --- a/lib/mongoid/scroll/base64_encoded_cursor.rb +++ b/lib/mongoid/scroll/base64_encoded_cursor.rb @@ -18,7 +18,8 @@ def initialize(value, options = {}) field_name: parsed['field_name'], direction: parsed['direction'], include_current: parsed['include_current'], - tiebreak_id: parsed['tiebreak_id'] && !parsed['tiebreak_id'].empty? ? BSON::ObjectId.from_string(parsed['tiebreak_id']) : nil + tiebreak_id: parsed['tiebreak_id'] && !parsed['tiebreak_id'].empty? ? BSON::ObjectId.from_string(parsed['tiebreak_id']) : nil, + previous: parsed['previous'] } else super nil, options @@ -32,7 +33,8 @@ def to_s field_name: field_name, direction: direction, include_current: include_current, - tiebreak_id: tiebreak_id && tiebreak_id.to_s + tiebreak_id: tiebreak_id && tiebreak_id.to_s, + previous: previous }.to_json) end end diff --git a/spec/mongoid/base64_encoded_cursor_spec.rb b/spec/mongoid/base64_encoded_cursor_spec.rb index b5b9876..8d65484 100644 --- a/spec/mongoid/base64_encoded_cursor_spec.rb +++ b/spec/mongoid/base64_encoded_cursor_spec.rb @@ -3,17 +3,18 @@ describe Mongoid::Scroll::Base64EncodedCursor do context 'new' do context 'an empty cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6bnVsbCwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOm51bGx9' } + let(:base64_string) { 'eyJ2YWx1ZSI6bnVsbCwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOm51bGwsInByZXZpb3VzIjpmYWxzZX0=' } subject do Mongoid::Scroll::Base64EncodedCursor.new base64_string end its(:tiebreak_id) { should be_nil } its(:value) { should be_nil } its(:criteria) { should eq({}) } + its(:previous) { should be_falsy } its(:to_s) { should eq(base64_string) } end context 'a string field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6ImEgc3RyaW5nIiwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2M2RmODA5NDQzNDE3YzdkMmIxMDIifQ==' } + let(:base64_string) { 'eyJ2YWx1ZSI6ImEgc3RyaW5nIiwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2M2RmODA5NDQzNDE3YzdkMmIxMDIiLCJwcmV2aW91cyI6ZmFsc2V9' } let(:a_value) { 'a string' } let(:tiebreak_id) { BSON::ObjectId.from_string('64063df809443417c7d2b102') } let(:criteria) do @@ -32,10 +33,11 @@ its(:value) { should eq a_value } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } + its(:previous) { should be_falsy } its(:to_s) { should eq(base64_string) } end context 'an id field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6IjY0MDY0NTg0MDk0NDM0MjgxZmE3MWFiMiIsImZpZWxkX3R5cGUiOiJCU09OOjpPYmplY3RJZCIsImZpZWxkX25hbWUiOiJpZCIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2NDU4NDA5NDQzNDI4MWZhNzFhYjIifQ==' } + let(:base64_string) { 'eyJ2YWx1ZSI6IjY0MDY0NTg0MDk0NDM0MjgxZmE3MWFiMiIsImZpZWxkX3R5cGUiOiJCU09OOjpPYmplY3RJZCIsImZpZWxkX25hbWUiOiJpZCIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2NDU4NDA5NDQzNDI4MWZhNzFhYjIiLCJwcmV2aW91cyI6ZmFsc2V9' } let(:a_value) { BSON::ObjectId('64064584094434281fa71ab2') } let(:tiebreak_id) { a_value } let(:criteria) do @@ -52,10 +54,11 @@ its(:value) { should eq a_value } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } + its(:previous) { should be_falsy } its(:to_s) { should eq(base64_string) } end context 'an integer field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6MTAsImZpZWxkX3R5cGUiOiJJbnRlZ2VyIiwiZmllbGRfbmFtZSI6ImFfaW50ZWdlciIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2M2RmODA5NDQzNDE3YzdkMmIxMDgifQ==' } + let(:base64_string) { 'eyJ2YWx1ZSI6MTAsImZpZWxkX3R5cGUiOiJJbnRlZ2VyIiwiZmllbGRfbmFtZSI6ImFfaW50ZWdlciIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2M2RmODA5NDQzNDE3YzdkMmIxMDgiLCJwcmV2aW91cyI6ZmFsc2V9' } let(:a_value) { 10 } let(:tiebreak_id) { BSON::ObjectId('64063df809443417c7d2b108') } let(:criteria) do @@ -74,10 +77,11 @@ its(:value) { should eq a_value } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } + its(:previous) { should be_falsy } its(:to_s) { should eq(base64_string) } end context 'a date/time field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzU5MDEyMy4wLCJmaWVsZF90eXBlIjoiRGF0ZVRpbWUiLCJmaWVsZF9uYW1lIjoiYV9kYXRldGltZSIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2NDNhNzA5NDQzNDIzOWYyZGJmODYifQ==' } + let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzU5MDEyMy4wLCJmaWVsZF90eXBlIjoiRGF0ZVRpbWUiLCJmaWVsZF9uYW1lIjoiYV9kYXRldGltZSIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2NDNhNzA5NDQzNDIzOWYyZGJmODYiLCJwcmV2aW91cyI6ZmFsc2V9' } let(:a_value) { DateTime.new(2013, 12, 21, 1, 42, 3, 'UTC') } let(:tiebreak_id) { BSON::ObjectId('640643a7094434239f2dbf86') } let(:criteria) do @@ -94,10 +98,11 @@ its(:value) { should eq a_value } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } + its(:previous) { should be_falsy } its(:to_s) { should eq(base64_string) } end context 'a date field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzU4NDAwMCwiZmllbGRfdHlwZSI6IkRhdGUiLCJmaWVsZF9uYW1lIjoiYV9kYXRlIiwiZGlyZWN0aW9uIjoxLCJpbmNsdWRlX2N1cnJlbnQiOmZhbHNlLCJ0aWVicmVha19pZCI6IjY0MDY0MmM5MDk0NDM0MjEyYzRkNDQyMCJ9' } + let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzU4NDAwMCwiZmllbGRfdHlwZSI6IkRhdGUiLCJmaWVsZF9uYW1lIjoiYV9kYXRlIiwiZGlyZWN0aW9uIjoxLCJpbmNsdWRlX2N1cnJlbnQiOmZhbHNlLCJ0aWVicmVha19pZCI6IjY0MDY0MmM5MDk0NDM0MjEyYzRkNDQyMCIsInByZXZpb3VzIjpmYWxzZX0=' } let(:tiebreak_id) { BSON::ObjectId('640642c9094434212c4d4420') } let(:a_value) { Date.new(2013, 12, 21) } let(:criteria) do @@ -114,10 +119,11 @@ its(:value) { should eq a_value } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } + its(:previous) { should be_falsy } its(:to_s) { should eq(base64_string) } end context 'a time field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzYwNTcyMy4wLCJmaWVsZF90eXBlIjoiVGltZSIsImZpZWxkX25hbWUiOiJhX3RpbWUiLCJkaXJlY3Rpb24iOjEsImluY2x1ZGVfY3VycmVudCI6ZmFsc2UsInRpZWJyZWFrX2lkIjoiNjQwNjNkNGEwOTQ0MzQxNjZiZDA1M2VkIn0=' } + let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzYwNTcyMy4wLCJmaWVsZF90eXBlIjoiVGltZSIsImZpZWxkX25hbWUiOiJhX3RpbWUiLCJkaXJlY3Rpb24iOjEsImluY2x1ZGVfY3VycmVudCI6ZmFsc2UsInRpZWJyZWFrX2lkIjoiNjQwNjNkNGEwOTQ0MzQxNjZiZDA1M2VkIiwicHJldmlvdXMiOmZhbHNlfQ==' } let(:item_id) { BSON::ObjectId('640636f209443407333b46d4') } let(:a_value) { Time.new(2013, 12, 21, 6, 2, 3, '+00:00').utc } let(:tiebreak_id) { BSON::ObjectId('64063d4a094434166bd053ed') } @@ -136,6 +142,7 @@ its(:tiebreak_id) { tiebreak_id } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } + its(:previous) { should be_falsy } its(:to_s) { should eq(base64_string) } end context 'an invalid field cursor' do @@ -229,5 +236,22 @@ end.to raise_error Mongoid::Scroll::Errors::UnsupportedFieldTypeError, /The type of the field 'a_array' is not supported: Array./ end end + + it 'encode and decode previous option' do + feed_item = Feed::Item.create! + cursor = Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: 'id', field_type: BSON::ObjectId, previous: true + expect(Mongoid::Scroll::Base64EncodedCursor.new(cursor.to_s).previous).to be_truthy + end + context 'a cursor with previous set to true' do + let(:field_type) { BSON::ObjectId } + let(:field_name) { 'id' } + let(:feed_item) { Feed::Item.create! } + subject do + Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: field_name, field_type: field_type, previous: true + end + its(:value) { should eq feed_item._id } + its(:field_type) { should eq field_type.to_s } + its(:previous) { should be_truthy } + end end end From 76287e88fe2e754c9e8808f91e660e35e1ad90f9 Mon Sep 17 00:00:00 2001 From: GCorbel Date: Tue, 20 Aug 2024 16:11:28 -0400 Subject: [PATCH 04/15] take values before the first page --- lib/mongoid/criteria/scrollable.rb | 4 +++- spec/mongoid/criteria_spec.rb | 21 ++++++++++----------- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/lib/mongoid/criteria/scrollable.rb b/lib/mongoid/criteria/scrollable.rb index b324a2d..08ad1c1 100644 --- a/lib/mongoid/criteria/scrollable.rb +++ b/lib/mongoid/criteria/scrollable.rb @@ -14,9 +14,11 @@ def scroll(cursor_or_type = nil, &_block) direction = scroll_direction(criteria) raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options) cursor_criteria = build_cursor_criteria(criteria, cursor) + first_in_page = nil if block_given? cursor_criteria.order_by(_id: direction).each do |record| - yield record, cursor_from_record(cursor_type, record, cursor_options), cursor_from_record(cursor_type, record, cursor_options.merge(previous: true)) + first_in_page ||= record + yield record, cursor_from_record(cursor_type, record, cursor_options), cursor_from_record(cursor_type, first_in_page, cursor_options.merge(previous: true)) end else cursor_criteria diff --git a/spec/mongoid/criteria_spec.rb b/spec/mongoid/criteria_spec.rb index d51eba8..7d23dce 100644 --- a/spec/mongoid/criteria_spec.rb +++ b/spec/mongoid/criteria_spec.rb @@ -163,22 +163,21 @@ it 'can scroll back with the previous cursor' do first_cursor = nil second_cursor = nil - third_cursor = nil - Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, cursor| - second_cursor = cursor + + Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, next_cursor| + second_cursor = next_cursor end - Feed::Item.asc(field_name).limit(2).scroll(second_cursor) do |_, next_cursor, previous_cursor| + Feed::Item.asc(field_name).limit(2).scroll(second_cursor) do |_, _, previous_cursor| first_cursor = previous_cursor - third_cursor = next_cursor end - first_item = Feed::Item.asc(field_name).to_a[0] - from_item = Feed::Item.asc(field_name).scroll(first_cursor).to_a.first - expect(from_item).to eq first_item + records = [] + Feed::Item.asc(field_name).limit(2).scroll(first_cursor) do |record| + records << record + end + expect(records).to eq Feed::Item.asc(field_name).limit(2).to_a - fifth_item = Feed::Item.asc(field_name).to_a[4] - from_item = Feed::Item.asc(field_name).scroll(third_cursor).to_a.first - expect(from_item).to eq fifth_item + expect(Feed::Item.asc(field_name).scroll(first_cursor).to_a).to eq Feed::Item.asc(field_name).limit(2).to_a end end end From 3ca3244c17dc11ccc60a3c2030a76fb6771d2d87 Mon Sep 17 00:00:00 2001 From: GCorbel Date: Wed, 21 Aug 2024 15:26:06 -0400 Subject: [PATCH 05/15] keep the ordering when fetching previous records --- lib/mongoid/criteria/scrollable.rb | 27 ++++++++++++++++++--------- spec/mongoid/criteria_spec.rb | 23 +++++++++++++---------- 2 files changed, 31 insertions(+), 19 deletions(-) diff --git a/lib/mongoid/criteria/scrollable.rb b/lib/mongoid/criteria/scrollable.rb index 08ad1c1..769e3d6 100644 --- a/lib/mongoid/criteria/scrollable.rb +++ b/lib/mongoid/criteria/scrollable.rb @@ -11,17 +11,15 @@ def scroll(cursor_or_type = nil, &_block) criteria.merge!(default_sort) if no_sort_option? cursor_options = build_cursor_options(criteria) cursor = cursor.is_a?(cursor_type) ? cursor : new_cursor(cursor_type, cursor, cursor_options) - direction = scroll_direction(criteria) raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options) - cursor_criteria = build_cursor_criteria(criteria, cursor) - first_in_page = nil + records = find_records(criteria, cursor) if block_given? - cursor_criteria.order_by(_id: direction).each do |record| - first_in_page ||= record - yield record, cursor_from_record(cursor_type, record, cursor_options), cursor_from_record(cursor_type, first_in_page, cursor_options.merge(previous: true)) + first_cursor = cursor_from_record(cursor_type, records.first, cursor_options.merge(previous: true)) + records.each do |record| + yield record, cursor_from_record(cursor_type, record, cursor_options), first_cursor end else - cursor_criteria + records end end @@ -63,10 +61,21 @@ def new_cursor(cursor_type, cursor, cursor_options) cursor_type.new(cursor, cursor_options) end - def build_cursor_criteria(criteria, cursor) + def find_records(criteria, cursor) cursor_criteria = criteria.dup cursor_criteria.selector = { '$and' => [criteria.selector, cursor.criteria] } - cursor_criteria + if cursor.previous && criteria.options[:limit] + pipeline = [ + { '$match' => cursor_criteria.selector }, + { '$sort' => { cursor.field_name => -cursor.direction } }, + { '$limit' => criteria.options[:limit] }, + { '$sort' => { cursor.field_name => cursor.direction } } + ] + aggregation = cursor_criteria.view.aggregate(pipeline) + aggregation.map { Mongoid::Factory.from_db(cursor_criteria.klass, _1) } + else + cursor_criteria.order_by(_id: scroll_direction(criteria)) + end end def cursor_from_record(cursor_type, record, cursor_options) diff --git a/spec/mongoid/criteria_spec.rb b/spec/mongoid/criteria_spec.rb index 7d23dce..1c1901c 100644 --- a/spec/mongoid/criteria_spec.rb +++ b/spec/mongoid/criteria_spec.rb @@ -161,23 +161,26 @@ end end it 'can scroll back with the previous cursor' do - first_cursor = nil - second_cursor = nil + cursor = nil + first_previous_cursor = nil + second_previous_cursor = nil Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, next_cursor| - second_cursor = next_cursor + cursor = next_cursor end - Feed::Item.asc(field_name).limit(2).scroll(second_cursor) do |_, _, previous_cursor| - first_cursor = previous_cursor + + Feed::Item.asc(field_name).limit(2).scroll(cursor) do |_, next_cursor, previous_cursor| + cursor = next_cursor + first_previous_cursor = previous_cursor end - records = [] - Feed::Item.asc(field_name).limit(2).scroll(first_cursor) do |record| - records << record + Feed::Item.asc(field_name).limit(2).scroll(cursor) do |_, _, previous_cursor| + second_previous_cursor = previous_cursor end - expect(records).to eq Feed::Item.asc(field_name).limit(2).to_a - expect(Feed::Item.asc(field_name).scroll(first_cursor).to_a).to eq Feed::Item.asc(field_name).limit(2).to_a + records = Feed::Item.asc(field_name) + expect(Feed::Item.asc(field_name).limit(2).scroll(first_previous_cursor)).to eq(records.limit(2)) + expect(Feed::Item.asc(field_name).limit(2).scroll(second_previous_cursor)).to eq(records.skip(2).limit(2)) end end end From 089a5185d87cacdbf6e3324fc7cb6a7d988470ac Mon Sep 17 00:00:00 2001 From: GCorbel Date: Wed, 21 Aug 2024 17:00:19 -0400 Subject: [PATCH 06/15] keep the ordering when fetching previous records --- lib/mongo/scrollable.rb | 38 +++++++++++++++++++++--------- lib/mongoid/criteria/scrollable.rb | 5 ++-- spec/mongo/collection_view_spec.rb | 32 ++++++++++++------------- 3 files changed, 46 insertions(+), 29 deletions(-) diff --git a/lib/mongo/scrollable.rb b/lib/mongo/scrollable.rb index 0449cee..23eeeeb 100644 --- a/lib/mongo/scrollable.rb +++ b/lib/mongo/scrollable.rb @@ -16,21 +16,37 @@ def scroll(cursor_or_type = nil, options = nil, &_block) cursor_options = { field_name: scroll_field, direction: scroll_direction }.merge(options) cursor = cursor && cursor.is_a?(cursor_type) ? cursor : cursor_type.new(cursor, cursor_options) raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options) - # make a view - view = Mongo::Collection::View.new( - view.collection, - view.selector.merge(cursor.criteria), - sort: (view.sort || {}).merge(_id: scroll_direction), - skip: skip, - limit: limit - ) + + records = nil + if cursor.previous && limit + # scroll backards by reversing the sort order, limit and then reverse again + pipeline = [ + { '$match' => view.selector.merge(cursor.criteria) }, + { '$sort' => { scroll_field => -scroll_direction } }, + { '$limit' => limit }, + { '$sort' => { scroll_field => scroll_direction } } + ] + aggregation_options = view.options.except(:sort) + records = view.aggregate(pipeline, aggregation_options) + else + # make a view + records = Mongo::Collection::View.new( + view.collection, + view.selector.merge(cursor.criteria), + sort: (view.sort || {}).merge(_id: scroll_direction), + skip: skip, + limit: limit + ) + end # scroll if block_given? - view.each do |record| - yield record, cursor_type.from_record(record, cursor_options), cursor_type.from_record(record, cursor_options.merge(previous: true)) + previous_cursor = nil + records.each do |record| + previous_cursor ||= cursor_type.from_record(record, cursor_options.merge(previous: true)) + yield record, cursor_type.from_record(record, cursor_options), previous_cursor end else - view + records end end end diff --git a/lib/mongoid/criteria/scrollable.rb b/lib/mongoid/criteria/scrollable.rb index 769e3d6..303d5c3 100644 --- a/lib/mongoid/criteria/scrollable.rb +++ b/lib/mongoid/criteria/scrollable.rb @@ -14,9 +14,10 @@ def scroll(cursor_or_type = nil, &_block) raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options) records = find_records(criteria, cursor) if block_given? - first_cursor = cursor_from_record(cursor_type, records.first, cursor_options.merge(previous: true)) + previous_cursor = nil records.each do |record| - yield record, cursor_from_record(cursor_type, record, cursor_options), first_cursor + previous_cursor ||= cursor_from_record(cursor_type, record, cursor_options.merge(previous: true)) + yield record, cursor_from_record(cursor_type, record, cursor_options), previous_cursor end else records diff --git a/spec/mongo/collection_view_spec.rb b/spec/mongo/collection_view_spec.rb index d2dd237..c20409c 100644 --- a/spec/mongo/collection_view_spec.rb +++ b/spec/mongo/collection_view_spec.rb @@ -107,26 +107,26 @@ expect(cursor.tiebreak_id).to eq record['_id'] end it 'can scroll back with the previous cursor' do - records = [] - first_cursor = nil - second_cursor = nil - third_cursor = nil - criteria = Mongoid.default_client['feed_items'].find.limit(2) - criteria.scroll(cursor_type) do |record, next_cursor| - second_cursor = next_cursor + cursor = nil + first_previous_cursor = nil + second_previous_cursor = nil + + Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor_type, field_type: field_type) do |_, next_cursor| + cursor = next_cursor end - criteria.scroll(second_cursor) do |record, next_cursor, previous_cursor| - first_cursor = previous_cursor - third_cursor = next_cursor + + Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor, field_type: field_type) do |_, next_cursor, previous_cursor| + cursor = next_cursor + first_previous_cursor = previous_cursor end - first_item = Mongoid.default_client['feed_items'].find.to_a[0] - from_item = Mongoid.default_client['feed_items'].find.scroll(first_cursor).to_a.first - expect(from_item).to eq first_item + Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor, field_type: field_type) do |_, _, previous_cursor| + second_previous_cursor = previous_cursor + end - fifth_item = Mongoid.default_client['feed_items'].find.to_a[4] - from_item = Mongoid.default_client['feed_items'].find.scroll(third_cursor).to_a.first - expect(from_item).to eq fifth_item + records = Mongoid.default_client['feed_items'].find.sort(field_name => 1) + expect(Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(first_previous_cursor, field_type: field_type).to_a).to eq(records.limit(2).to_a) + expect(Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(second_previous_cursor, field_type: field_type).to_a).to eq(records.skip(2).limit(2).to_a) end end end From e50e73d03dfb88fa4493403b8eba8e8da69f61f1 Mon Sep 17 00:00:00 2001 From: GCorbel Date: Thu, 22 Aug 2024 16:42:06 -0400 Subject: [PATCH 07/15] add entry to the change log --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3753475..1d14e29 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ### 1.0.2 (Next) -* Your contribution here. +* [#38](https://github.com/mongoid/mongoid-scroll/pull/38): gives the previous cursor in the scroll block - [@GCorbel](https://github.com/GCorbel). ### 1.0.1 (2023/03/15) From 97876646762fb17e80b390dff5c7dc6f7db01c2e Mon Sep 17 00:00:00 2001 From: GCorbel Date: Thu, 22 Aug 2024 16:44:08 -0400 Subject: [PATCH 08/15] Fix with Ruby 2.6 --- lib/mongoid/criteria/scrollable.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/mongoid/criteria/scrollable.rb b/lib/mongoid/criteria/scrollable.rb index 303d5c3..8bf5ade 100644 --- a/lib/mongoid/criteria/scrollable.rb +++ b/lib/mongoid/criteria/scrollable.rb @@ -73,7 +73,7 @@ def find_records(criteria, cursor) { '$sort' => { cursor.field_name => cursor.direction } } ] aggregation = cursor_criteria.view.aggregate(pipeline) - aggregation.map { Mongoid::Factory.from_db(cursor_criteria.klass, _1) } + aggregation.map { |record| Mongoid::Factory.from_db(cursor_criteria.klass, record) } else cursor_criteria.order_by(_id: scroll_direction(criteria)) end From 3e9475a559024cdd6d5e9cd079b42af7610fcb2c Mon Sep 17 00:00:00 2001 From: GCorbel Date: Thu, 22 Aug 2024 16:53:35 -0400 Subject: [PATCH 09/15] Update the README and CHANGELOG --- CHANGELOG.md | 2 +- README.md | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1d14e29..6a3a507 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ ### 1.0.2 (Next) -* [#38](https://github.com/mongoid/mongoid-scroll/pull/38): gives the previous cursor in the scroll block - [@GCorbel](https://github.com/GCorbel). +* [#38](https://github.com/mongoid/mongoid-scroll/pull/38): Allow to reverse the scroll - [@GCorbel](https://github.com/GCorbel). ### 1.0.1 (2023/03/15) diff --git a/README.md b/README.md index 7e177df..9a0e19e 100644 --- a/README.md +++ b/README.md @@ -79,9 +79,17 @@ end Resume iterating using the previously saved cursor. ```ruby -Feed::Item.desc(:position).limit(5).scroll(saved_cursor) do |record, next_cursor| +Feed::Item.desc(:position).limit(5).scroll(saved_cursor) do |record, _, previous_cursor| # each record, one-by-one - saved_cursor = next_cursor + saved_cursor = previous_cursor +end +``` + +A cursor to get the previous records is available. + +```ruby +Feed::Item.desc(:position).limit(5).scroll(saved_cursor) do |record| + # Loop over the 5 first records end ``` @@ -179,7 +187,7 @@ Feed::Item.desc(:created_at).scroll(cursor) # Raises a Mongoid::Scroll::Errors:: ### Standard Cursor -The `Mongoid::Scroll::Cursor` encodes a value and a tiebreak ID separated by `:`, and does not include other options, such as scroll direction. Take extra care not to pass a cursor into a scroll with different options. +The `Mongoid::Scroll::Cursor` encodes a value and a tiebreak ID separated by `:`, and does not include other options, such as scroll direction. Take extra care not to pass a cursor into a scroll with different options. ### Base64 Encoded Cursor From 0d7637fe4b512a8937a382aef6db901e8ffcf212 Mon Sep 17 00:00:00 2001 From: GCorbel Date: Mon, 26 Aug 2024 13:25:24 -0400 Subject: [PATCH 10/15] minor changes --- CHANGELOG.md | 3 ++- README.md | 9 +++++---- lib/mongo/scrollable.rb | 2 +- lib/mongoid/scroll/version.rb | 2 +- 4 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a3a507..129346b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ -### 1.0.2 (Next) +### 1.1.0 (Next) * [#38](https://github.com/mongoid/mongoid-scroll/pull/38): Allow to reverse the scroll - [@GCorbel](https://github.com/GCorbel). +* Your contribution here. ### 1.0.1 (2023/03/15) diff --git a/README.md b/README.md index 9a0e19e..96c13fa 100644 --- a/README.md +++ b/README.md @@ -76,7 +76,7 @@ Feed::Item.desc(:position).limit(5).scroll do |record, next_cursor| end ``` -Resume iterating using the previously saved cursor. +Resume iterating using saved cursor and save the cursor to go backward. ```ruby Feed::Item.desc(:position).limit(5).scroll(saved_cursor) do |record, _, previous_cursor| @@ -85,11 +85,12 @@ Feed::Item.desc(:position).limit(5).scroll(saved_cursor) do |record, _, previous end ``` -A cursor to get the previous records is available. +Loop over the first records again. ```ruby -Feed::Item.desc(:position).limit(5).scroll(saved_cursor) do |record| - # Loop over the 5 first records +Feed::Item.desc(:position).limit(5).scroll(saved_cursor) do |record, next_cursor| + # each record, one-by-one + saved_cursor = next_cursor end ``` diff --git a/lib/mongo/scrollable.rb b/lib/mongo/scrollable.rb index 23eeeeb..39b865b 100644 --- a/lib/mongo/scrollable.rb +++ b/lib/mongo/scrollable.rb @@ -19,7 +19,7 @@ def scroll(cursor_or_type = nil, options = nil, &_block) records = nil if cursor.previous && limit - # scroll backards by reversing the sort order, limit and then reverse again + # scroll backwards by reversing the sort order, limit and then reverse again pipeline = [ { '$match' => view.selector.merge(cursor.criteria) }, { '$sort' => { scroll_field => -scroll_direction } }, diff --git a/lib/mongoid/scroll/version.rb b/lib/mongoid/scroll/version.rb index 7a7a2b7..0225d54 100644 --- a/lib/mongoid/scroll/version.rb +++ b/lib/mongoid/scroll/version.rb @@ -1,5 +1,5 @@ module Mongoid module Scroll - VERSION = '1.0.2'.freeze + VERSION = '1.1.0'.freeze end end From a1766fbe59637b21e8cb50de6ce966c0eeed5ea5 Mon Sep 17 00:00:00 2001 From: GCorbel Date: Tue, 27 Aug 2024 12:07:11 -0400 Subject: [PATCH 11/15] change to use type --- README.md | 28 ++++++++------- lib/mongo/scrollable.rb | 4 +-- lib/mongoid/criteria/scrollable.rb | 4 +-- lib/mongoid/scroll/base64_encoded_cursor.rb | 4 +-- lib/mongoid/scroll/base_cursor.rb | 10 +++--- spec/mongoid/base64_encoded_cursor_spec.rb | 38 ++++++++++----------- 6 files changed, 45 insertions(+), 43 deletions(-) diff --git a/README.md b/README.md index 96c13fa..272cc18 100644 --- a/README.md +++ b/README.md @@ -69,35 +69,37 @@ end Scroll by `:position` and save a cursor to the last item. ```ruby -saved_cursor = nil -Feed::Item.desc(:position).limit(5).scroll do |record, next_cursor| +saved_previous_cursor = nil +saved_next_cursor = nil + +Feed::Item.desc(:position).limit(5).scroll do |record, next_cursor, previous_cursor| # each record, one-by-one - saved_cursor = next_cursor + saved_next_cursor = next_cursor end ``` Resume iterating using saved cursor and save the cursor to go backward. ```ruby -Feed::Item.desc(:position).limit(5).scroll(saved_cursor) do |record, _, previous_cursor| +Feed::Item.desc(:position).limit(5).scroll(saved_next_cursor) do |record, next_cursor, previous_cursor| # each record, one-by-one - saved_cursor = previous_cursor + saved_previous_cursor = previous_previous_cursor end ``` Loop over the first records again. ```ruby -Feed::Item.desc(:position).limit(5).scroll(saved_cursor) do |record, next_cursor| +Feed::Item.desc(:position).limit(5).scroll(saved_previous_cursor) do |record, next_cursor, previous_cursor| # each record, one-by-one - saved_cursor = next_cursor + saved_next_cursor = next_cursor end ``` The iteration finishes when no more records are available. You can also finish iterating over the remaining records by omitting the query limit. ```ruby -Feed::Item.desc(:position).scroll(saved_cursor) do |record, next_cursor| +Feed::Item.desc(:position).limit(5).scroll(saved_next_cursor) do |record, next_cursor, previous_cursor| # each record, one-by-one end ``` @@ -107,19 +109,19 @@ end Scroll a `Mongo::Collection::View` and save a cursor to the last item. You must also supply a `field_type` of the sort criteria. ```ruby -saved_cursor = nil -client[:feed_items].find.sort(position: -1).limit(5).scroll(nil, { field_type: DateTime }) do |record, next_cursor| +saved_next_cursor = nil +client[:feed_items].find.sort(position: -1).limit(5).scroll(nil, { field_type: DateTime }) do |record, next_cursor, previous_cursor| # each record, one-by-one - saved_cursor = next_cursor + saved_next_cursor = next_cursor end ``` Resume iterating using the previously saved cursor. ```ruby -session[:feed_items].find.sort(position: -1).limit(5).scroll(saved_cursor, { field_type: DateTime }) do |record, next_cursor| +session[:feed_items].find.sort(position: -1).limit(5).scroll(saved_next_cursor, { field_type: DateTime }) do |record, next_cursor, previous_cursor| # each record, one-by-one - saved_cursor = next_cursor + saved_next_cursor = next_cursor end ``` diff --git a/lib/mongo/scrollable.rb b/lib/mongo/scrollable.rb index 39b865b..144abd1 100644 --- a/lib/mongo/scrollable.rb +++ b/lib/mongo/scrollable.rb @@ -18,7 +18,7 @@ def scroll(cursor_or_type = nil, options = nil, &_block) raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options) records = nil - if cursor.previous && limit + if cursor.type == :previous && limit # scroll backwards by reversing the sort order, limit and then reverse again pipeline = [ { '$match' => view.selector.merge(cursor.criteria) }, @@ -42,7 +42,7 @@ def scroll(cursor_or_type = nil, options = nil, &_block) if block_given? previous_cursor = nil records.each do |record| - previous_cursor ||= cursor_type.from_record(record, cursor_options.merge(previous: true)) + previous_cursor ||= cursor_type.from_record(record, cursor_options.merge(type: :previous)) yield record, cursor_type.from_record(record, cursor_options), previous_cursor end else diff --git a/lib/mongoid/criteria/scrollable.rb b/lib/mongoid/criteria/scrollable.rb index 8bf5ade..8ee72c5 100644 --- a/lib/mongoid/criteria/scrollable.rb +++ b/lib/mongoid/criteria/scrollable.rb @@ -16,7 +16,7 @@ def scroll(cursor_or_type = nil, &_block) if block_given? previous_cursor = nil records.each do |record| - previous_cursor ||= cursor_from_record(cursor_type, record, cursor_options.merge(previous: true)) + previous_cursor ||= cursor_from_record(cursor_type, record, cursor_options.merge(type: :previous)) yield record, cursor_from_record(cursor_type, record, cursor_options), previous_cursor end else @@ -65,7 +65,7 @@ def new_cursor(cursor_type, cursor, cursor_options) def find_records(criteria, cursor) cursor_criteria = criteria.dup cursor_criteria.selector = { '$and' => [criteria.selector, cursor.criteria] } - if cursor.previous && criteria.options[:limit] + if cursor.type == :previous && criteria.options[:limit] pipeline = [ { '$match' => cursor_criteria.selector }, { '$sort' => { cursor.field_name => -cursor.direction } }, diff --git a/lib/mongoid/scroll/base64_encoded_cursor.rb b/lib/mongoid/scroll/base64_encoded_cursor.rb index a58a564..28c5d3b 100644 --- a/lib/mongoid/scroll/base64_encoded_cursor.rb +++ b/lib/mongoid/scroll/base64_encoded_cursor.rb @@ -19,7 +19,7 @@ def initialize(value, options = {}) direction: parsed['direction'], include_current: parsed['include_current'], tiebreak_id: parsed['tiebreak_id'] && !parsed['tiebreak_id'].empty? ? BSON::ObjectId.from_string(parsed['tiebreak_id']) : nil, - previous: parsed['previous'] + type: parsed['type'].try(:to_sym) } else super nil, options @@ -34,7 +34,7 @@ def to_s direction: direction, include_current: include_current, tiebreak_id: tiebreak_id && tiebreak_id.to_s, - previous: previous + type: type }.to_json) end end diff --git a/lib/mongoid/scroll/base_cursor.rb b/lib/mongoid/scroll/base_cursor.rb index 41ac877..9fc2267 100644 --- a/lib/mongoid/scroll/base_cursor.rb +++ b/lib/mongoid/scroll/base_cursor.rb @@ -1,7 +1,7 @@ module Mongoid module Scroll class BaseCursor - attr_accessor :value, :tiebreak_id, :field_type, :field_name, :direction, :include_current, :previous + attr_accessor :value, :tiebreak_id, :field_type, :field_name, :direction, :include_current, :type def initialize(value, options = {}) @value = value @@ -10,7 +10,7 @@ def initialize(value, options = {}) @field_name = options[:field_name] @direction = options[:direction] || 1 @include_current = options[:include_current] || false - @previous = options[:previous] || false + @type = options[:type] || :next end def criteria @@ -88,7 +88,7 @@ def extract_field_options(options) field_name: field_name.to_s, direction: options[:direction] || 1, include_current: options[:include_current] || false, - previous: options[:previous] || false + type: options[:type].try(:to_sym) || :next } elsif options && (field = options[:field]) { @@ -96,13 +96,13 @@ def extract_field_options(options) field_name: field.name.to_s, direction: options[:direction] || 1, include_current: options[:include_current] || false, - previous: options[:previous] || false + type: options[:type].try(:to_sym) || :next } end end def compare_direction - dir = previous ? -direction : direction + dir = type == :previous ? -direction : direction dir == 1 ? '$gt' : '$lt' end diff --git a/spec/mongoid/base64_encoded_cursor_spec.rb b/spec/mongoid/base64_encoded_cursor_spec.rb index 8d65484..a373130 100644 --- a/spec/mongoid/base64_encoded_cursor_spec.rb +++ b/spec/mongoid/base64_encoded_cursor_spec.rb @@ -3,18 +3,18 @@ describe Mongoid::Scroll::Base64EncodedCursor do context 'new' do context 'an empty cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6bnVsbCwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOm51bGwsInByZXZpb3VzIjpmYWxzZX0=' } + let(:base64_string) { 'eyJ2YWx1ZSI6bnVsbCwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOm51bGwsInR5cGUiOiJuZXh0In0=' } subject do Mongoid::Scroll::Base64EncodedCursor.new base64_string end its(:tiebreak_id) { should be_nil } its(:value) { should be_nil } its(:criteria) { should eq({}) } - its(:previous) { should be_falsy } + its(:type) { should eq(:next) } its(:to_s) { should eq(base64_string) } end context 'a string field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6ImEgc3RyaW5nIiwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2M2RmODA5NDQzNDE3YzdkMmIxMDIiLCJwcmV2aW91cyI6ZmFsc2V9' } + let(:base64_string) { 'eyJ2YWx1ZSI6ImEgc3RyaW5nIiwiZmllbGRfdHlwZSI6IlN0cmluZyIsImZpZWxkX25hbWUiOiJhX3N0cmluZyIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2M2RmODA5NDQzNDE3YzdkMmIxMDIiLCJ0eXBlIjoibmV4dCJ9' } let(:a_value) { 'a string' } let(:tiebreak_id) { BSON::ObjectId.from_string('64063df809443417c7d2b102') } let(:criteria) do @@ -33,11 +33,11 @@ its(:value) { should eq a_value } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } - its(:previous) { should be_falsy } + its(:type) { should eq(:next) } its(:to_s) { should eq(base64_string) } end context 'an id field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6IjY0MDY0NTg0MDk0NDM0MjgxZmE3MWFiMiIsImZpZWxkX3R5cGUiOiJCU09OOjpPYmplY3RJZCIsImZpZWxkX25hbWUiOiJpZCIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2NDU4NDA5NDQzNDI4MWZhNzFhYjIiLCJwcmV2aW91cyI6ZmFsc2V9' } + let(:base64_string) { 'eyJ2YWx1ZSI6IjY0MDY0NTg0MDk0NDM0MjgxZmE3MWFiMiIsImZpZWxkX3R5cGUiOiJCU09OOjpPYmplY3RJZCIsImZpZWxkX25hbWUiOiJpZCIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2NDU4NDA5NDQzNDI4MWZhNzFhYjIiLCJ0eXBlIjoibmV4dCJ9' } let(:a_value) { BSON::ObjectId('64064584094434281fa71ab2') } let(:tiebreak_id) { a_value } let(:criteria) do @@ -54,11 +54,11 @@ its(:value) { should eq a_value } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } - its(:previous) { should be_falsy } + its(:type) { should eq(:next) } its(:to_s) { should eq(base64_string) } end context 'an integer field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6MTAsImZpZWxkX3R5cGUiOiJJbnRlZ2VyIiwiZmllbGRfbmFtZSI6ImFfaW50ZWdlciIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2M2RmODA5NDQzNDE3YzdkMmIxMDgiLCJwcmV2aW91cyI6ZmFsc2V9' } + let(:base64_string) { 'eyJ2YWx1ZSI6MTAsImZpZWxkX3R5cGUiOiJJbnRlZ2VyIiwiZmllbGRfbmFtZSI6ImFfaW50ZWdlciIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2M2RmODA5NDQzNDE3YzdkMmIxMDgiLCJ0eXBlIjoibmV4dCJ9' } let(:a_value) { 10 } let(:tiebreak_id) { BSON::ObjectId('64063df809443417c7d2b108') } let(:criteria) do @@ -77,11 +77,11 @@ its(:value) { should eq a_value } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } - its(:previous) { should be_falsy } + its(:type) { should eq(:next) } its(:to_s) { should eq(base64_string) } end context 'a date/time field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzU5MDEyMy4wLCJmaWVsZF90eXBlIjoiRGF0ZVRpbWUiLCJmaWVsZF9uYW1lIjoiYV9kYXRldGltZSIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2NDNhNzA5NDQzNDIzOWYyZGJmODYiLCJwcmV2aW91cyI6ZmFsc2V9' } + let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzU5MDEyMy4wLCJmaWVsZF90eXBlIjoiRGF0ZVRpbWUiLCJmaWVsZF9uYW1lIjoiYV9kYXRldGltZSIsImRpcmVjdGlvbiI6MSwiaW5jbHVkZV9jdXJyZW50IjpmYWxzZSwidGllYnJlYWtfaWQiOiI2NDA2NDNhNzA5NDQzNDIzOWYyZGJmODYiLCJ0eXBlIjoibmV4dCJ9' } let(:a_value) { DateTime.new(2013, 12, 21, 1, 42, 3, 'UTC') } let(:tiebreak_id) { BSON::ObjectId('640643a7094434239f2dbf86') } let(:criteria) do @@ -98,11 +98,11 @@ its(:value) { should eq a_value } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } - its(:previous) { should be_falsy } + its(:type) { should eq(:next) } its(:to_s) { should eq(base64_string) } end context 'a date field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzU4NDAwMCwiZmllbGRfdHlwZSI6IkRhdGUiLCJmaWVsZF9uYW1lIjoiYV9kYXRlIiwiZGlyZWN0aW9uIjoxLCJpbmNsdWRlX2N1cnJlbnQiOmZhbHNlLCJ0aWVicmVha19pZCI6IjY0MDY0MmM5MDk0NDM0MjEyYzRkNDQyMCIsInByZXZpb3VzIjpmYWxzZX0=' } + let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzU4NDAwMCwiZmllbGRfdHlwZSI6IkRhdGUiLCJmaWVsZF9uYW1lIjoiYV9kYXRlIiwiZGlyZWN0aW9uIjoxLCJpbmNsdWRlX2N1cnJlbnQiOmZhbHNlLCJ0aWVicmVha19pZCI6IjY0MDY0MmM5MDk0NDM0MjEyYzRkNDQyMCIsInR5cGUiOiJuZXh0In0=' } let(:tiebreak_id) { BSON::ObjectId('640642c9094434212c4d4420') } let(:a_value) { Date.new(2013, 12, 21) } let(:criteria) do @@ -119,11 +119,11 @@ its(:value) { should eq a_value } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } - its(:previous) { should be_falsy } + its(:type) { should eq(:next) } its(:to_s) { should eq(base64_string) } end context 'a time field cursor' do - let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzYwNTcyMy4wLCJmaWVsZF90eXBlIjoiVGltZSIsImZpZWxkX25hbWUiOiJhX3RpbWUiLCJkaXJlY3Rpb24iOjEsImluY2x1ZGVfY3VycmVudCI6ZmFsc2UsInRpZWJyZWFrX2lkIjoiNjQwNjNkNGEwOTQ0MzQxNjZiZDA1M2VkIiwicHJldmlvdXMiOmZhbHNlfQ==' } + let(:base64_string) { 'eyJ2YWx1ZSI6MTM4NzYwNTcyMy4wLCJmaWVsZF90eXBlIjoiVGltZSIsImZpZWxkX25hbWUiOiJhX3RpbWUiLCJkaXJlY3Rpb24iOjEsImluY2x1ZGVfY3VycmVudCI6ZmFsc2UsInRpZWJyZWFrX2lkIjoiNjQwNjNkNGEwOTQ0MzQxNjZiZDA1M2VkIiwidHlwZSI6Im5leHQifQ==' } let(:item_id) { BSON::ObjectId('640636f209443407333b46d4') } let(:a_value) { Time.new(2013, 12, 21, 6, 2, 3, '+00:00').utc } let(:tiebreak_id) { BSON::ObjectId('64063d4a094434166bd053ed') } @@ -142,7 +142,7 @@ its(:tiebreak_id) { tiebreak_id } its(:tiebreak_id) { should eq tiebreak_id } its(:criteria) { should eq(criteria) } - its(:previous) { should be_falsy } + its(:type) { should eq(:next) } its(:to_s) { should eq(base64_string) } end context 'an invalid field cursor' do @@ -237,21 +237,21 @@ end end - it 'encode and decode previous option' do + it 'encode and decode type option' do feed_item = Feed::Item.create! - cursor = Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: 'id', field_type: BSON::ObjectId, previous: true - expect(Mongoid::Scroll::Base64EncodedCursor.new(cursor.to_s).previous).to be_truthy + cursor = Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: 'id', field_type: BSON::ObjectId, type: :previous + expect(Mongoid::Scroll::Base64EncodedCursor.new(cursor.to_s).type).to eq(:previous) end context 'a cursor with previous set to true' do let(:field_type) { BSON::ObjectId } let(:field_name) { 'id' } let(:feed_item) { Feed::Item.create! } subject do - Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: field_name, field_type: field_type, previous: true + Mongoid::Scroll::Base64EncodedCursor.from_record feed_item, field_name: field_name, field_type: field_type, type: :previous end its(:value) { should eq feed_item._id } its(:field_type) { should eq field_type.to_s } - its(:previous) { should be_truthy } + its(:type) { should eq(:previous) } end end end From 0a76df82cbcc1861b0d6bdbcc38409b1a261f9c3 Mon Sep 17 00:00:00 2001 From: GCorbel Date: Tue, 27 Aug 2024 18:03:51 -0400 Subject: [PATCH 12/15] change to use an iterator object --- CHANGELOG.md | 2 +- README.md | 32 ++++----- UPGRADING.md | 22 +++++- lib/mongo/scrollable.rb | 6 +- lib/mongoid-scroll.rb | 1 + lib/mongoid/criteria/scrollable.rb | 6 +- lib/mongoid/criteria/scrollable/iterator.rb | 14 ++++ lib/mongoid/scroll/version.rb | 2 +- spec/mongo/collection_view_spec.rb | 34 +++++----- spec/mongoid/criteria_spec.rb | 74 ++++++++++----------- 10 files changed, 118 insertions(+), 75 deletions(-) create mode 100644 lib/mongoid/criteria/scrollable/iterator.rb diff --git a/CHANGELOG.md b/CHANGELOG.md index 129346b..94006e5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,4 @@ -### 1.1.0 (Next) +### 2.0.0 (Next) * [#38](https://github.com/mongoid/mongoid-scroll/pull/38): Allow to reverse the scroll - [@GCorbel](https://github.com/GCorbel). * Your contribution here. diff --git a/README.md b/README.md index 272cc18..d6aeee5 100644 --- a/README.md +++ b/README.md @@ -69,38 +69,38 @@ end Scroll by `:position` and save a cursor to the last item. ```ruby -saved_previous_cursor = nil -saved_next_cursor = nil +saved_iterator = nil -Feed::Item.desc(:position).limit(5).scroll do |record, next_cursor, previous_cursor| +Feed::Item.desc(:position).limit(5).scroll do |record, iterator| # each record, one-by-one - saved_next_cursor = next_cursor + saved_iterator = iterator end ``` Resume iterating using saved cursor and save the cursor to go backward. ```ruby -Feed::Item.desc(:position).limit(5).scroll(saved_next_cursor) do |record, next_cursor, previous_cursor| +Feed::Item.desc(:position).limit(5).scroll(saved_iterator.next_cursor) do |record, iterator| # each record, one-by-one - saved_previous_cursor = previous_previous_cursor + saved_iterator = iterator end ``` Loop over the first records again. ```ruby -Feed::Item.desc(:position).limit(5).scroll(saved_previous_cursor) do |record, next_cursor, previous_cursor| +Feed::Item.desc(:position).limit(5).scroll(saved_previous_cursor) do |record, iterator| # each record, one-by-one - saved_next_cursor = next_cursor + saved_iterator = iterator end ``` The iteration finishes when no more records are available. You can also finish iterating over the remaining records by omitting the query limit. ```ruby -Feed::Item.desc(:position).limit(5).scroll(saved_next_cursor) do |record, next_cursor, previous_cursor| +Feed::Item.desc(:position).limit(5).scroll(saved_iterator.next_cursor) do |record, iterator| # each record, one-by-one + saved_iterator = iterator end ``` @@ -109,19 +109,19 @@ end Scroll a `Mongo::Collection::View` and save a cursor to the last item. You must also supply a `field_type` of the sort criteria. ```ruby -saved_next_cursor = nil -client[:feed_items].find.sort(position: -1).limit(5).scroll(nil, { field_type: DateTime }) do |record, next_cursor, previous_cursor| +saved_iterator = nil +client[:feed_items].find.sort(position: -1).limit(5).scroll(nil, { field_type: DateTime }) do |record, iterator| # each record, one-by-one - saved_next_cursor = next_cursor + saved_iterator = iterator end ``` Resume iterating using the previously saved cursor. ```ruby -session[:feed_items].find.sort(position: -1).limit(5).scroll(saved_next_cursor, { field_type: DateTime }) do |record, next_cursor, previous_cursor| +session[:feed_items].find.sort(position: -1).limit(5).scroll(iterator.next_cursor, { field_type: DateTime }) do |record, iterator| # each record, one-by-one - saved_next_cursor = next_cursor + saved_iterator = iterator end ``` @@ -197,8 +197,8 @@ The `Mongoid::Scroll::Cursor` encodes a value and a tiebreak ID separated by `:` The `Mongoid::Scroll::Base64EncodedCursor` can be used instead of `Mongoid::Scroll::Cursor` to generate a base64-encoded string (using RFC 4648) containing all the information needed to rebuild a cursor. ```ruby -Feed::Item.desc(:position).limit(5).scroll(Mongoid::Scroll::Base64EncodedCursor) do |record, next_cursor| - # next_cursor is of type Mongoid::Scroll::Base64EncodedCursor +Feed::Item.desc(:position).limit(5).scroll(Mongoid::Scroll::Base64EncodedCursor) do |record, iterator| + # iterator.next_cursor is of type Mongoid::Scroll::Base64EncodedCursor end ``` diff --git a/UPGRADING.md b/UPGRADING.md index 67334d4..83e92da 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -1,5 +1,25 @@ # Upgrading +## Upgrading to >= 2.0.0 + +The second argument passed to the block in `Mongoid::Criteria::Scrollable#scroll` and `Mongo::Scrollable#scroll` has changed. It is now an instance of `Mongoid::Criteria::Scrollable` that provides two methods: `next_cursor` and `previous_cursor`. The `next_cursor` method returns the same cursor as in versions prior to 2.0.0. + +For example, this code: + +```ruby +Feed::Item.asc(field_name).limit(2).scroll(cursor) do |_, next_cursor| + cursor = next_cursor +end +``` + +Should be updated to: + +``` +Feed::Item.asc(field_name).limit(2).scroll(cursor) do |_, iterator| + cursor = iterator.next_cursor +end +``` + ## Upgrading to >= 1.0.0 ### Mismatched Sort Fields @@ -9,6 +29,6 @@ Both `Mongoid::Criteria::Scrollable#scroll` and `Mongo::Scrollable` now raise a For example, the following code will now raise a `MismatchedSortFieldsError` because we set a different field name (`position`) from the `created_at` field used to sort in `scroll`. ```ruby -cursor.field_name = "position" +cursor.field_name = "position" Feed::Item.desc(:created_at).scroll(cursor) ``` diff --git a/lib/mongo/scrollable.rb b/lib/mongo/scrollable.rb index 144abd1..48d6d91 100644 --- a/lib/mongo/scrollable.rb +++ b/lib/mongo/scrollable.rb @@ -43,7 +43,11 @@ def scroll(cursor_or_type = nil, options = nil, &_block) previous_cursor = nil records.each do |record| previous_cursor ||= cursor_type.from_record(record, cursor_options.merge(type: :previous)) - yield record, cursor_type.from_record(record, cursor_options), previous_cursor + iterator = Mongoid::Criteria::Scrollable::Iterator.new( + previous_cursor: previous_cursor, + next_cursor: cursor_type.from_record(record, cursor_options) + ) + yield record, iterator end else records diff --git a/lib/mongoid-scroll.rb b/lib/mongoid-scroll.rb index b2e7088..7671ab2 100644 --- a/lib/mongoid-scroll.rb +++ b/lib/mongoid-scroll.rb @@ -11,5 +11,6 @@ require 'mongoid/scroll/base64_encoded_cursor' require 'mongoid/criteria/scrollable/fields' require 'mongoid/criteria/scrollable/cursors' +require 'mongoid/criteria/scrollable/iterator' require 'mongo/scrollable' if Object.const_defined?(:Mongo) require 'mongoid/criteria/scrollable' diff --git a/lib/mongoid/criteria/scrollable.rb b/lib/mongoid/criteria/scrollable.rb index 8ee72c5..cab0213 100644 --- a/lib/mongoid/criteria/scrollable.rb +++ b/lib/mongoid/criteria/scrollable.rb @@ -17,7 +17,11 @@ def scroll(cursor_or_type = nil, &_block) previous_cursor = nil records.each do |record| previous_cursor ||= cursor_from_record(cursor_type, record, cursor_options.merge(type: :previous)) - yield record, cursor_from_record(cursor_type, record, cursor_options), previous_cursor + iterator = Mongoid::Criteria::Scrollable::Iterator.new( + previous_cursor: previous_cursor, + next_cursor: cursor_from_record(cursor_type, record, cursor_options) + ) + yield record, iterator end else records diff --git a/lib/mongoid/criteria/scrollable/iterator.rb b/lib/mongoid/criteria/scrollable/iterator.rb new file mode 100644 index 0000000..fb11b91 --- /dev/null +++ b/lib/mongoid/criteria/scrollable/iterator.rb @@ -0,0 +1,14 @@ +module Mongoid + class Criteria + module Scrollable + class Iterator + attr_accessor :previous_cursor, :next_cursor + + def initialize(previous_cursor:, next_cursor:) + @previous_cursor = previous_cursor + @next_cursor = next_cursor + end + end + end + end +end diff --git a/lib/mongoid/scroll/version.rb b/lib/mongoid/scroll/version.rb index 0225d54..96bfa3d 100644 --- a/lib/mongoid/scroll/version.rb +++ b/lib/mongoid/scroll/version.rb @@ -1,5 +1,5 @@ module Mongoid module Scroll - VERSION = '1.1.0'.freeze + VERSION = '2.0.0'.freeze end end diff --git a/spec/mongo/collection_view_spec.rb b/spec/mongo/collection_view_spec.rb index c20409c..27a0dd5 100644 --- a/spec/mongo/collection_view_spec.rb +++ b/spec/mongo/collection_view_spec.rb @@ -57,7 +57,7 @@ context 'default' do it 'scrolls all' do records = [] - Mongoid.default_client['feed_items'].find.scroll(cursor_type) do |record, _next_cursor| + Mongoid.default_client['feed_items'].find.scroll(cursor_type) do |record, iterator| records << record end expect(records.size).to eq 10 @@ -68,7 +68,7 @@ context field_type do it 'scrolls all with a block' do records = [] - Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor_type, field_type: field_type) do |record, _next_cursor| + Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor_type, field_type: field_type) do |record, iterator| records << record end expect(records.size).to eq 10 @@ -77,15 +77,15 @@ it 'scrolls all with a break' do records = [] cursor = nil - Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(5).scroll(cursor_type, field_type: field_type) do |record, next_cursor| + Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(5).scroll(cursor_type, field_type: field_type) do |record, iterator| records << record - cursor = next_cursor + cursor = iterator.next_cursor expect(cursor).to be_a cursor_type end expect(records.size).to eq 5 - Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor, field_type: field_type) do |record, next_cursor| + Mongoid.default_client['feed_items'].find.sort(field_name => 1).scroll(cursor, field_type: field_type) do |record, iterator| records << record - cursor = next_cursor + cursor = iterator.next_cursor expect(cursor).to be_a cursor_type end expect(records.size).to eq 10 @@ -93,7 +93,7 @@ end it 'scrolls in descending order' do records = [] - Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).scroll(cursor_type, field_type: field_type, field_name: field_name) do |record, _next_cursor| + Mongoid.default_client['feed_items'].find.sort(field_name => -1).limit(3).scroll(cursor_type, field_type: field_type, field_name: field_name) do |record, iterator| records << record end expect(records.size).to eq 3 @@ -111,17 +111,17 @@ first_previous_cursor = nil second_previous_cursor = nil - Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor_type, field_type: field_type) do |_, next_cursor| - cursor = next_cursor + Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor_type, field_type: field_type) do |_, iterator| + cursor = iterator.next_cursor end - Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor, field_type: field_type) do |_, next_cursor, previous_cursor| - cursor = next_cursor - first_previous_cursor = previous_cursor + Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor, field_type: field_type) do |_, iterator| + cursor = iterator.next_cursor + first_previous_cursor = iterator.previous_cursor end - Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor, field_type: field_type) do |_, _, previous_cursor| - second_previous_cursor = previous_cursor + Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor, field_type: field_type) do |_, iterator| + second_previous_cursor = iterator.previous_cursor end records = Mongoid.default_client['feed_items'].find.sort(field_name => 1) @@ -146,12 +146,12 @@ it "scrolls by #{sort_order}" do records = [] cursor = nil - Mongoid.default_client['feed_items'].find.sort(sort_order).limit(2).scroll(cursor_type) do |record, next_cursor| + Mongoid.default_client['feed_items'].find.sort(sort_order).limit(2).scroll(cursor_type) do |record, iterator| records << record - cursor = next_cursor + cursor = iterator.next_cursor end expect(records.size).to eq 2 - Mongoid.default_client['feed_items'].find.sort(sort_order).scroll(cursor) do |record, _next_cursor| + Mongoid.default_client['feed_items'].find.sort(sort_order).scroll(cursor) do |record, iterator| records << record end expect(records.size).to eq 3 diff --git a/spec/mongoid/criteria_spec.rb b/spec/mongoid/criteria_spec.rb index 1c1901c..512ef7d 100644 --- a/spec/mongoid/criteria_spec.rb +++ b/spec/mongoid/criteria_spec.rb @@ -51,7 +51,7 @@ context 'default' do it 'scrolls all' do records = [] - Feed::Item.all.scroll(cursor_type) do |record, _next_cursor| + Feed::Item.all.scroll(cursor_type) do |record, _iterator| records << record end expect(records.size).to eq 10 @@ -63,11 +63,11 @@ criteria.limit(2).scroll(cursor_type) expect(criteria).to eq original_criteria cursor = nil - criteria.limit(2).scroll(cursor) do |_record, next_cursor| - cursor = next_cursor + criteria.limit(2).scroll(cursor) do |_record, iterator| + cursor = iterator.next_cursor end - criteria.scroll(cursor) do |_record, next_cursor| - cursor = next_cursor + criteria.scroll(cursor) do |_record, iterator| + cursor = iterator.next_cursor end expect(criteria).to eq original_criteria end @@ -83,7 +83,7 @@ context field_type do it 'scrolls all with a block' do records = [] - Feed::Item.asc(field_name).scroll(cursor_type) do |record, _next_cursor| + Feed::Item.asc(field_name).scroll(cursor_type) do |record, iterator| records << record end expect(records.size).to eq 10 @@ -92,14 +92,14 @@ it 'scrolls all with a break' do records = [] cursor = nil - Feed::Item.asc(field_name).limit(5).scroll(cursor_type) do |record, next_cursor| + Feed::Item.asc(field_name).limit(5).scroll(cursor_type) do |record, iterator| records << record - cursor = next_cursor + cursor = iterator.next_cursor end expect(records.size).to eq 5 - Feed::Item.asc(field_name).scroll(cursor) do |record, next_cursor| + Feed::Item.asc(field_name).scroll(cursor) do |record, iterator| records << record - cursor = next_cursor + cursor = iterator.next_cursor end expect(records.size).to eq 10 expect(records).to eq Feed::Item.all.to_a @@ -107,9 +107,9 @@ it 'scrolls from a cursor' do last_record = nil cursor = nil - Feed::Item.asc(field_name).limit(5).scroll(cursor_type) do |record, next_cursor| + Feed::Item.asc(field_name).limit(5).scroll(cursor_type) do |record, iterator| last_record = record - cursor = next_cursor + cursor = iterator.next_cursor end sixth_item = Feed::Item.asc(field_name).to_a[5] from_item = Feed::Item.asc(field_name).scroll(cursor).to_a.first @@ -118,9 +118,9 @@ it 'includes the current record when Mongoid::Scroll::Cursor#include_current is true' do last_record = nil cursor = nil - Feed::Item.asc(field_name).limit(5).scroll(cursor_type) do |record, next_cursor| + Feed::Item.asc(field_name).limit(5).scroll(cursor_type) do |record, iterator| last_record = record - cursor = next_cursor + cursor = iterator.next_cursor end fifth_item = last_record cursor.include_current = true @@ -129,7 +129,7 @@ end it 'scrolls in descending order' do records = [] - Feed::Item.desc(field_name).limit(3).scroll(cursor_type) do |record, _next_cursor| + Feed::Item.desc(field_name).limit(3).scroll(cursor_type) do |record, _iterator| records << record end expect(records.size).to eq 3 @@ -145,16 +145,16 @@ end it 'can be reused' do ids = Feed::Item.asc(field_name).limit(2).map(&:id) - Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, cursor| - cursor.include_current = true - expect(Feed::Item.asc(field_name).limit(2).scroll(cursor).pluck(:id)).to eq ids + Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, iterator| + iterator.next_cursor.include_current = true + expect(Feed::Item.asc(field_name).limit(2).scroll(iterator.next_cursor).pluck(:id)).to eq ids break end end it 'can be re-created and reused' do ids = Feed::Item.asc(field_name).limit(2).map(&:id) - Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, cursor| - new_cursor = cursor_type.new(cursor.to_s, field_type: field_type, field_name: field_name) + Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, iterator| + new_cursor = cursor_type.new(iterator.next_cursor.to_s, field_type: field_type, field_name: field_name) new_cursor.include_current = true expect(Feed::Item.asc(field_name).limit(2).scroll(new_cursor).pluck(:id)).to eq ids break @@ -165,17 +165,17 @@ first_previous_cursor = nil second_previous_cursor = nil - Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, next_cursor| - cursor = next_cursor + Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, iterator| + cursor = iterator.next_cursor end - Feed::Item.asc(field_name).limit(2).scroll(cursor) do |_, next_cursor, previous_cursor| - cursor = next_cursor - first_previous_cursor = previous_cursor + Feed::Item.asc(field_name).limit(2).scroll(cursor) do |_, iterator| + cursor = iterator.next_cursor + first_previous_cursor = iterator.previous_cursor end - Feed::Item.asc(field_name).limit(2).scroll(cursor) do |_, _, previous_cursor| - second_previous_cursor = previous_cursor + Feed::Item.asc(field_name).limit(2).scroll(cursor) do |_, iterator| + second_previous_cursor = iterator.previous_cursor end records = Feed::Item.asc(field_name) @@ -212,16 +212,16 @@ ).asc(:a_time) records = [] cursor = nil - criteria.limit(2).scroll(cursor_type) do |record, next_cursor| + criteria.limit(2).scroll(cursor_type) do |record, iterator| records << record - cursor = next_cursor + cursor = iterator.next_cursor end expect(records.size).to eq 2 expect(records.map(&:name)).to eq ['Feed Item 0', 'Feed Item 1'] records = [] - criteria.limit(2).scroll(cursor) do |record, next_cursor| + criteria.limit(2).scroll(cursor) do |record, iterator| records << record - cursor = next_cursor + cursor = iterator.next_cursor end expect(records.size).to eq 1 expect(records.map(&:name)).to eq ['Feed Item 2'] @@ -235,7 +235,7 @@ it 'respects embedded queries' do records = [] criteria = @item.embedded_items.limit(2) - criteria.scroll(cursor_type) do |record, _next_cursor| + criteria.scroll(cursor_type) do |record, _iterator| records << record end expect(records.size).to eq 1 @@ -257,12 +257,12 @@ it "scrolls by #{sort_order}" do records = [] cursor = nil - Feed::Item.order_by(sort_order).limit(2).scroll(cursor_type) do |record, next_cursor| + Feed::Item.order_by(sort_order).limit(2).scroll(cursor_type) do |record, iterator| records << record - cursor = next_cursor + cursor = iterator.next_cursor end expect(records.size).to eq 2 - Feed::Item.order_by(sort_order).scroll(cursor) do |record, _next_cursor| + Feed::Item.order_by(sort_order).scroll(cursor) do |record, _iterator| records << record end expect(records.size).to eq 3 @@ -290,9 +290,9 @@ it 'doesn\'t lose the precision when rebuilding the cursor' do records = [] cursor = nil - Feed::Item.order_by(a_datetime: 1).limit(2).scroll(cursor_type) do |record, next_cursor| + Feed::Item.order_by(a_datetime: 1).limit(2).scroll(cursor_type) do |record, iterator| records << record - cursor = next_cursor + cursor = iterator.next_cursor end expect(records).to eq [item_1, item_2] cursor = cursor_type.new(cursor.to_s, field: Feed::Item.fields['a_datetime']) From 9c20b524cb9c352d6be8ae475ca2a66f5b06e340 Mon Sep 17 00:00:00 2001 From: GCorbel Date: Tue, 27 Aug 2024 18:31:09 -0400 Subject: [PATCH 13/15] refactor --- lib/mongo/scrollable.rb | 2 +- lib/mongoid/criteria/scrollable.rb | 2 +- spec/mongo/collection_view_spec.rb | 21 ++++++++++----------- spec/mongoid/criteria_spec.rb | 21 ++++++++++----------- 4 files changed, 22 insertions(+), 24 deletions(-) diff --git a/lib/mongo/scrollable.rb b/lib/mongo/scrollable.rb index 48d6d91..1f40b59 100644 --- a/lib/mongo/scrollable.rb +++ b/lib/mongo/scrollable.rb @@ -18,7 +18,7 @@ def scroll(cursor_or_type = nil, options = nil, &_block) raise_mismatched_sort_fields_error!(cursor, cursor_options) if different_sort_fields?(cursor, cursor_options) records = nil - if cursor.type == :previous && limit + if cursor.type == :previous # scroll backwards by reversing the sort order, limit and then reverse again pipeline = [ { '$match' => view.selector.merge(cursor.criteria) }, diff --git a/lib/mongoid/criteria/scrollable.rb b/lib/mongoid/criteria/scrollable.rb index cab0213..3e89327 100644 --- a/lib/mongoid/criteria/scrollable.rb +++ b/lib/mongoid/criteria/scrollable.rb @@ -69,7 +69,7 @@ def new_cursor(cursor_type, cursor, cursor_options) def find_records(criteria, cursor) cursor_criteria = criteria.dup cursor_criteria.selector = { '$and' => [criteria.selector, cursor.criteria] } - if cursor.type == :previous && criteria.options[:limit] + if cursor.type == :previous pipeline = [ { '$match' => cursor_criteria.selector }, { '$sort' => { cursor.field_name => -cursor.direction } }, diff --git a/spec/mongo/collection_view_spec.rb b/spec/mongo/collection_view_spec.rb index 27a0dd5..2f329e3 100644 --- a/spec/mongo/collection_view_spec.rb +++ b/spec/mongo/collection_view_spec.rb @@ -107,26 +107,25 @@ expect(cursor.tiebreak_id).to eq record['_id'] end it 'can scroll back with the previous cursor' do - cursor = nil - first_previous_cursor = nil - second_previous_cursor = nil + first_iterator = nil + second_iterator = nil + third_iterator = nil Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor_type, field_type: field_type) do |_, iterator| - cursor = iterator.next_cursor + first_iterator = iterator end - Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor, field_type: field_type) do |_, iterator| - cursor = iterator.next_cursor - first_previous_cursor = iterator.previous_cursor + Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(first_iterator.next_cursor, field_type: field_type) do |_, iterator| + second_iterator = iterator end - Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(cursor, field_type: field_type) do |_, iterator| - second_previous_cursor = iterator.previous_cursor + Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(second_iterator.next_cursor, field_type: field_type) do |_, iterator| + third_iterator = iterator end records = Mongoid.default_client['feed_items'].find.sort(field_name => 1) - expect(Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(first_previous_cursor, field_type: field_type).to_a).to eq(records.limit(2).to_a) - expect(Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(second_previous_cursor, field_type: field_type).to_a).to eq(records.skip(2).limit(2).to_a) + expect(Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(second_iterator.previous_cursor, field_type: field_type).to_a).to eq(records.limit(2).to_a) + expect(Mongoid.default_client['feed_items'].find.sort(field_name => 1).limit(2).scroll(third_iterator.previous_cursor, field_type: field_type).to_a).to eq(records.skip(2).limit(2).to_a) end end end diff --git a/spec/mongoid/criteria_spec.rb b/spec/mongoid/criteria_spec.rb index 512ef7d..26e5cd4 100644 --- a/spec/mongoid/criteria_spec.rb +++ b/spec/mongoid/criteria_spec.rb @@ -161,26 +161,25 @@ end end it 'can scroll back with the previous cursor' do - cursor = nil - first_previous_cursor = nil - second_previous_cursor = nil + first_iterator = nil + second_iterator = nil + third_iterator = nil Feed::Item.asc(field_name).limit(2).scroll(cursor_type) do |_, iterator| - cursor = iterator.next_cursor + first_iterator = iterator end - Feed::Item.asc(field_name).limit(2).scroll(cursor) do |_, iterator| - cursor = iterator.next_cursor - first_previous_cursor = iterator.previous_cursor + Feed::Item.asc(field_name).limit(2).scroll(first_iterator.next_cursor) do |_, iterator| + second_iterator = iterator end - Feed::Item.asc(field_name).limit(2).scroll(cursor) do |_, iterator| - second_previous_cursor = iterator.previous_cursor + Feed::Item.asc(field_name).limit(2).scroll(second_iterator.next_cursor) do |_, iterator| + third_iterator = iterator end records = Feed::Item.asc(field_name) - expect(Feed::Item.asc(field_name).limit(2).scroll(first_previous_cursor)).to eq(records.limit(2)) - expect(Feed::Item.asc(field_name).limit(2).scroll(second_previous_cursor)).to eq(records.skip(2).limit(2)) + expect(Feed::Item.asc(field_name).limit(2).scroll(second_iterator.previous_cursor)).to eq(records.limit(2)) + expect(Feed::Item.asc(field_name).limit(2).scroll(third_iterator.previous_cursor)).to eq(records.skip(2).limit(2)) end end end From aa019aa9ec1121b3b7ed49fea0e891b7a870288b Mon Sep 17 00:00:00 2001 From: Guirec Corbel Date: Thu, 29 Aug 2024 09:55:48 -0400 Subject: [PATCH 14/15] Fix typos Co-authored-by: Daniel (dB.) Doubrovkine --- README.md | 4 ++-- UPGRADING.md | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index d6aeee5..6657577 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ end Loop over the first records again. ```ruby -Feed::Item.desc(:position).limit(5).scroll(saved_previous_cursor) do |record, iterator| +Feed::Item.desc(:position).limit(5).scroll(saved_iterator.previous_cursor) do |record, iterator| # each record, one-by-one saved_iterator = iterator end @@ -119,7 +119,7 @@ end Resume iterating using the previously saved cursor. ```ruby -session[:feed_items].find.sort(position: -1).limit(5).scroll(iterator.next_cursor, { field_type: DateTime }) do |record, iterator| +session[:feed_items].find.sort(position: -1).limit(5).scroll(saved_iterator.next_cursor, { field_type: DateTime }) do |record, iterator| # each record, one-by-one saved_iterator = iterator end diff --git a/UPGRADING.md b/UPGRADING.md index 83e92da..286f7fc 100644 --- a/UPGRADING.md +++ b/UPGRADING.md @@ -2,7 +2,7 @@ ## Upgrading to >= 2.0.0 -The second argument passed to the block in `Mongoid::Criteria::Scrollable#scroll` and `Mongo::Scrollable#scroll` has changed. It is now an instance of `Mongoid::Criteria::Scrollable` that provides two methods: `next_cursor` and `previous_cursor`. The `next_cursor` method returns the same cursor as in versions prior to 2.0.0. +The second argument yielded in the block in `Mongoid::Criteria::Scrollable#scroll` and `Mongo::Scrollable#scroll` has changed from a cursor to an instance of `Mongoid::Criteria::Scrollable` which provides `next_cursor` and `previous_cursor`. The `next_cursor` method returns the same cursor as in versions prior to 2.0.0. For example, this code: From 7f13aca8e9852ca42fccfdca2cb4d878d87f3d27 Mon Sep 17 00:00:00 2001 From: GCorbel Date: Thu, 29 Aug 2024 10:29:01 -0400 Subject: [PATCH 15/15] throw an error when type is unsupported --- lib/config/locales/en.yml | 5 ++++- lib/mongoid/scroll/base_cursor.rb | 2 ++ lib/mongoid/scroll/errors.rb | 1 + lib/mongoid/scroll/errors/unsupported_type_error.rb | 11 +++++++++++ spec/mongoid/cursor_spec.rb | 8 ++++++++ 5 files changed, 26 insertions(+), 1 deletion(-) create mode 100644 lib/mongoid/scroll/errors/unsupported_type_error.rb diff --git a/lib/config/locales/en.yml b/lib/config/locales/en.yml index 99ee357..6de3698 100644 --- a/lib/config/locales/en.yml +++ b/lib/config/locales/en.yml @@ -27,4 +27,7 @@ en: message: "Unsupported field type." summary: "The type of the field '%{field}' is not supported: %{type}." resolution: "Please open a feature request in https://github.com/mongoid/mongoid-scroll." - + unsupported_type: + message: "Unsupported type." + summary: "The type supplied in the cursor is not supported: %{type}." + resolution: "The cursor type can be either ':previous' or ':next'." diff --git a/lib/mongoid/scroll/base_cursor.rb b/lib/mongoid/scroll/base_cursor.rb index 9fc2267..ce69d42 100644 --- a/lib/mongoid/scroll/base_cursor.rb +++ b/lib/mongoid/scroll/base_cursor.rb @@ -11,6 +11,8 @@ def initialize(value, options = {}) @direction = options[:direction] || 1 @include_current = options[:include_current] || false @type = options[:type] || :next + + raise Mongoid::Scroll::Errors::UnsupportedTypeError.new(type: @type) if ![:previous, :next].include?(@type) end def criteria diff --git a/lib/mongoid/scroll/errors.rb b/lib/mongoid/scroll/errors.rb index 2b20e4d..ecb685a 100644 --- a/lib/mongoid/scroll/errors.rb +++ b/lib/mongoid/scroll/errors.rb @@ -6,3 +6,4 @@ require 'mongoid/scroll/errors/invalid_base64_cursor_error' require 'mongoid/scroll/errors/no_such_field_error' require 'mongoid/scroll/errors/unsupported_field_type_error' +require 'mongoid/scroll/errors/unsupported_type_error' diff --git a/lib/mongoid/scroll/errors/unsupported_type_error.rb b/lib/mongoid/scroll/errors/unsupported_type_error.rb new file mode 100644 index 0000000..3743e69 --- /dev/null +++ b/lib/mongoid/scroll/errors/unsupported_type_error.rb @@ -0,0 +1,11 @@ +module Mongoid + module Scroll + module Errors + class UnsupportedTypeError < Mongoid::Scroll::Errors::Base + def initialize(opts = {}) + super(compose_message('unsupported_type', opts)) + end + end + end + end +end diff --git a/spec/mongoid/cursor_spec.rb b/spec/mongoid/cursor_spec.rb index d50aaa7..4e59771 100644 --- a/spec/mongoid/cursor_spec.rb +++ b/spec/mongoid/cursor_spec.rb @@ -131,6 +131,14 @@ end.to raise_error ArgumentError end end + context 'an invalid type cursor' do + let(:feed_item) { Feed::Item.create!(a_string: 'astring') } + it 'raises Mongoid::Scroll::Errors::UnsupportedTypeError' do + expect do + Mongoid::Scroll::Cursor.new "#{feed_item.a_string}:#{feed_item.id}", field_name: 'a_string', field_type: String, include_current: true, type: :invalid + end.to raise_error Mongoid::Scroll::Errors::UnsupportedTypeError, /The type supplied in the cursor is not supported: invalid./ + end + end context 'a cursor with include_current set to true' do let(:feed_item) { Feed::Item.create!(a_string: 'astring') } subject do