Allow paginating over nullable columns

CHANGELOG.md

@@ -1,5 +1,18 @@
 ## master (unreleased)
 
+- Allow paginating over nullable columns
+
+    Previously, the error was raised when cursor values contained `nil`s. Now, it is possible to paginate
+    over columns containing `nil` values. You need to explicitly configure which columns are nullable,
+    otherwise columns are considered as non-nullable by the gem.
+
+    ```ruby
+    paginator = users.cursor_paginate(order: [:name, :id], nullable_columns: [:name])
+    ```
+
+    Note that it is not recommended to use this feature, because the complexity of produced SQL queries can have
+    a very negative impact on the database performance. It is better to paginate using only non-nullable columns.
+
 - Fix paginating over relations with joins, includes and custom ordering
 - Add ability to incrementally configure a paginator
 

lib/activerecord_cursor_paginate/cursor.rb

@@ -7,12 +7,12 @@ module ActiveRecordCursorPaginate
   # @private
   class Cursor
     class << self
-      def from_record(record, columns:)
+      def from_record(record, columns:, nullable_columns: nil)
         values = columns.map { |column| record[column] }
-        new(columns: columns, values: values)
+        new(columns: columns, values: values, nullable_columns: nullable_columns)
       end
 
-      def decode(cursor_string:, columns:)
+      def decode(cursor_string:, columns:, nullable_columns: nil)
         decoded = JSON.parse(Base64.urlsafe_decode64(cursor_string))
 
         if (columns.size == 1 && decoded.is_a?(Array)) ||
@@ -28,7 +28,7 @@ def decode(cursor_string:, columns:)
             deserialize_time_if_needed(decoded)
           end
 
-        new(columns: columns, values: decoded)
+        new(columns: columns, values: decoded, nullable_columns: nullable_columns)
       rescue ArgumentError, JSON::ParserError # ArgumentError is raised by strict_decode64
         raise InvalidCursorError, "The given cursor `#{cursor_string}` could not be decoded"
       end
@@ -46,11 +46,16 @@ def deserialize_time_if_needed(value)
 
     attr_reader :columns, :values
 
-    def initialize(columns:, values:)
+    def initialize(columns:, values:, nullable_columns: nil)
       @columns = Array.wrap(columns)
       @values = Array.wrap(values)
+      @nullable_columns = Array.wrap(nullable_columns)
+
+      nil_index = @values.index(nil)
+      if nil_index && !@nullable_columns.include?(@columns[nil_index])
+        raise ArgumentError, "Cursor value is nil for a column that is not in the :nullable_columns list"
+      end
 
-      raise ArgumentError, "Cursor values can not be nil" if @values.any?(nil)
       raise ArgumentError, ":columns and :values have different sizes" if @columns.size != @values.size
     end
 

lib/activerecord_cursor_paginate/extension.rb

@@ -10,9 +10,8 @@ module Extension
     #   paginator = Post.cursor_paginate(limit: 2, after: "Mg")
     #   page = paginator.fetch
     #
-    def cursor_paginate(after: nil, before: nil, limit: nil, order: nil, append_primary_key: true)
-      relation = (is_a?(ActiveRecord::Relation) ? self : all)
-      Paginator.new(relation, after: after, before: before, limit: limit, order: order, append_primary_key: append_primary_key)
+    def cursor_paginate(**options)
+      Paginator.new(all, **options)
     end
     alias cursor_pagination cursor_paginate
   end

lib/activerecord_cursor_paginate/page.rb

@@ -10,11 +10,12 @@ class Page
     #
     attr_reader :records
 
-    def initialize(records, order_columns:, has_previous: false, has_next: false)
+    def initialize(records, order_columns:, has_previous: false, has_next: false, nullable_columns: nil)
       @records = records
       @order_columns = order_columns
       @has_previous = has_previous
       @has_next = has_next
+      @nullable_columns = nullable_columns
     end
 
     # Number of records in this page.
@@ -79,7 +80,7 @@ def cursors
     private
       def cursor_for_record(record)
         if record
-          cursor = Cursor.from_record(record, columns: @order_columns)
+          cursor = Cursor.from_record(record, columns: @order_columns, nullable_columns: @nullable_columns)
           cursor.encode
         end
       end

lib/activerecord_cursor_paginate/paginator.rb

@@ -40,9 +40,15 @@ class Paginator
     #   should be implicitly appended to the list of sorting columns. It may be useful
     #   to disable it for the table with a UUID primary key or when the sorting is done by a
     #   combination of columns that are already unique.
+    # @param nullable_columns [Symbol, String, nil, Array] Columns which are nullable.
+    #   By default, all columns are considered as non-nullable, if not in this list.
+    #   It is not recommended to use this feature, because the complexity of produced SQL
+    #   queries can have a very negative impact on the database performance. It is better
+    #   to paginate using only non-nullable columns.
+    #
     # @raise [ArgumentError] If any parameter is not valid
     #
-    def initialize(relation, before: nil, after: nil, limit: nil, order: nil, append_primary_key: true)
+    def initialize(relation, before: nil, after: nil, limit: nil, order: nil, append_primary_key: true, nullable_columns: nil)
       unless relation.is_a?(ActiveRecord::Relation)
         raise ArgumentError, "relation is not an ActiveRecord::Relation"
       end
@@ -51,10 +57,20 @@ def initialize(relation, before: nil, after: nil, limit: nil, order: nil, append
       @primary_key = @relation.primary_key
       @append_primary_key = append_primary_key
 
+      @cursor = @current_cursor = nil
+      @is_forward_pagination = true
+      @before = @after = nil
+      @page_size = nil
+      @limit = nil
+      @columns = []
+      @directions = []
+      @order = nil
+
       self.before = before
       self.after = after
       self.limit = limit
       self.order = order
+      self.nullable_columns = nullable_columns
     end
 
     def before=(value)
@@ -92,6 +108,21 @@ def order=(value)
       @order = value
     end
 
+    def nullable_columns=(value)
+      value = Array.wrap(value)
+      value = value.map { |column| column.is_a?(Symbol) ? column.to_s : column }
+
+      if (value - @columns).any?
+        raise ArgumentError, ":nullable_columns should include only column names from the :order option"
+      end
+
+      if value.include?(@columns.last)
+        raise ArgumentError, "Last order column can not be nullable"
+      end
+
+      @nullable_columns = value
+    end
+
     # Get the paginated result.
     # @return [ActiveRecordCursorPaginate::Page]
     #
@@ -119,7 +150,8 @@ def fetch
         records,
         order_columns: cursor_column_names,
         has_next: has_next_page,
-        has_previous: has_previous_page
+        has_previous: has_previous_page,
+        nullable_columns: @nullable_columns
       )
 
       advance_by_page(page) unless page.empty?
@@ -199,7 +231,7 @@ def build_cursor_relation(cursor)
         relation = relation.reorder(@columns.zip(pagination_directions).to_h)
 
         if cursor
-          decoded_cursor = Cursor.decode(cursor_string: cursor, columns: @columns)
+          decoded_cursor = Cursor.decode(cursor_string: cursor, columns: @columns, nullable_columns: @nullable_columns)
           relation = apply_cursor(relation, decoded_cursor)
         end
 
@@ -215,19 +247,38 @@ def cursor_column_names
       end
 
       def apply_cursor(relation, cursor)
-        operators = @directions.map { |direction| pagination_operator(direction) }
-        cursor_positions = cursor.columns.zip(cursor.values, operators)
+        cursor_positions = cursor.columns.zip(cursor.values, @directions)
 
         where_clause = nil
-        cursor_positions.reverse_each.with_index do |(column, value, operator), index|
-          where_clause =
-            if index == 0
-              arel_column(column).public_send(operator, value)
+
+        cursor_positions.reverse_each.with_index do |(column, value, direction), index|
+          previous_where_clause = where_clause
+
+          operator = pagination_operator(direction)
+          arel_column = arel_column(column)
+
+          # The last column can't be nil.
+          if index == 0
+            where_clause = arel_column.public_send(operator, value)
+          elsif value.nil?
+            if nulls_at_end?(direction)
+              # We are at the section with nulls, which is at the end ([x, x, null, null, null])
+              where_clause = arel_column.eq(nil).and(previous_where_clause)
             else
-              arel_column(column).public_send(operator, value).or(
-                arel_column(column).eq(value).and(where_clause)
-              )
+              # We are at the section with nulls, which is at the beginning ([null, null, null, x, x])
+              where_clause = arel_column.not_eq(nil)
+              where_clause = arel_column.eq(nil).and(previous_where_clause).or(where_clause)
+            end
+          else
+            where_clause = arel_column.public_send(operator, value).or(
+              arel_column.eq(value).and(previous_where_clause)
+            )
+
+            if nullable_column?(column) && nulls_at_end?(direction)
+              # Since column's value is not null, nulls can only be at the end.
+              where_clause = arel_column.eq(nil).or(where_clause)
             end
+          end
         end
 
         relation.where(where_clause)
@@ -267,5 +318,20 @@ def advance_by_page(page)
             page.previous_cursor
           end
       end
+
+      def nulls_at_end?(direction)
+        (direction == :asc && !small_nulls?) || (direction == :desc && small_nulls?)
+      end
+
+      def small_nulls?
+        # PostgreSQL considers NULLs larger than any value,
+        # opposite for SQLite and MySQL.
+        db_config = @relation.klass.connection_pool.db_config
+        db_config.adapter !~ /postg/ # postgres and postgis
+      end
+
+      def nullable_column?(column)
+        @nullable_columns.include?(column)
+      end
   end
 end

test/paginator_test.rb

@@ -36,6 +36,39 @@ def test_paginates_by_id_with_custom_conditions
     assert_equal((4..9).to_a, users.pluck(:id))
   end
 
+  def test_paginating_by_nullable_cursor_column
+    p = Project.cursor_paginate(order: [:name], nullable_columns: [:name], limit: 2)
+
+    records = []
+    p.pages.each do |page|
+      records.concat(page.records)
+    end
+
+    assert_equal Project.order(:name, :id).to_a, records
+  end
+
+  def test_paginating_by_multiple_nullable_cursor_columns_in_asc_order
+    p = Project.cursor_paginate(order: { name: :asc, organization_id: :desc }, nullable_columns: [:name, :organization_id], limit: 2)
+
+    records = []
+    p.pages.each do |page|
+      records.concat(page.records)
+    end
+
+    assert_equal Project.order(name: :asc, organization_id: :desc, id: :asc).to_a, records
+  end
+
+  def test_paginating_by_multiple_nullable_cursor_columns_in_desc_order
+    p = Project.cursor_paginate(order: { name: :desc, organization_id: :asc }, nullable_columns: [:name, :organization_id], limit: 2)
+
+    records = []
+    p.pages.each do |page|
+      records.concat(page.records)
+    end
+
+    assert_equal Project.order(name: :desc, organization_id: :asc, id: :asc).to_a, records
+  end
+
   def test_uses_default_limit
     ActiveRecordCursorPaginate.config.stub(:default_page_size, 4) do
       p = User.cursor_paginate
@@ -246,7 +279,14 @@ def test_raises_when_order_column_is_not_selected
     error = assert_raises(ArgumentError) do
       p.fetch
     end
-    assert_equal("Cursor values can not be nil", error.message)
+    assert_match(/Cursor value is nil/, error.message)
+  end
+
+  def test_raises_when_last_order_column_is_nullable
+    error = assert_raises(ArgumentError) do
+      User.cursor_paginate(order: [:created_at, :company_id], nullable_columns: [:company_id], append_primary_key: false)
+    end
+    assert_match(/Last order column can not be nullable/, error.message)
   end
 
   def test_works_with_composite_primary_keys

test/test_helper.rb

@@ -28,7 +28,9 @@
   end
 
   create_table :projects do |t|
+    t.integer :organization_id
     t.integer :user_id
+    t.string :name
     t.integer :stars
   end
 
@@ -82,10 +84,10 @@ class NoPkTable < ActiveRecord::Base
 CpkUser.insert_all!(users) if ActiveRecord.gem_version >= Gem::Version.new("7.1")
 
 projects = [
-  { id: 1, user_id: 2, stars: 5 },
-  { id: 2, user_id: 1, stars: 10 },
-  { id: 3, user_id: 1, stars: 9 },
-  { id: 4, user_id: 3, stars: 2 },
-  { id: 5, user_id: 2, stars: 6 }
+  { id: 1, user_id: 2, organization_id: 2, stars: 5, name: "Ruby on Rails" },
+  { id: 2, user_id: 1, organization_id: 1, stars: 10, name: nil },
+  { id: 3, user_id: 1, organization_id: nil, stars: 9, name: nil },
+  { id: 4, user_id: 3, organization_id: nil, stars: 2, name: "PostgreSQL" },
+  { id: 5, user_id: 2, organization_id: 2, stars: 6, name: "Linux" }
 ]
 Project.insert_all!(projects)