diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 0fbb0b79..0f6d5c59 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,10 +15,10 @@ jobs: strategy: fail-fast: false matrix: - ruby: [2.0, 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, '3.0', jruby-9.1, jruby-9.2] + ruby: ['2.0', 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, '3.0', 3.1, 3.2, jruby-9.1, jruby-9.2, jruby-9.3, jruby-9.4] env: [NEW_RAILS, OLD_RAILS] exclude: - - ruby: 2.0 + - ruby: '2.0' env: NEW_RAILS - ruby: 2.1 env: NEW_RAILS diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c9d2a81..419048fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,8 @@ Unreleased Changes ------------------ +* Feature - Implement the `BatchGetItem` operation (#122) + 2.9.0 (2022-11-16) ------------------ diff --git a/README.md b/README.md index e4d475da..1e47e38a 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,72 @@ item.active = false item.save ``` +### `BatchGetItem` and `BatchWriteItem` +Aws Record provides [BatchGetItem](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method) +and [BatchWriteItem](https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method) +support for aws-record models. + +More info under the following documentation: +* [Batch](https://docs.aws.amazon.com/sdk-for-ruby/aws-record/api/Aws/Record/Batch.html) +* [BatchWrite](https://docs.aws.amazon.com/sdk-for-ruby/aws-record/api/Aws/Record/BatchWrite.html) +* [BatchRead](https://docs.aws.amazon.com/sdk-for-ruby/aws-record/api/Aws/Record/BatchRead.html) + +See examples below to see the feature in action. + +**`BatchGetItem` Example:** +```ruby +class Lunch + include Aws::Record + integer_attr :id, hash_key: true + string_attr :name, range_key: true +end + +class Dessert + include Aws::Record + integer_attr :id, hash_key: true + string_attr :name, range_key: true +end + +# batch operations +operation = Aws::Record::Batch.read do |db| + db.find(Lunch, id: 1, name: 'Papaya Salad') + db.find(Lunch, id: 2, name: 'BLT Sandwich') + db.find(Dessert, id: 1, name: 'Apple Pie') +end + +# BatchRead is enumerable and handles pagination +operation.each { |item| item.id } + +# Alternatively, BatchRead provides a lower level interface through: execute!, complete? and items. +# Unprocessed items can be processed by calling: +operation.execute! until operation.complete? +``` + +**`BatchWriteItem` Example:** +```ruby +class Breakfast + include Aws::Record + integer_attr :id, hash_key: true + string_attr :name, range_key: true + string_attr :body +end + +# setup +eggs = Breakfast.new(id: 1, name: "eggs").save! +waffles = Breakfast.new(id: 2, name: "waffles") +pancakes = Breakfast.new(id: 3, name: "pancakes") + +# batch operations +operation = Aws::Record::Batch.write(client: Breakfast.dynamodb_client) do |db| + db.put(waffles) + db.delete(eggs) + db.put(pancakes) +end + +# unprocessed items can be retried by calling Aws::Record::BatchWrite#execute! +operation.execute! until operation.complete? +``` + ### Inheritance Support Aws Record models can be extended using standard ruby inheritance. The child model must include `Aws::Record` in their model and the following will be inherited: diff --git a/features/batch/batch.feature b/features/batch/batch.feature new file mode 100644 index 00000000..0a32315f --- /dev/null +++ b/features/batch/batch.feature @@ -0,0 +1,73 @@ +# language: en + +@dynamodb @batch +Feature: Amazon DynamoDB Batch + This feature tests the ability to use the batch read and write item APIs via + aws-record. To run these tests, you will need to have valid AWS credentials + that are accessible with the AWS SDK for Ruby's standard credential provider + chain. In practice, this means a shared credential file or environment + variables with your credentials. These tests may have some AWS costs associated + with running them since AWS resources are created and destroyed within + these tests. + + Background: + Given a Parent model with definition: + """ + set_table_name('FoodTable') + integer_attr :id, hash_key: true, database_attribute_name: 'Food ID' + string_attr :dish, range_key: true + boolean_attr :spicy + """ + And a Parent model with TableConfig of: + """ + Aws::Record::TableConfig.define do |t| + t.model_class(ParentTableModel) + t.read_capacity_units(2) + t.write_capacity_units(2) + t.client_options(region: "us-east-1") + end + """ + When we migrate the TableConfig + Then eventually the table should exist in DynamoDB + And a Child model with definition: + """ + set_table_name('DessertTable') + boolean_attr :gluten_free + """ + And a Child model with TableConfig of: + """ + Aws::Record::TableConfig.define do |t| + t.model_class(ChildTableModel) + t.read_capacity_units(2) + t.write_capacity_units(2) + t.client_options(region: "us-east-1") + end + """ + When we migrate the TableConfig + Then eventually the table should exist in DynamoDB + + Scenario: Perform a batch set of writes and read + When we make a batch write call with following Parent and Child model items: + """ + [ + { "model": "Parent", "id": 1, "dish": "Papaya Salad", "spicy": true }, + { "model": "Parent", "id": 2, "dish": "Hamburger", "spicy": false }, + { "model": "Child", "id": 1, "dish": "Apple Pie", "spicy": false, "gluten_free": false } + ] + """ + And we make a batch read call for the following Parent and Child model item keys: + """ + [ + { "model": "Parent", "id": 1, "dish": "Papaya Salad" }, + { "model": "Parent", "id": 2, "dish": "Hamburger" }, + { "model": "Child", "id": 1, "dish": "Apple Pie" } + ] + """ + Then we expect the batch read result to include the following items: + """ + [ + { "id": 1, "dish": "Papaya Salad", "spicy": true }, + { "id": 2, "dish": "Hamburger", "spicy": false }, + { "id": 1, "dish": "Apple Pie", "spicy": false, "gluten_free": false } + ] + """ \ No newline at end of file diff --git a/features/batch/step_definitions.rb b/features/batch/step_definitions.rb new file mode 100644 index 00000000..63791f0e --- /dev/null +++ b/features/batch/step_definitions.rb @@ -0,0 +1,62 @@ +# frozen_string_literal: true + +And(/^a (Parent|Child) model with TableConfig of:$/) do |model, code_block| + case model + when 'Parent' + ParentTableModel = @parent + when 'Child' + ChildTableModel = @model + else + raise 'Model must be either a Parent or Child' + end + + @table_config = eval(code_block) +end + +When(/^we make a batch write call with following Parent and Child model items:$/) do |string| + item_data = JSON.parse(string, symbolize_names: true) + + Aws::Record::Batch.write do |db| + item_data.each do |item| + case item[:model] + when 'Parent' + db.put(@parent.new(remove_model_key(item))) + when 'Child' + db.put(@model.new(remove_model_key(item))) + else + raise 'Model must be either a Parent or Child' + end + end + end +end + +And(/^we make a batch read call for the following Parent and Child model item keys:$/) do |string| + key_batch = JSON.parse(string, symbolize_names: true) + + @batch_read_result = Aws::Record::Batch.read do |db| + key_batch.each do |item_key| + case item_key[:model] + when 'Parent' + db.find(@parent, remove_model_key(item_key)) + when 'Child' + db.find(@model, remove_model_key(item_key)) + else + raise 'Model must be either a Parent or Child' + end + end + end +end + +Then(/^we expect the batch read result to include the following items:$/) do |string| + expected = JSON.parse(string, symbolize_names: true) + actual = @batch_read_result.items.map(&:to_h) + expect(expected.count).to eq(actual.count) + expect(expected.all? { |e| actual.include?(e) }).to be_truthy +end + +private + +def remove_model_key(item) + item.delete(:model) + item +end diff --git a/lib/aws-record.rb b/lib/aws-record.rb index 857febe3..af50848d 100644 --- a/lib/aws-record.rb +++ b/lib/aws-record.rb @@ -1,15 +1,4 @@ -# Copyright 2015-2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not -# use this file except in compliance with the License. A copy of the License is -# located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is distributed on -# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions -# and limitations under the License. +# frozen_string_literal: true require 'aws-sdk-dynamodb' require_relative 'aws-record/record/client_configuration' @@ -30,6 +19,7 @@ require_relative 'aws-record/record/version' require_relative 'aws-record/record/transactions' require_relative 'aws-record/record/buildable_search' +require_relative 'aws-record/record/batch_read' require_relative 'aws-record/record/batch_write' require_relative 'aws-record/record/batch' require_relative 'aws-record/record/marshalers/string_marshaler' diff --git a/lib/aws-record/record/batch.rb b/lib/aws-record/record/batch.rb index ed5e17f3..a5789f88 100644 --- a/lib/aws-record/record/batch.rb +++ b/lib/aws-record/record/batch.rb @@ -1,15 +1,4 @@ -# Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not -# use this file except in compliance with the License. A copy of the License is -# located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is distributed on -# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions -# and limitations under the License. +# frozen_string_literal: true module Aws module Record @@ -17,6 +6,28 @@ class Batch extend ClientConfiguration class << self + # Provides a thin wrapper to the + # {https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method Aws::DynamoDB::Client#batch_write_item} + # method. Up to 25 +PutItem+ or +DeleteItem+ operations are supported. + # A single request may write up to 16 MB of data, with each item having a + # write limit of 400 KB. + # + # *Note*: this operation does not support dirty attribute handling, + # nor does it enforce safe write operations (i.e. update vs new record + # checks). + # + # This call may partially execute write operations. Failed operations + # are returned as {BatchWrite.unprocessed_items unprocessed_items} (i.e. the + # table fails to meet requested write capacity). Any unprocessed + # items may be retried by calling {BatchWrite.execute! .execute!} + # again. You can determine if the request needs to be retried by calling + # the {BatchWrite.complete? .complete?} method - which returns +true+ + # when all operations have been completed. + # + # Please see + # {https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Programming.Errors.html#Programming.Errors.BatchOperations Batch Operations and Error Handling} + # in the DynamoDB Developer Guide for more details. + # # @example Usage Example # class Breakfast # include Aws::Record @@ -38,29 +49,7 @@ class << self # end # # # unprocessed items can be retried by calling Aws::Record::BatchWrite#execute! - # operation.execute! unless operation.complete? - # - # Provides a thin wrapper to the - # {https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#batch_write_item-instance_method Aws::DynamoDB::Client#batch_write_item} - # method. Up to 25 +PutItem+ or +DeleteItem+ operations are supported. - # A single rquest may write up to 16 MB of data, with each item having a - # write limit of 400 KB. - # - # *Note*: this operation does not support dirty attribute handling, - # nor does it enforce safe write operations (i.e. update vs new record - # checks). - # - # This call may partially execute write operations. Failed operations - # are returned as +Aws::Record::BatchWrite#unprocessed_items+ (i.e. the - # table fails to meet requested write capacity). Any unprocessed - # items may be retried by calling +Aws::Record::BatchWrite#execute!+ - # again. You can determine if the request needs to be retried by calling - # the +Aws::Record::BatchWrite#complete?+ method - which returns +true+ - # when all operations have been completed. - # - # Please see - # {https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Programming.Errors.html#Programming.Errors.BatchOperations Batch Operations and Error Handling} - # in the DynamoDB Developer Guide for more details. + # operation.execute! until operation.complete? # # @param [Hash] opts the options you wish to use to create the client. # Note that if you include the option +:client+, all other options @@ -76,6 +65,81 @@ def write(opts = {}, &block) block.call(batch) batch.execute! end + + # Provides support for the + # {https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method + # Aws::DynamoDB::Client#batch_get_item} for aws-record models. + # + # +Aws::Record::Batch+ is Enumerable and using Enumerable methods will handle + # paging through all requested keys automatically. Alternatively, a lower level + # interface is available. You can determine if there are any unprocessed keys by calling + # {BatchRead.complete? .complete?} and any unprocessed keys can be processed by + # calling {BatchRead.execute! .execute!}. You can access all processed items + # through {BatchRead.items .items}. + # + # The +batch_get_item+ supports up to 100 operations in a single call and a single + # operation can retrieve up to 16 MB of data. + # + # +Aws::Record::BatchRead+ can take more than 100 item keys. The first 100 requests + # will be processed and the remaining requests will be stored. + # When using Enumerable methods, any pending item keys will be automatically + # processed and the new items will be added to +items+. + # Alternately, use {BatchRead.execute! .execute!} to process any pending item keys. + # + # All processed operations can be accessed by {BatchRead.items items} - which is an + # array of modeled items from the response. The items will be unordered since + # DynamoDB does not return items in any particular order. + # + # If a requested item does not exist in the database, it is not returned in the response. + # + # If there is a returned item from the call and there's no reference model class + # to be found, the item will not show up under +items+. + # + # @example Usage Example + # class Lunch + # include Aws::Record + # integer_attr :id, hash_key: true + # string_attr :name, range_key: true + # end + # + # class Dessert + # include Aws::Record + # integer_attr :id, hash_key: true + # string_attr :name, range_key: true + # end + # + # # batch operations + # operation = Aws::Record::Batch.read do |db| + # db.find(Lunch, id: 1, name: 'Papaya Salad') + # db.find(Lunch, id: 2, name: 'BLT Sandwich') + # db.find(Dessert, id: 1, name: 'Apple Pie') + # end + # + # # BatchRead is enumerable and handles pagination + # operation.each { |item| item.id } + # + # # Alternatively, BatchRead provides a lower level + # # interface through: execute!, complete? and items. + # # Unprocessed items can be processed by calling: + # operation.execute! until operation.complete? + # + # @param [Hash] opts the options you wish to use to create the client. + # Note that if you include the option +:client+, all other options + # will be ignored. See the documentation for other options in the + # {https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#initialize-instance_method + # AWS SDK for Ruby}. + # @option opts [Aws::DynamoDB::Client] :client allows you to pass in your + # own pre-configured client. + # @return [Aws::Record::BatchRead] An instance that contains modeled items + # from the +BatchGetItem+ result and stores unprocessed keys to be + # manually processed later. + def read(opts = {}, &block) + batch = BatchRead.new(client: _build_client(opts)) + block.call(batch) + batch.execute! + batch + end + end end end diff --git a/lib/aws-record/record/batch_read.rb b/lib/aws-record/record/batch_read.rb new file mode 100644 index 00000000..d18c4144 --- /dev/null +++ b/lib/aws-record/record/batch_read.rb @@ -0,0 +1,186 @@ +# frozen_string_literal: true + +module Aws + module Record + class BatchRead + include Enumerable + + # @api private + BATCH_GET_ITEM_LIMIT = 100 + + # @param [Aws::DynamoDB::Client] client the DynamoDB SDK client. + def initialize(opts = {}) + @client = opts[:client] + end + + # Append the item keys to a batch read request. + # + # See {Batch.read} for example usage. + # @param [Aws::Record] klass a model class that includes {Aws::Record} + # @param [Hash] key attribute-value pairs for the key you wish to search for. + # @raise [Aws::Record::Errors::KeyMissing] if your option parameters + # do not include all item keys defined in the model. + # @raise [ArgumentError] if the provided item keys is a duplicate request + # in the same instance. + def find(klass, key = {}) + unprocessed_key = format_unprocessed_key(klass, key) + store_unprocessed_key(klass, unprocessed_key) + store_item_class(klass, unprocessed_key) + end + + # Perform a +batch_get_item+ request. + # + # This method processes the first 100 item keys and + # returns an array of new modeled items. + # + # See {Batch.read} for example usage. + # @return [Array] an array of unordered new items + def execute! + operation_keys = unprocessed_keys[0..BATCH_GET_ITEM_LIMIT - 1] + @unprocessed_keys = unprocessed_keys[BATCH_GET_ITEM_LIMIT..-1] || [] + + operations = build_operations(operation_keys) + result = @client.batch_get_item(request_items: operations) + new_items = build_items(result.responses) + items.concat(new_items) + + unless result.unprocessed_keys.nil? + update_unprocessed_keys(result.unprocessed_keys) + end + + new_items + end + + # Provides an enumeration of the results from the +batch_get_item+ request + # and handles pagination. + # + # Any pending item keys will be automatically processed and be + # added to the {#items}. + # + # See {Batch.read} for example usage. + # @yieldparam [Aws::Record] item a modeled item + # @return [Enumerable] an enumeration over the results of + # +batch_get_item+ request. + def each + return enum_for(:each) unless block_given? + + @items.each { |item| yield item } + until complete? + new_items = execute! + new_items.each { |new_item| yield new_item } + end + end + + # Indicates if all item keys have been processed. + # + # See {Batch.read} for example usage. + # @return [Boolean] +true+ if all item keys has been processed, +false+ otherwise. + def complete? + unprocessed_keys.none? + end + + # Returns an array of modeled items. The items are marshaled into classes used in {#find} method. + # These items will be unordered since DynamoDB does not return items in any particular order. + # + # See {Batch.read} for example usage. + # @return [Array] an array of modeled items from the +batch_get_item+ call. + def items + @items ||= [] + end + + private + + def unprocessed_keys + @unprocessed_keys ||= [] + end + + def item_classes + @item_classes ||= Hash.new { |h, k| h[k] = [] } + end + + def format_unprocessed_key(klass, key) + item_key = {} + attributes = klass.attributes + klass.keys.each_value do |attr_sym| + unless key[attr_sym] + raise Errors::KeyMissing, "Missing required key #{attr_sym} in #{key}" + end + + attr_name = attributes.storage_name_for(attr_sym) + item_key[attr_name] = attributes.attribute_for(attr_sym) + .serialize(key[attr_sym]) + end + item_key + end + + def store_unprocessed_key(klass, unprocessed_key) + unprocessed_keys << { keys: unprocessed_key, table_name: klass.table_name } + end + + def store_item_class(klass, key) + if item_classes.include?(klass.table_name) + item_classes[klass.table_name].each do |item| + if item[:keys] == key && item[:class] != klass + raise ArgumentError, 'Provided item keys is a duplicate request' + end + end + end + item_classes[klass.table_name] << { keys: key, class: klass } + end + + def build_operations(keys) + operations = Hash.new { |h, k| h[k] = { keys: [] } } + keys.each do |key| + operations[key[:table_name]][:keys] << key[:keys] + end + operations + end + + def build_items(item_responses) + new_items = [] + item_responses.each do |table, unprocessed_items| + unprocessed_items.each do |item| + item_class = find_item_class(table, item) + if item_class.nil? && @client.config.logger + @client.config.logger.warn( + 'Unexpected response from service.'\ + "Received: #{item}. Skipping above item and continuing" + ) + else + new_items << build_item(item, item_class) + end + end + end + new_items + end + + def update_unprocessed_keys(keys) + keys.each do |table_name, table_values| + table_values.keys.each do |key| + unprocessed_keys << { keys: key, table_name: table_name } + end + end + end + + def find_item_class(table, item) + selected_item = item_classes[table].find { |item_info| contains_keys?(item, item_info[:keys]) } + selected_item[:class] if selected_item + end + + def contains_keys?(item, keys) + item.merge(keys) == item + end + + def build_item(item, item_class) + new_item_opts = {} + item.each do |db_name, value| + name = item_class.attributes.db_to_attribute_name(db_name) + new_item_opts[name] = value + end + item = item_class.new(new_item_opts) + item.clean! + item + end + end + end +end diff --git a/lib/aws-record/record/batch_write.rb b/lib/aws-record/record/batch_write.rb index 87134f9a..a5b4ae52 100644 --- a/lib/aws-record/record/batch_write.rb +++ b/lib/aws-record/record/batch_write.rb @@ -15,12 +15,14 @@ module Aws module Record class BatchWrite # @param [Aws::DynamoDB::Client] client the DynamoDB SDK client. - def initialize(client:) - @client = client + def initialize(opts = {}) + @client = opts[:client] end # Append a +PutItem+ operation to a batch write request. # + # See {Batch.write} for example usage. + # # @param [Aws::Record] record a model class that includes {Aws::Record}. def put(record) table_name, params = record_put_params(record) @@ -30,6 +32,7 @@ def put(record) # Append a +DeleteItem+ operation to a batch write request. # + # See {Batch.write} for example usage. # @param [Aws::Record] record a model class that includes {Aws::Record}. def delete(record) table_name, params = record_delete_params(record) @@ -39,6 +42,7 @@ def delete(record) # Perform a +batch_write_item+ request. # + # See {Batch.write} for example usage. # @return [Aws::Record::BatchWrite] an instance that provides access to # unprocessed items and allows for retries. def execute! @@ -49,6 +53,7 @@ def execute! # Indicates if all items have been processed. # + # See {Batch.write} for example usage. # @return [Boolean] +true+ if +unprocessed_items+ is empty, +false+ # otherwise def complete? @@ -58,6 +63,7 @@ def complete? # Returns all +DeleteItem+ and +PutItem+ operations that have not yet been # processed successfully. # + # See {Batch.write} for example usage. # @return [Hash] All operations that have not yet successfully completed. def unprocessed_items operations diff --git a/lib/aws-record/record/item_collection.rb b/lib/aws-record/record/item_collection.rb index 651aacfd..3ffc01bf 100644 --- a/lib/aws-record/record/item_collection.rb +++ b/lib/aws-record/record/item_collection.rb @@ -54,7 +54,7 @@ def each(&block) # match your search. # # @return [Array] an array of the record items found in the - # first page of reponses from the query or scan call. + # first page of responses from the query or scan call. def page search_response = items @last_evaluated_key = search_response.last_evaluated_key diff --git a/lib/aws-record/record/item_operations.rb b/lib/aws-record/record/item_operations.rb index 7f641da8..3dd47fd8 100644 --- a/lib/aws-record/record/item_operations.rb +++ b/lib/aws-record/record/item_operations.rb @@ -1,15 +1,4 @@ -# Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not -# use this file except in compliance with the License. A copy of the License is -# located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is distributed on -# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions -# and limitations under the License. +# frozen_string_literal: true module Aws module Record @@ -514,6 +503,43 @@ def find_with_opts(opts) end end + + # @example Usage Example + # class MyModel + # include Aws::Record + # integer_attr :id, hash_key: true + # string_attr :name, range_key: true + # end + # + # # returns a homogenous list of items + # foo_items = MyModel.find_all( + # [ + # {id: 1, name: 'n1'}, + # {id: 2, name: 'n2'} + # ]) + # + # Provides support for the + # {https://docs.aws.amazon.com/sdk-for-ruby/v3/api/Aws/DynamoDB/Client.html#batch_get_item-instance_method + # Aws::DynamoDB::Client#batch_get_item} for your model. + # + # This method will take a list of keys and return an instance of +Aws::Record::BatchRead+ + # + # See {Batch.read} for more details. + # @param [Array] keys an array of item key hashes you wish to search for. + # @return [Aws::Record::BatchRead] An instance that contains modeled items + # from the +BatchGetItem+ result and stores unprocessed keys to be + # manually processed later. + # @raise [Aws::Record::Errors::KeyMissing] if your param hashes do not + # include all the keys defined in model. + # @raise [ArgumentError] if the provided keys are a duplicate. + def find_all(keys) + Aws::Record::Batch.read do |db| + keys.each do |key| + db.find(self, key) + end + end + end + # @example Usage Example # class MyModel # include Aws::Record diff --git a/spec/aws-record/record/batch_spec.rb b/spec/aws-record/record/batch_spec.rb index 177779ce..bcbef99b 100644 --- a/spec/aws-record/record/batch_spec.rb +++ b/spec/aws-record/record/batch_spec.rb @@ -1,33 +1,26 @@ -# Copyright 2015-2021 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not -# use this file except in compliance with the License. A copy of the License is -# located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is distributed on -# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions -# and limitations under the License. - -Planet = Class.new do - include(Aws::Record) - integer_attr(:id, hash_key: true) - string_attr(:name, range_key: true) -end +# frozen_string_literal: true + +require 'spec_helper' describe Aws::Record::Batch do - let(:client) { Aws::DynamoDB::Client.new(stub_responses: true) } + let(:stub_logger) { double(info: nil) } - before(:each) do - Planet.configure_client(client: client) - end + let(:stub_client) { Aws::DynamoDB::Client.new(stub_responses: true, logger: stub_logger) } describe '.write' do + Planet = Class.new do + include(Aws::Record) + integer_attr(:id, hash_key: true) + string_attr(:name, range_key: true) + end + + before(:each) do + Planet.configure_client(client: stub_client) + end + let(:pluto) { Planet.find(id: 9, name: 'pluto') } let(:result) do - described_class.write(client: client) do |db| + described_class.write(client: stub_client) do |db| db.put(Planet.new(id: 1, name: 'mercury')) db.put(Planet.new(id: 2, name: 'venus')) db.put(Planet.new(id: 3, name: 'earth')) @@ -41,7 +34,7 @@ end before(:each) do - client.stub_responses( + stub_client.stub_responses( :get_item, item: { 'id' => 9, @@ -52,7 +45,7 @@ context 'when all operations succeed' do before(:each) do - client.stub_responses( + stub_client.stub_responses( :batch_write_item, unprocessed_items: {} ) @@ -69,7 +62,7 @@ context 'when some operations fail' do before(:each) do - client.stub_responses( + stub_client.stub_responses( :batch_write_item, unprocessed_items: { 'planet' => [ @@ -89,4 +82,206 @@ end end end + + describe '.read' do + let(:food) do + Class.new do + include(Aws::Record) + set_table_name('FoodTable') + integer_attr(:id, hash_key: true, database_attribute_name: 'Food ID') + string_attr(:dish, range_key: true) + boolean_attr(:spicy) + end + end + + let(:breakfast) do + Class.new(food) do + include(Aws::Record) + boolean_attr(:gluten_free) + end + end + + let(:drink) do + Class.new do + include(Aws::Record) + set_table_name('DrinkTable') + integer_attr(:id, hash_key: true) + string_attr(:drink) + end + end + + before(:each) do + Aws::Record::Batch.configure_client(client: stub_client) + end + + context 'when all operations succeed' do + before(:each) do + stub_client.stub_responses( + :batch_get_item, + responses: { + 'FoodTable' => [ + { 'Food ID' => 1, 'dish' => 'Pasta', 'spicy' => false }, + { 'Food ID' => 2, 'dish' => 'Waffles', 'spicy' => false, 'gluten_free' => true } + ], + 'DrinkTable' => [ + { 'id' => 1, 'drink' => 'Hot Chocolate' } + ] + } + ) + end + + let(:result) do + Aws::Record::Batch.read(client: stub_client) do |db| + db.find(food, id: 1, dish: 'Pasta') + db.find(breakfast, id: 2, dish: 'Waffles') + db.find(drink, id: 1) + end + end + + it 'reads a batch of operations and returns modeled items' do + expect(result).to be_an(Aws::Record::BatchRead) + expect(result.items.size).to eq(3) + expect(result.items[0].class).to eq(food) + expect(result.items[1].class).to eq(breakfast) + expect(result.items[2].class).to eq(drink) + expect(result.items[0].dirty?).to be_falsey + expect(result.items[1].dirty?).to be_falsey + expect(result.items[2].dirty?).to be_falsey + expect(result.items[0].spicy).to be_falsey + expect(result.items[1].spicy).to be_falsey + expect(result.items[1].gluten_free).to be_truthy + expect(result.items[2].drink).to eq('Hot Chocolate') + end + + it 'is complete' do + expect(result).to be_complete + end + end + + context 'when there are more than 100 records' do + let(:response_array) do + (1..99).each.map do |i| + { 'Food ID' => i, 'dish' => "Food#{i}", 'spicy' => false } + end + end + + before(:each) do + stub_client.stub_responses( + :batch_get_item, + { + responses: { + 'FoodTable' => response_array + }, + unprocessed_keys: { + 'FoodTable' => { + keys: [ + { 'Food ID' => 100, 'dish' => 'Food100' } + ] + } + } + } + ) + end + + let(:result) do + Aws::Record::Batch.read(client: stub_client) do |db| + (1..101).each do |i| + db.find(food, id: i, dish: "Food#{i}") + end + end + end + + it 'reads batch of operations and returns most processed items' do + expect(result).to be_an(Aws::Record::BatchRead) + expect(result.items.size).to eq(99) + end + + it 'is not complete' do + expect(result).to_not be_complete + end + + it 'can process the remaining records by running execute' do + expect(result).to_not be_complete + stub_client.stub_responses( + :batch_get_item, + responses: { + 'FoodTable' => [ + { 'Food ID' => 100, 'dish' => 'Food100', 'spicy' => false }, + { 'Food ID' => 101, 'dish' => 'Food101', 'spicy' => false } + ] + } + ) + result.execute! + expect(result).to be_complete + expect(result).to be_an(Aws::Record::BatchRead) + expect(result.items.size).to eq(101) + end + + it 'is a enumerable' do + expect(result).to_not be_complete + stub_client.stub_responses( + :batch_get_item, + responses: { + 'FoodTable' => [ + { 'Food ID' => 100, 'dish' => 'Food100', 'spicy' => false }, + { 'Food ID' => 101, 'dish' => 'Food101', 'spicy' => false } + ] + } + ) + result.each.with_index(1) do |item, expected_id| + expect(item.id).to eq(expected_id) + end + expect(result.to_a.size).to eq(101) + end + + end + + it 'raises when a record is missing a key' do + expect { + Aws::Record::Batch.read(client: stub_client) do |db| + db.find(food, id: 1) + end + }.to raise_error(Aws::Record::Errors::KeyMissing) + end + + it 'raises when there is a duplicate item key' do + expect { + Aws::Record::Batch.read(client: stub_client) do |db| + db.find(food, id: 1, dish: 'Pancakes') + db.find(breakfast, id: 1, dish: 'Pancakes') + end + }.to raise_error(ArgumentError) + end + + it 'raises exception when BatchGetItem raises an exception' do + stub_client.stub_responses( + :batch_get_item, + 'ProvisionedThroughputExceededException' + ) + expect { + Aws::Record::Batch.read(client: stub_client) do |db| + db.find(food, id: 1, dish: 'Omurice') + db.find(breakfast, id: 2, dish: 'Omelette') + end + }.to raise_error(Aws::DynamoDB::Errors::ProvisionedThroughputExceededException) + end + + it 'warns when unable to model item from response' do + stub_client.stub_responses( + :batch_get_item, + responses: { + 'FoodTable' => [ + { 'Food ID' => 1, 'dish' => 'Pasta', 'spicy' => false } + ], + 'DinnerTable' => [ + { 'id' => 1, 'dish' => 'Spaghetti' } + ] + } + ) + expect(stub_logger).to receive(:warn).with(/Unexpected response from service/) + Aws::Record::Batch.read(client: stub_client) do |db| + db.find(food, id: 1, dish: 'Pasta') + end + end + end end diff --git a/spec/aws-record/record/item_operations_spec.rb b/spec/aws-record/record/item_operations_spec.rb index 3fc58535..f5166bb9 100644 --- a/spec/aws-record/record/item_operations_spec.rb +++ b/spec/aws-record/record/item_operations_spec.rb @@ -1,15 +1,4 @@ -# Copyright 2015-2016 Amazon.com, Inc. or its affiliates. All Rights Reserved. -# -# Licensed under the Apache License, Version 2.0 (the "License"). You may not -# use this file except in compliance with the License. A copy of the License is -# located at -# -# http://aws.amazon.com/apache2.0/ -# -# or in the "license" file accompanying this file. This file is distributed on -# an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express -# or implied. See the License for the specific language governing permissions -# and limitations under the License. +# frozen_string_literal: true require 'spec_helper' @@ -343,6 +332,24 @@ module Record end end + describe '#find_all' do + let(:keys) do + [ + {id: 1, date: '2022-12-24'}, + {id: 2, date: '2022-12-25'}, + {id: 3, date: '2022-12-26'}, + ] + end + + it 'passes the correct class and key arguments to BatchRead' do + mock_batch_read = double + expect(Batch).to receive(:read).and_yield(mock_batch_read).and_return(mock_batch_read) + keys.each { |key| expect(mock_batch_read).to receive(:find).with(klass, key) } + result = klass.find_all(keys) + expect(result).to eql(mock_batch_read) + end + end + describe "#update" do it 'can find and update an item from Amazon DynamoDB' do klass.configure_client(client: stub_client)