Merge pull request #93 from btnguyen2k/dev

Prepare to release next version

README.md

@@ -123,6 +123,17 @@ fmt.Println("RowsAffected:", rowsAffected2) // output "RowsAffected: 1"
 >
 > You can use [EXISTS function](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-functions.exists.html) for condition checking.
 
+## Caveats
+
+**Numerical values** are stored in DynamoDB as floating point numbers. Hence, numbers are always read back as `float64`. 
+See [DynamoDB document](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html#HowItWorks.DataTypes) for details on DynamoDB's supported data types.
+
+**A single query can only return up to [1MB of data](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.Pagination.html)**.
+In the case of `SELECT` query, the driver automatically issues additional queries to fetch the remaining data if needed.
+However, returned rows may not be in the expected order specified by `ORDER BY` clause. 
+That means, rows returned from the query `SELECT * FROM table_name WHERE category='Laptop' ORDER BY id` may not be in
+the expected order if all matched rows do not fit in 1MB of data.
+
 ## License
 
 This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details.

SQL_DOCUMENT.md

@@ -49,14 +49,16 @@ Sample result:
 > 
 >       dbrows, err := db.Query(`SELECT * FROM "session" WHERE app='frontend' LIMIT 10`)
 >
-> Note: the value for `LIMIT` must be a positive integer.
+> Note:
+> - The `LIMIT` clause is extension offered by `godynamodb` and is not part of [PartiQL syntax](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.select.html).
+> - The value for `LIMIT` must be a _positive integer_.
 
 > Since [v0.4.0](RELEASE-NOTES.md), `godynamodb` supports ConsistentRead for `SELECT` statement via clause `WITH ConsistentRead=true` or `WITH Consistent_Read=true`.
 > Example:
 >
 >       dbrows, err := db.Query(`SELECT * FROM "session" WHERE app='frontend' WITH ConsistentRead=true`)
 >
-> Note: the WITH clause must be placed at the end of the SELECT statement.
+> Note: the WITH clause must be placed _at the end_ of the SELECT statement.
 
 ## UPDATE
 

conn.go

@@ -131,16 +131,76 @@ func (c *Conn) executeContext(ctx context.Context, stmt *Stmt, values []driver.N
 	if len(params) > 0 {
 		input.Parameters = params
 	}
-
 	if consistentRead, ok := stmt.withOpts["CONSISTENT_READ"]; ok {
 		input.ConsistentRead = aws.Bool(consistentRead.FirstBool())
 	} else if consistentRead, ok = stmt.withOpts["CONSISTENTREAD"]; ok {
 		input.ConsistentRead = aws.Bool(consistentRead.FirstBool())
 	}
 
-	output, err := c.client.ExecuteStatement(c.ensureContext(ctx), input)
+	if !reSelect.MatchString(stmt.query) {
+		output, err := c.client.ExecuteStatement(c.ensureContext(ctx), input)
+		return func() *dynamodb.ExecuteStatementOutput {
+			return output
+		}, err
+	}
+
+	return c.executeSelectContext(ctx, stmt, input)
+}
+
+// SELECT query could be paged, need to fetch all pages
+func (c *Conn) executeSelectContext(ctx context.Context, stmt *Stmt, input *dynamodb.ExecuteStatementInput) (executeStatementOutputWrapper, error) {
+	ctx = c.ensureContext(ctx)
+	var firstOutput *dynamodb.ExecuteStatementOutput
+	var err error
+	var limitNumItems int32 = 0
+	if stmt.limit != nil {
+		limitNumItems = *stmt.limit
+	}
+	//idx := 0                         // FIXME
+	//fetched := make(map[string]bool) // FIXME
+	for {
+		var output *dynamodb.ExecuteStatementOutput
+		output, err = c.client.ExecuteStatement(ctx, input)
+		if err != nil {
+			return func() *dynamodb.ExecuteStatementOutput {
+				return output
+			}, err
+		}
+
+		//// FIXME
+		//idx++
+		//for _, item := range output.Items {
+		//	fetched[item["id"].(*types.AttributeValueMemberS).Value] = true
+		//}
+		//fmt.Printf("[DEBUG] %2d / %s (LIMIT %#v) / LastEvaluatedKey: %d - NextToken: %5v / Fetched: %2d - Total: %2d\n", idx, stmt.query, stmt.limit, len(output.LastEvaluatedKey), output.NextToken != nil, len(output.Items), len(fetched))
+		//// END FIXME
+
+		if firstOutput == nil {
+			firstOutput = output
+		} else {
+			firstOutput.ResultMetadata = output.ResultMetadata
+			firstOutput.LastEvaluatedKey = output.LastEvaluatedKey
+			firstOutput.NextToken = output.NextToken
+			firstOutput.ConsumedCapacity = output.ConsumedCapacity
+			firstOutput.Items = append(firstOutput.Items, output.Items...)
+		}
+		input.NextToken = output.NextToken
+
+		//merge result
+		if limitNumItems > 0 {
+			if len(firstOutput.Items) >= int(limitNumItems) {
+				firstOutput.Items = firstOutput.Items[:limitNumItems]
+				break
+			}
+			input.Limit = aws.Int32(limitNumItems - int32(len(firstOutput.Items)))
+		}
+
+		if output.NextToken == nil {
+			break
+		}
+	}
 	return func() *dynamodb.ExecuteStatementOutput {
-		return output
+		return firstOutput
 	}, err
 }
 

module_test/godynamo_test.go

@@ -24,9 +24,9 @@ var (
 
 func Test_OpenDatabase(t *testing.T) {
 	testName := "Test_OpenDatabase"
-	driver := "godynamo"
+	dbdriver := "godynamo"
 	dsn := "dummy"
-	db, err := sql.Open(driver, dsn)
+	db, err := sql.Open(dbdriver, dsn)
 	if err != nil {
 		t.Fatalf("%s failed: %s", testName, err)
 	}
@@ -55,12 +55,12 @@ func TestConn_ValuesToNamedValues(t *testing.T) {
 /*----------------------------------------------------------------------*/
 
 func _openDb(t *testing.T, testName string) *sql.DB {
-	driver := "godynamo"
+	dbdriver := "godynamo"
 	url := strings.ReplaceAll(os.Getenv("AWS_DYNAMODB_URL"), `"`, "")
 	if url == "" {
 		t.Skipf("%s skipped", testName)
 	}
-	db, err := sql.Open(driver, url)
+	db, err := sql.Open(dbdriver, url)
 	if err != nil {
 		t.Fatalf("%s failed: %s", testName+"/sql.Open", err)
 	}
@@ -72,12 +72,12 @@ func _openDb(t *testing.T, testName string) *sql.DB {
 func TestDriver_Conn(t *testing.T) {
 	testName := "TestDriver_Conn"
 	db := _openDb(t, testName)
-	defer db.Close()
+	defer func() { _ = db.Close() }()
 	conn, err := db.Conn(context.Background())
 	if err != nil {
 		t.Fatalf("%s failed: %s", testName, err)
 	}
-	defer conn.Close()
+	defer func() { _ = conn.Close() }()
 }
 
 // func TestDriver_Transaction(t *testing.T) {
@@ -94,7 +94,7 @@ func TestDriver_Conn(t *testing.T) {
 func TestDriver_Open(t *testing.T) {
 	testName := "TestDriver_Open"
 	db := _openDb(t, testName)
-	defer db.Close()
+	defer func() { _ = db.Close() }()
 	if err := db.Ping(); err != nil {
 		t.Fatalf("%s failed: %s", testName, err)
 	}
@@ -103,7 +103,7 @@ func TestDriver_Open(t *testing.T) {
 func TestDriver_Close(t *testing.T) {
 	testName := "TestDriver_Close"
 	db := _openDb(t, testName)
-	defer db.Close()
+	defer func() { _ = db.Close() }()
 	if err := db.Ping(); err != nil {
 		t.Fatalf("%s failed: %s", testName, err)
 	}