diff --git a/README.md b/README.md index 7004e40..8c98d3f 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,17 @@ fmt.Println("RowsAffected:", rowsAffected2) // output "RowsAffected: 1" > > You can use [EXISTS function](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-functions.exists.html) for condition checking. +## Caveats + +**Numerical values** are stored in DynamoDB as floating point numbers. Hence, numbers are always read back as `float64`. +See [DynamoDB document](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html#HowItWorks.DataTypes) for details on DynamoDB's supported data types. + +**A single query can only return up to [1MB of data](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.Pagination.html)**. +In the case of `SELECT` query, the driver automatically issues additional queries to fetch the remaining data if needed. +However, returned rows may not be in the expected order specified by `ORDER BY` clause. +That means, rows returned from the query `SELECT * FROM table_name WHERE category='Laptop' ORDER BY id` may not be in +the expected order if all matched rows do not fit in 1MB of data. + ## License This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details. diff --git a/SQL_DOCUMENT.md b/SQL_DOCUMENT.md index 320f2eb..9e9da29 100644 --- a/SQL_DOCUMENT.md +++ b/SQL_DOCUMENT.md @@ -49,14 +49,16 @@ Sample result: > > dbrows, err := db.Query(`SELECT * FROM "session" WHERE app='frontend' LIMIT 10`) > -> Note: the value for `LIMIT` must be a positive integer. +> Note: +> - The `LIMIT` clause is extension offered by `godynamodb` and is not part of [PartiQL syntax](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.select.html). +> - The value for `LIMIT` must be a _positive integer_. > Since [v0.4.0](RELEASE-NOTES.md), `godynamodb` supports ConsistentRead for `SELECT` statement via clause `WITH ConsistentRead=true` or `WITH Consistent_Read=true`. > Example: > > dbrows, err := db.Query(`SELECT * FROM "session" WHERE app='frontend' WITH ConsistentRead=true`) > -> Note: the WITH clause must be placed at the end of the SELECT statement. +> Note: the WITH clause must be placed _at the end_ of the SELECT statement. ## UPDATE diff --git a/conn.go b/conn.go index bf0cfda..2b1f0ca 100644 --- a/conn.go +++ b/conn.go @@ -131,16 +131,76 @@ func (c *Conn) executeContext(ctx context.Context, stmt *Stmt, values []driver.N if len(params) > 0 { input.Parameters = params } - if consistentRead, ok := stmt.withOpts["CONSISTENT_READ"]; ok { input.ConsistentRead = aws.Bool(consistentRead.FirstBool()) } else if consistentRead, ok = stmt.withOpts["CONSISTENTREAD"]; ok { input.ConsistentRead = aws.Bool(consistentRead.FirstBool()) } - output, err := c.client.ExecuteStatement(c.ensureContext(ctx), input) + if !reSelect.MatchString(stmt.query) { + output, err := c.client.ExecuteStatement(c.ensureContext(ctx), input) + return func() *dynamodb.ExecuteStatementOutput { + return output + }, err + } + + return c.executeSelectContext(ctx, stmt, input) +} + +// SELECT query could be paged, need to fetch all pages +func (c *Conn) executeSelectContext(ctx context.Context, stmt *Stmt, input *dynamodb.ExecuteStatementInput) (executeStatementOutputWrapper, error) { + ctx = c.ensureContext(ctx) + var firstOutput *dynamodb.ExecuteStatementOutput + var err error + var limitNumItems int32 = 0 + if stmt.limit != nil { + limitNumItems = *stmt.limit + } + //idx := 0 // FIXME + //fetched := make(map[string]bool) // FIXME + for { + var output *dynamodb.ExecuteStatementOutput + output, err = c.client.ExecuteStatement(ctx, input) + if err != nil { + return func() *dynamodb.ExecuteStatementOutput { + return output + }, err + } + + //// FIXME + //idx++ + //for _, item := range output.Items { + // fetched[item["id"].(*types.AttributeValueMemberS).Value] = true + //} + //fmt.Printf("[DEBUG] %2d / %s (LIMIT %#v) / LastEvaluatedKey: %d - NextToken: %5v / Fetched: %2d - Total: %2d\n", idx, stmt.query, stmt.limit, len(output.LastEvaluatedKey), output.NextToken != nil, len(output.Items), len(fetched)) + //// END FIXME + + if firstOutput == nil { + firstOutput = output + } else { + firstOutput.ResultMetadata = output.ResultMetadata + firstOutput.LastEvaluatedKey = output.LastEvaluatedKey + firstOutput.NextToken = output.NextToken + firstOutput.ConsumedCapacity = output.ConsumedCapacity + firstOutput.Items = append(firstOutput.Items, output.Items...) + } + input.NextToken = output.NextToken + + //merge result + if limitNumItems > 0 { + if len(firstOutput.Items) >= int(limitNumItems) { + firstOutput.Items = firstOutput.Items[:limitNumItems] + break + } + input.Limit = aws.Int32(limitNumItems - int32(len(firstOutput.Items))) + } + + if output.NextToken == nil { + break + } + } return func() *dynamodb.ExecuteStatementOutput { - return output + return firstOutput }, err } diff --git a/module_test/godynamo_test.go b/module_test/godynamo_test.go index bed0776..0d7675c 100644 --- a/module_test/godynamo_test.go +++ b/module_test/godynamo_test.go @@ -24,9 +24,9 @@ var ( func Test_OpenDatabase(t *testing.T) { testName := "Test_OpenDatabase" - driver := "godynamo" + dbdriver := "godynamo" dsn := "dummy" - db, err := sql.Open(driver, dsn) + db, err := sql.Open(dbdriver, dsn) if err != nil { t.Fatalf("%s failed: %s", testName, err) } @@ -55,12 +55,12 @@ func TestConn_ValuesToNamedValues(t *testing.T) { /*----------------------------------------------------------------------*/ func _openDb(t *testing.T, testName string) *sql.DB { - driver := "godynamo" + dbdriver := "godynamo" url := strings.ReplaceAll(os.Getenv("AWS_DYNAMODB_URL"), `"`, "") if url == "" { t.Skipf("%s skipped", testName) } - db, err := sql.Open(driver, url) + db, err := sql.Open(dbdriver, url) if err != nil { t.Fatalf("%s failed: %s", testName+"/sql.Open", err) } @@ -72,12 +72,12 @@ func _openDb(t *testing.T, testName string) *sql.DB { func TestDriver_Conn(t *testing.T) { testName := "TestDriver_Conn" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() conn, err := db.Conn(context.Background()) if err != nil { t.Fatalf("%s failed: %s", testName, err) } - defer conn.Close() + defer func() { _ = conn.Close() }() } // func TestDriver_Transaction(t *testing.T) { @@ -94,7 +94,7 @@ func TestDriver_Conn(t *testing.T) { func TestDriver_Open(t *testing.T) { testName := "TestDriver_Open" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() if err := db.Ping(); err != nil { t.Fatalf("%s failed: %s", testName, err) } @@ -103,7 +103,7 @@ func TestDriver_Open(t *testing.T) { func TestDriver_Close(t *testing.T) { testName := "TestDriver_Close" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() if err := db.Ping(); err != nil { t.Fatalf("%s failed: %s", testName, err) } diff --git a/module_test/stmt_bigtable_test.go b/module_test/stmt_bigtable_test.go new file mode 100644 index 0000000..9f84a70 --- /dev/null +++ b/module_test/stmt_bigtable_test.go @@ -0,0 +1,292 @@ +package godynamo_test + +import ( + "fmt" + "strings" + "testing" +) + +func Test_BigTable(t *testing.T) { + testName := "Test_BigTable" + db := _openDb(t, testName) + defer func() { _ = db.Close() }() + _initTest(db) + + if _, err := db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=3 WITH wcu=5`, tblTestTemp)); err != nil { + t.Fatalf("%s failed: %s", testName+"/create_table", err) + } + type Row struct { + id string + dataChar string + dataVchar string + dataBinchar []byte + dataText string + dataUchar string + dataUvchar string + dataUtext string + dataClob string + dataUclob string + dataBlob []byte + } + rowArr := make([]Row, 0) + numRows := 100 + unicodeStr := "Chào buổi sáng, доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + unicodeStrLong := "Chào buổi sáng, đây sẽ là một đoạn văn bản dài. доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + sqlStm := `INSERT INTO "%s" VALUE {'id': ?, 'dataChar': ?, 'dataVchar': ?, 'dataBinchar': ?, 'dataText': ?, 'dataUchar': ?, 'dataUvchar': ?, 'dataUtext': ?, 'dataClob': ?, 'dataUclob': ?, 'dataBlob': ?}` + sqlStm = fmt.Sprintf(sqlStm, tblTestTemp) + for i := 1; i < numRows; i++ { + id := fmt.Sprintf("%03d", i) + row := Row{ + id: id, + dataChar: "CHAR " + id, + dataVchar: "VCHAR " + id, + dataBinchar: []byte("BINCHAR " + id), + dataText: strings.Repeat("This is supposed to be a long text ", i*2), + dataUchar: unicodeStr, + dataUvchar: unicodeStr, + dataUtext: strings.Repeat(unicodeStr, i*2), + dataClob: strings.Repeat("This is supposed to be a long text ", i*10), + dataUclob: strings.Repeat(unicodeStrLong, i*10), + dataBlob: []byte(strings.Repeat("This is supposed to be a long text ", i*10)), + } + rowArr = append(rowArr, row) + params := []interface{}{row.id, row.dataChar, + row.dataVchar, row.dataBinchar, row.dataText, row.dataUchar, row.dataUvchar, row.dataUtext, + row.dataClob, row.dataUclob, row.dataBlob} + _, err := db.Exec(sqlStm, params...) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/insert", err) + } + } + + dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s`, tblTestTemp)) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/select", err) + } + rows, err := _fetchAllRows(dbrows) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) + } + if len(rows) != len(rowArr) { + t.Fatalf("%s failed: expected %d rows but received %d", testName, len(rowArr), len(rows)) + } +} + +func Test_BigTable_withWHERE(t *testing.T) { + testName := "Test_BigTable_withWHERE" + db := _openDb(t, testName) + defer func() { _ = db.Close() }() + _initTest(db) + + if _, err := db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=3 WITH wcu=5`, tblTestTemp)); err != nil { + t.Fatalf("%s failed: %s", testName+"/create_table", err) + } + type Row struct { + id string + dataChar string + dataVchar string + dataBinchar []byte + dataText string + dataUchar string + dataUvchar string + dataUtext string + dataClob string + dataUclob string + dataBlob []byte + } + rowArr := make([]Row, 0) + numRows := 100 + unicodeStr := "Chào buổi sáng, доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + unicodeStrLong := "Chào buổi sáng, đây sẽ là một đoạn văn bản dài. доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + sqlStm := `INSERT INTO "%s" VALUE {'id': ?, 'dataChar': ?, 'dataVchar': ?, 'dataBinchar': ?, 'dataText': ?, 'dataUchar': ?, 'dataUvchar': ?, 'dataUtext': ?, 'dataClob': ?, 'dataUclob': ?, 'dataBlob': ?}` + sqlStm = fmt.Sprintf(sqlStm, tblTestTemp) + for i := 1; i < numRows; i++ { + id := fmt.Sprintf("%03d", i) + row := Row{ + id: id, + dataChar: "CHAR " + id, + dataVchar: "VCHAR " + id, + dataBinchar: []byte("BINCHAR " + id), + dataText: strings.Repeat("This is supposed to be a long text ", i*2), + dataUchar: unicodeStr, + dataUvchar: unicodeStr, + dataUtext: strings.Repeat(unicodeStr, i*2), + dataClob: strings.Repeat("This is supposed to be a long text ", i*10), + dataUclob: strings.Repeat(unicodeStrLong, i*10), + dataBlob: []byte(strings.Repeat("This is supposed to be a long text ", i*10)), + } + rowArr = append(rowArr, row) + params := []interface{}{row.id, row.dataChar, + row.dataVchar, row.dataBinchar, row.dataText, row.dataUchar, row.dataUvchar, row.dataUtext, + row.dataClob, row.dataUclob, row.dataBlob} + _, err := db.Exec(sqlStm, params...) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/insert", err) + } + } + + dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s WHERE id>'012'`, tblTestTemp)) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/select", err) + } + rows, err := _fetchAllRows(dbrows) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) + } + if len(rows) != len(rowArr)-12 { + t.Fatalf("%s failed: expected %d rows but received %d", testName, len(rowArr)-12, len(rows)) + } +} + +func Test_BigTable_withLIMIT(t *testing.T) { + testName := "Test_BigTable_withLIMIT" + db := _openDb(t, testName) + defer func() { _ = db.Close() }() + _initTest(db) + + if _, err := db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=3 WITH wcu=5`, tblTestTemp)); err != nil { + t.Fatalf("%s failed: %s", testName+"/create_table", err) + } + type Row struct { + id string + dataChar string + dataVchar string + dataBinchar []byte + dataText string + dataUchar string + dataUvchar string + dataUtext string + dataClob string + dataUclob string + dataBlob []byte + } + rowArr := make([]Row, 0) + numRows := 100 + unicodeStr := "Chào buổi sáng, доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + unicodeStrLong := "Chào buổi sáng, đây sẽ là một đoạn văn bản dài. доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + sqlStm := `INSERT INTO "%s" VALUE {'id': ?, 'dataChar': ?, 'dataVchar': ?, 'dataBinchar': ?, 'dataText': ?, 'dataUchar': ?, 'dataUvchar': ?, 'dataUtext': ?, 'dataClob': ?, 'dataUclob': ?, 'dataBlob': ?}` + sqlStm = fmt.Sprintf(sqlStm, tblTestTemp) + for i := 1; i < numRows; i++ { + id := fmt.Sprintf("%03d", i) + row := Row{ + id: id, + dataChar: "CHAR " + id, + dataVchar: "VCHAR " + id, + dataBinchar: []byte("BINCHAR " + id), + dataText: strings.Repeat("This is supposed to be a long text ", i*2), + dataUchar: unicodeStr, + dataUvchar: unicodeStr, + dataUtext: strings.Repeat(unicodeStr, i*2), + dataClob: strings.Repeat("This is supposed to be a long text ", i*10), + dataUclob: strings.Repeat(unicodeStrLong, i*10), + dataBlob: []byte(strings.Repeat("This is supposed to be a long text ", i*10)), + } + rowArr = append(rowArr, row) + params := []interface{}{row.id, row.dataChar, + row.dataVchar, row.dataBinchar, row.dataText, row.dataUchar, row.dataUvchar, row.dataUtext, + row.dataClob, row.dataUclob, row.dataBlob} + _, err := db.Exec(sqlStm, params...) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/insert", err) + } + } + + limit := 13 + dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s LIMIT %d`, tblTestTemp, limit)) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/select", err) + } + rows, err := _fetchAllRows(dbrows) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) + } + if len(rows) != limit { + t.Fatalf("%s failed: expected %d rows but received %d", testName, limit, len(rows)) + } +} + +//func Test_BigTable_withORDERBY(t *testing.T) { +// testName := "Test_BigTable_withORDERBY" +// db := _openDb(t, testName) +// defer func() { _ = db.Close() }() +// _initTest(db) +// +// if _, err := db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=category:string WITH sk=id:string WITH rcu=11 WITH wcu=11`, tblTestTemp)); err != nil { +// t.Fatalf("%s failed: %s", testName+"/create_table", err) +// } +// catList := []string{"PC", "Laptop", "Tablet", "Other"} +// rand.Shuffle(len(catList), func(i, j int) { catList[i], catList[j] = catList[j], catList[i] }) +// catCount := map[string]int{"PC": 0, "Laptop": 0, "Tablet": 0, "Other": 0} +// type Row struct { +// id string +// category string +// dataChar string +// dataVchar string +// dataBinchar []byte +// dataText string +// dataUchar string +// dataUvchar string +// dataUtext string +// dataClob string +// dataUclob string +// dataBlob []byte +// } +// rowArr := make([]Row, 0) +// numRows := 100 +// unicodeStr := "Chào buổi sáng, доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" +// unicodeStrLong := "Chào buổi sáng, đây sẽ là một đoạn văn bản dài. доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" +// sqlStm := `INSERT INTO "%s" VALUE {'id': ?, 'category': ?, 'dataChar': ?, 'dataVchar': ?, 'dataBinchar': ?, 'dataText': ?, 'dataUchar': ?, 'dataUvchar': ?, 'dataUtext': ?, 'dataClob': ?, 'dataUclob': ?, 'dataBlob': ?}` +// sqlStm = fmt.Sprintf(sqlStm, tblTestTemp) +// for i := 1; i < numRows; i++ { +// id := fmt.Sprintf("%03d", i) +// cat := catList[rand.Intn(len(catList))] +// catCount[cat]++ +// row := Row{ +// id: id, +// category: cat, +// dataChar: "CHAR " + id, +// dataVchar: "VCHAR " + id, +// dataBinchar: []byte("BINCHAR " + id), +// dataText: strings.Repeat("This is supposed to be a long text ", i*2), +// dataUchar: unicodeStr, +// dataUvchar: unicodeStr, +// dataUtext: strings.Repeat(unicodeStr, i*2), +// dataClob: strings.Repeat("This is supposed to be a long text ", i*10), +// dataUclob: strings.Repeat(unicodeStrLong, i*10), +// dataBlob: []byte(strings.Repeat("This is supposed to be a long text ", i*10)), +// } +// rowArr = append(rowArr, row) +// params := []interface{}{row.id, row.category, +// row.dataChar, row.dataVchar, row.dataBinchar, row.dataText, row.dataUchar, row.dataUvchar, row.dataUtext, +// row.dataClob, row.dataUclob, row.dataBlob} +// _, err := db.Exec(sqlStm, params...) +// if err != nil { +// t.Fatalf("%s failed: %s", testName+"/insert", err) +// } +// } +// +// for _, cat := range catList { +// dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s WHERE category=? ORDER BY id DESC`, tblTestTemp), cat) +// if err != nil { +// t.Fatalf("%s failed: %s", testName+"/select", err) +// } +// rows, err := _fetchAllRows(dbrows) +// if err != nil { +// t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) +// } +// if catCount[cat] != len(rows) { +// t.Fatalf("%s failed: expected %d rows but received %d", testName, catCount[cat], len(rows)) +// } +// for i, row := range rows { +// fmt.Printf("[DEBUG] %2d: %5s - %#v\n", i, row["category"], row["id"]) +// if row["category"] != cat { +// t.Fatalf("%s failed: expected category %s but received %s", testName, cat, row["category"]) +// } +// if i > 0 { +// if row["id"].(string) > rows[i-1]["id"].(string) { +// t.Fatalf("%s failed: expected id %s < %s", testName, row["id"], rows[i-1]["id"]) +// } +// } +// } +// } +//} diff --git a/module_test/stmt_document_test.go b/module_test/stmt_document_test.go index e04d80e..e41ed1e 100644 --- a/module_test/stmt_document_test.go +++ b/module_test/stmt_document_test.go @@ -15,7 +15,7 @@ import ( func Test_Query_Insert(t *testing.T) { testName := "Test_Query_Insert" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _, err := db.Query(fmt.Sprintf(`INSERT INTO %s VALUE {}`, tblTestTemp)) if err == nil || strings.Index(err.Error(), "not supported") < 0 { @@ -26,10 +26,10 @@ func Test_Query_Insert(t *testing.T) { func Test_Exec_Insert(t *testing.T) { testName := "Test_Exec_Insert" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) - db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=1 WITH wcu=1`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=1 WITH wcu=1`, tblTestTemp)) testData := []struct { name string @@ -66,7 +66,7 @@ func Test_Exec_Insert(t *testing.T) { func Test_Exec_Select(t *testing.T) { testName := "Test_Exec_Select" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _, err := db.Exec(fmt.Sprintf(`SELECT * FROM "%s" WHERE id='a'`, tblTestTemp)) if err == nil || strings.Index(err.Error(), "not supported") < 0 { @@ -77,7 +77,7 @@ func Test_Exec_Select(t *testing.T) { func Test_Query_Select(t *testing.T) { testName := "Test_Query_Select" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) _, err := db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=app:string WITH SK=user:string WITH rcu=3 WITH wcu=3`, tblTestTemp)) @@ -115,7 +115,7 @@ func Test_Query_Select(t *testing.T) { func Test_Query_Select_withLimit(t *testing.T) { testName := "Test_Query_Select_withLimit" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) _, err := db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=app:string WITH SK=user:string WITH rcu=5 WITH wcu=5`, tblTestTemp)) @@ -155,6 +155,9 @@ func Test_Query_Select_withLimit(t *testing.T) { t.Fatalf("%s failed: %s", testName, err) } if len(limitRows) != 2 { + for i, row := range limitRows { + fmt.Printf("%d: %#v\n", i, row) + } t.Fatalf("%s failed: expected %#v row but received %#v", testName+"/select", 2, len(limitRows)) } } @@ -162,12 +165,12 @@ func Test_Query_Select_withLimit(t *testing.T) { func Test_Exec_Delete(t *testing.T) { testName := "Test_Exec_Delete" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) // setup table - db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tblTestTemp)) - db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=app:string WITH SK=user:string WITH rcu=5 WITH wcu=5`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=app:string WITH SK=user:string WITH rcu=5 WITH wcu=5`, tblTestTemp)) _, err := db.Exec(fmt.Sprintf(`INSERT INTO "%s" VALUE {'app': ?, 'user': ?, 'os': ?, 'active': ?, 'duration': ?}`, tblTestTemp), "app0", "user1", "Ubuntu", true, 12.34) if err != nil { t.Fatalf("%s failed: %s", testName+"/insert", err) @@ -222,12 +225,12 @@ func Test_Exec_Delete(t *testing.T) { func Test_Query_Delete(t *testing.T) { testName := "Test_Query_Delete" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) // setup table - db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tblTestTemp)) - db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=app:string WITH SK=user:string WITH rcu=5 WITH wcu=5`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=app:string WITH SK=user:string WITH rcu=5 WITH wcu=5`, tblTestTemp)) _, err := db.Exec(fmt.Sprintf(`INSERT INTO "%s" VALUE {'app': ?, 'user': ?, 'os': ?, 'active': ?, 'duration': ?}`, tblTestTemp), "app0", "user1", "Ubuntu", true, 12.34) if err != nil { t.Fatalf("%s failed: %s", testName+"/insert", err) @@ -287,12 +290,12 @@ func Test_Query_Delete(t *testing.T) { func Test_Exec_Update(t *testing.T) { testName := "Test_Exec_Update" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) // setup table - db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tblTestTemp)) - db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=app:string WITH SK=user:string WITH rcu=5 WITH wcu=5`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=app:string WITH SK=user:string WITH rcu=5 WITH wcu=5`, tblTestTemp)) _, err := db.Exec(fmt.Sprintf(`INSERT INTO "%s" VALUE {'app': ?, 'user': ?, 'platform': ?, 'location': ?}`, tblTestTemp), "app0", "user0", "Linux", "AU") if err != nil { t.Fatalf("%s failed: %s", testName+"/insert", err) @@ -329,12 +332,12 @@ func Test_Exec_Update(t *testing.T) { func Test_Query_Update(t *testing.T) { testName := "Test_Query_Update" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) // setup table - db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tblTestTemp)) - db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=app:string WITH SK=user:string WITH rcu=5 WITH wcu=5`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`DROP TABLE IF EXISTS %s`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=app:string WITH SK=user:string WITH rcu=5 WITH wcu=5`, tblTestTemp)) _, err := db.Exec(fmt.Sprintf(`INSERT INTO "%s" VALUE {'app': ?, 'user': ?, 'platform': ?, 'location': ?}`, tblTestTemp), "app0", "user0", "Linux", "AU") if err != nil { t.Fatalf("%s failed: %s", testName+"/insert", err) @@ -376,10 +379,10 @@ func Test_Query_Update(t *testing.T) { func TestResultResultSet_ColumnTypeDatabaseTypeName(t *testing.T) { testName := "TestResultResultSet_ColumnTypeDatabaseTypeName" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) - db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=1 WITH wcu=1`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=1 WITH wcu=1`, tblTestTemp)) testData := map[string]struct { val interface{} typ string diff --git a/module_test/stmt_table_test.go b/module_test/stmt_table_test.go index 5578b7b..29140ad 100644 --- a/module_test/stmt_table_test.go +++ b/module_test/stmt_table_test.go @@ -11,7 +11,7 @@ import ( func Test_Query_CreateTable(t *testing.T) { testName := "Test_Query_CreateTable" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _, err := db.Query(fmt.Sprintf("CREATE TABLE %s WITH pk=id:string", tblTestTemp)) if err == nil || strings.Index(err.Error(), "not supported") < 0 { @@ -22,7 +22,7 @@ func Test_Query_CreateTable(t *testing.T) { func Test_Exec_CreateTable_Query_DescribeTable(t *testing.T) { testName := "Test_Exec_CreateTable_Query_DescribeTable" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) testData := []struct { @@ -82,7 +82,7 @@ func Test_Exec_CreateTable_Query_DescribeTable(t *testing.T) { func Test_Exec_ListTables(t *testing.T) { testName := "Test_Exec_ListTables" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _, err := db.Exec("LIST TABLES") if err == nil || strings.Index(err.Error(), "not supported") < 0 { @@ -94,11 +94,11 @@ func Test_Query_ListTables(t *testing.T) { testName := "Test_Query_ListTables" db := _openDb(t, testName) _initTest(db) - defer db.Close() + defer func() { _ = db.Close() }() tableList := []string{tblTestTemp + "2", tblTestTemp + "1", tblTestTemp + "3", tblTestTemp + "0"} for _, tbl := range tableList { - db.Exec(fmt.Sprintf("CREATE TABLE %s WITH PK=id:string", tbl)) + _, _ = db.Exec(fmt.Sprintf("CREATE TABLE %s WITH PK=id:string", tbl)) } dbresult, err := db.Query(`LIST TABLES`) @@ -124,7 +124,7 @@ func Test_Query_ListTables(t *testing.T) { func Test_Query_AlterTable(t *testing.T) { testName := "Test_Query_AlterTable" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _, err := db.Query(fmt.Sprintf("ALTER TABLE %s WITH wcu=0 WITH rcu=0", tblTestTemp)) if err == nil || strings.Index(err.Error(), "not supported") < 0 { @@ -136,9 +136,9 @@ func Test_Exec_AlterTable(t *testing.T) { testName := "Test_Exec_AlterTable" db := _openDb(t, testName) _initTest(db) - defer db.Close() + defer func() { _ = db.Close() }() - db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=id:string`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=id:string`, tblTestTemp)) testData := []struct { name string sql string @@ -185,7 +185,7 @@ func Test_Exec_AlterTable(t *testing.T) { func Test_Query_DropTable(t *testing.T) { testName := "Test_Query_DropTable" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _, err := db.Query(fmt.Sprintf("DROP TABLE %s", tblTestTemp)) if err == nil || strings.Index(err.Error(), "not supported") < 0 { @@ -197,9 +197,9 @@ func Test_Exec_DropTable(t *testing.T) { testName := "Test_Exec_DropTable" db := _openDb(t, testName) _initTest(db) - defer db.Close() + defer func() { _ = db.Close() }() - db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=id:string`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH PK=id:string`, tblTestTemp)) testData := []struct { name string sql string @@ -229,7 +229,7 @@ func Test_Exec_DropTable(t *testing.T) { func Test_Exec_DescribeTable(t *testing.T) { testName := "Test_Exec_DescribeTable" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _, err := db.Exec(fmt.Sprintf("DESCRIBE TABLE %s", tblTestTemp)) if err == nil || strings.Index(err.Error(), "not supported") < 0 { @@ -240,7 +240,7 @@ func Test_Exec_DescribeTable(t *testing.T) { func TestRowsDescribeTable_ColumnTypeDatabaseTypeName(t *testing.T) { testName := "TestRowsDescribeTable_ColumnTypeDatabaseTypeName" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) expected := map[string]struct { diff --git a/module_test/stmt_test.go b/module_test/stmt_test.go index fbc5daa..3108b48 100644 --- a/module_test/stmt_test.go +++ b/module_test/stmt_test.go @@ -3,12 +3,15 @@ package godynamo_test import ( "database/sql" "encoding/json" + "errors" "fmt" "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + "github.com/btnguyen2k/godynamo" "reflect" "strconv" "strings" "testing" + "time" "github.com/btnguyen2k/consu/reddo" "github.com/btnguyen2k/consu/semita" @@ -51,8 +54,10 @@ func _initTest(db *sql.DB) { _, _ = db.Exec(`DROP TABLE IF EXISTS ` + tblTestNotExist) _, _ = db.Exec(`DROP TABLE IF EXISTS ` + tblTestNotExists) _, _ = db.Exec(`DROP TABLE IF EXISTS ` + tblTestTemp) + _ = godynamo.WaitForTableStatus(nil, db, tblTestTemp, []string{""}, 100*time.Millisecond) for i := 0; i < 10; i++ { _, _ = db.Exec(`DROP TABLE IF EXISTS ` + tblTestTemp + strconv.Itoa(i)) + _ = godynamo.WaitForTableStatus(nil, db, tblTestTemp+strconv.Itoa(i), []string{""}, 100*time.Millisecond) } } @@ -259,7 +264,7 @@ func _fetchAllRows(dbRows *sql.Rows) ([]map[string]interface{}, error) { row[colTypes[i].Name()] = vals[i] } rows = append(rows, row) - } else if err != sql.ErrNoRows { + } else if !errors.Is(err, sql.ErrNoRows) { return nil, err } } diff --git a/module_test/tx_test.go b/module_test/tx_test.go index 1a4c9e1..989c7ed 100644 --- a/module_test/tx_test.go +++ b/module_test/tx_test.go @@ -9,10 +9,10 @@ import ( func TestTx_Rollback(t *testing.T) { testName := "TestTx_Rollback" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) - db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=1 WITH wcu=1`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=1 WITH wcu=1`, tblTestTemp)) tx, err := db.Begin() if err != nil { @@ -43,10 +43,10 @@ func TestTx_Rollback(t *testing.T) { func TestTx_Commit_Insert(t *testing.T) { testName := "TestTx_Commit_Insert" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) - db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=1 WITH wcu=1`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=1 WITH wcu=1`, tblTestTemp)) tx, err := db.Begin() if err != nil { @@ -105,12 +105,12 @@ func TestTx_Commit_Insert(t *testing.T) { func TestTx_Commit_UpdateDelete(t *testing.T) { testName := "TestTx_Commit_UpdateDelete" db := _openDb(t, testName) - defer db.Close() + defer func() { _ = db.Close() }() _initTest(db) - db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=1 WITH wcu=1`, tblTestTemp)) - db.Exec(fmt.Sprintf(`INSERT INTO "%s" VALUE {'id': ?, 'active': ?}`, tblTestTemp), "1", true) - db.Exec(fmt.Sprintf(`INSERT INTO "%s" VALUE {'id': ?, 'grade': ?}`, tblTestTemp), "2", 2) + _, _ = db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=1 WITH wcu=1`, tblTestTemp)) + _, _ = db.Exec(fmt.Sprintf(`INSERT INTO "%s" VALUE {'id': ?, 'active': ?}`, tblTestTemp), "1", true) + _, _ = db.Exec(fmt.Sprintf(`INSERT INTO "%s" VALUE {'id': ?, 'grade': ?}`, tblTestTemp), "2", 2) tx, err := db.Begin() if err != nil { diff --git a/module_test_real/godynamo_test.go b/module_test_real/godynamo_test.go index b85df6b..d42b000 100644 --- a/module_test_real/godynamo_test.go +++ b/module_test_real/godynamo_test.go @@ -2,22 +2,20 @@ package godynamo_test import ( "database/sql" - "github.com/btnguyen2k/consu/reddo" _ "github.com/btnguyen2k/godynamo" "os" - "reflect" "strings" "testing" ) -var ( - typeM = reflect.TypeOf(make(map[string]interface{})) - typeL = reflect.TypeOf(make([]interface{}, 0)) - typeS = reddo.TypeString - typeBool = reddo.TypeBool - typeN = reddo.TypeFloat - typeTime = reddo.TypeTime -) +//var ( +// typeM = reflect.TypeOf(make(map[string]interface{})) +// typeL = reflect.TypeOf(make([]interface{}, 0)) +// typeS = reddo.TypeString +// typeBool = reddo.TypeBool +// typeN = reddo.TypeFloat +// typeTime = reddo.TypeTime +//) func _openDb(t *testing.T, testName string) *sql.DB { driver := "godynamo" diff --git a/module_test_real/stmt_bigtable_test.go b/module_test_real/stmt_bigtable_test.go new file mode 100644 index 0000000..4d759ec --- /dev/null +++ b/module_test_real/stmt_bigtable_test.go @@ -0,0 +1,117 @@ +package godynamo_test + +import ( + "fmt" + "math/rand" + "strings" + "testing" + "time" +) + +func Test_BigTable(t *testing.T) { + testName := "Test_BigTable" + db := _openDb(t, testName) + defer func() { _ = db.Close() }() + //_initTest(db) + // + //if _, err := db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=7 WITH wcu=20`, tblTestTemp)); err != nil { + // t.Fatalf("%s failed: %s", testName+"/create_table", err) + //} + //ctx, cancelF := context.WithTimeout(context.Background(), 10*time.Second) + //defer cancelF() + //err := godynamo.WaitForTableStatus(ctx, db, tblTestTemp, []string{"ACTIVE"}, 500*time.Millisecond) + //if err != nil { + // t.Fatalf("%s failed: %s", testName+"/WaitForTableStatus", err) + //} + + type Row struct { + id string + dataChar string + dataVchar string + dataBinchar []byte + dataText string + dataUchar string + dataUvchar string + dataUtext string + dataClob string + dataUclob string + dataBlob []byte + } + rowArr := make([]Row, 0) + numRows := 100 + unicodeStr := "Chào buổi sáng, доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + unicodeStrLong := "Chào buổi sáng, đây sẽ là một đoạn văn bản dài. доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + sqlStm := `INSERT INTO "%s" VALUE {'id': ?, 'dataChar': ?, 'dataVchar': ?, 'dataBinchar': ?, 'dataText': ?, 'dataUchar': ?, 'dataUvchar': ?, 'dataUtext': ?, 'dataClob': ?, 'dataUclob': ?, 'dataBlob': ?}` + sqlStm = fmt.Sprintf(sqlStm, tblTestTemp) + for i := 1; i < numRows; i++ { + id := fmt.Sprintf("%03d", i) + row := Row{ + id: id, + dataChar: "CHAR " + id, + dataVchar: "VCHAR " + id, + dataBinchar: []byte("BINCHAR " + id), + dataText: strings.Repeat("This is supposed to be a long text ", i*2), + dataUchar: unicodeStr, + dataUvchar: unicodeStr, + dataUtext: strings.Repeat(unicodeStr, i*2), + dataClob: strings.Repeat("This is supposed to be a long text ", i*10), + dataUclob: strings.Repeat(unicodeStrLong, i*10), + dataBlob: []byte(strings.Repeat("This is supposed to be a long text ", i*10)), + } + rowArr = append(rowArr, row) + //params := []interface{}{row.id, row.dataChar, + // row.dataVchar, row.dataBinchar, row.dataText, row.dataUchar, row.dataUvchar, row.dataUtext, + // row.dataClob, row.dataUclob, row.dataBlob} + //_, err := db.Exec(sqlStm, params...) + //if err != nil { + // t.Fatalf("%s failed: %s", testName+"/insert", err) + //} + //time.Sleep(time.Duration(4000+rand.Int63n(2000)) * time.Millisecond) + //fmt.Printf("[DEBUG] %v\n", i) + } + + { + dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s`, tblTestTemp)) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/select", err) + } + rows, err := _fetchAllRows(dbrows) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) + } + if len(rows) != len(rowArr) { + t.Fatalf("%s failed: expected %d rows but received %d", testName, len(rowArr), len(rows)) + } + time.Sleep(time.Duration(5000+rand.Int63n(2000)) * time.Millisecond) + } + + { + dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s WHERE id>'012'`, tblTestTemp)) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/select", err) + } + rows, err := _fetchAllRows(dbrows) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) + } + if len(rows) != len(rowArr)-12 { + t.Fatalf("%s failed: expected %d rows but received %d", testName, len(rowArr)-12, len(rows)) + } + time.Sleep(time.Duration(5000+rand.Int63n(2000)) * time.Millisecond) + } + + { + limit := 13 + dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s LIMIT %d`, tblTestTemp, limit)) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/select", err) + } + rows, err := _fetchAllRows(dbrows) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) + } + if len(rows) != limit { + t.Fatalf("%s failed: expected %d rows but received %d", testName, limit, len(rows)) + } + } +} diff --git a/module_test_real/stmt_test.go b/module_test_real/stmt_test.go index a0a53e2..ee5dc97 100644 --- a/module_test_real/stmt_test.go +++ b/module_test_real/stmt_test.go @@ -53,10 +53,10 @@ func _initTest(db *sql.DB) { _, _ = db.Exec(`DROP TABLE IF EXISTS ` + tblTestNotExist) _, _ = db.Exec(`DROP TABLE IF EXISTS ` + tblTestNotExists) _, _ = db.Exec(`DROP TABLE IF EXISTS ` + tblTestTemp) - _ = godynamo.WaitForTableStatus(nil, db, tblTestTemp, []string{""}, 10*time.Second) + _ = godynamo.WaitForTableStatus(nil, db, tblTestTemp, []string{""}, 500*time.Millisecond) for i := 0; i < 10; i++ { _, _ = db.Exec(`DROP TABLE IF EXISTS ` + tblTestTemp + strconv.Itoa(i)) - _ = godynamo.WaitForTableStatus(nil, db, tblTestTemp+strconv.Itoa(i), []string{""}, 10*time.Second) + _ = godynamo.WaitForTableStatus(nil, db, tblTestTemp+strconv.Itoa(i), []string{""}, 500*time.Millisecond) } }