From c49f75c9b9d340c64f0757a95c7d07552541a7fe Mon Sep 17 00:00:00 2001 From: Thanh Nguyen Date: Tue, 2 Jan 2024 20:50:20 +1100 Subject: [PATCH] Docs: update docs --- README.md | 11 ++ SQL_DOCUMENT.md | 6 +- module_test/stmt_bigtable_test.go | 233 +++++++++++++++++++++++++++--- 3 files changed, 226 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 7004e40..8c98d3f 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,17 @@ fmt.Println("RowsAffected:", rowsAffected2) // output "RowsAffected: 1" > > You can use [EXISTS function](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-functions.exists.html) for condition checking. +## Caveats + +**Numerical values** are stored in DynamoDB as floating point numbers. Hence, numbers are always read back as `float64`. +See [DynamoDB document](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/HowItWorks.NamingRulesDataTypes.html#HowItWorks.DataTypes) for details on DynamoDB's supported data types. + +**A single query can only return up to [1MB of data](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Query.Pagination.html)**. +In the case of `SELECT` query, the driver automatically issues additional queries to fetch the remaining data if needed. +However, returned rows may not be in the expected order specified by `ORDER BY` clause. +That means, rows returned from the query `SELECT * FROM table_name WHERE category='Laptop' ORDER BY id` may not be in +the expected order if all matched rows do not fit in 1MB of data. + ## License This project is licensed under the MIT License - see the [LICENSE.md](LICENSE.md) file for details. diff --git a/SQL_DOCUMENT.md b/SQL_DOCUMENT.md index 320f2eb..9e9da29 100644 --- a/SQL_DOCUMENT.md +++ b/SQL_DOCUMENT.md @@ -49,14 +49,16 @@ Sample result: > > dbrows, err := db.Query(`SELECT * FROM "session" WHERE app='frontend' LIMIT 10`) > -> Note: the value for `LIMIT` must be a positive integer. +> Note: +> - The `LIMIT` clause is extension offered by `godynamodb` and is not part of [PartiQL syntax](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/ql-reference.select.html). +> - The value for `LIMIT` must be a _positive integer_. > Since [v0.4.0](RELEASE-NOTES.md), `godynamodb` supports ConsistentRead for `SELECT` statement via clause `WITH ConsistentRead=true` or `WITH Consistent_Read=true`. > Example: > > dbrows, err := db.Query(`SELECT * FROM "session" WHERE app='frontend' WITH ConsistentRead=true`) > -> Note: the WITH clause must be placed at the end of the SELECT statement. +> Note: the WITH clause must be placed _at the end_ of the SELECT statement. ## UPDATE diff --git a/module_test/stmt_bigtable_test.go b/module_test/stmt_bigtable_test.go index aaaa5ea..ca24e7f 100644 --- a/module_test/stmt_bigtable_test.go +++ b/module_test/stmt_bigtable_test.go @@ -2,6 +2,7 @@ package godynamo_test import ( "fmt" + "math/rand" "strings" "testing" ) @@ -59,37 +60,214 @@ func Test_BigTable(t *testing.T) { } } - { - dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s`, tblTestTemp)) - if err != nil { - t.Fatalf("%s failed: %s", testName+"/select", err) + dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s`, tblTestTemp)) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/select", err) + } + rows, err := _fetchAllRows(dbrows) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) + } + if len(rows) != len(rowArr) { + t.Fatalf("%s failed: expected %d rows but received %d", testName, len(rowArr), len(rows)) + } +} + +func Test_BigTable_withWHERE(t *testing.T) { + testName := "Test_BigTable_withWHERE" + db := _openDb(t, testName) + defer func() { _ = db.Close() }() + _initTest(db) + + if _, err := db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=3 WITH wcu=5`, tblTestTemp)); err != nil { + t.Fatalf("%s failed: %s", testName+"/create_table", err) + } + type Row struct { + id string + dataChar string + dataVchar string + dataBinchar []byte + dataText string + dataUchar string + dataUvchar string + dataUtext string + dataClob string + dataUclob string + dataBlob []byte + } + rowArr := make([]Row, 0) + numRows := 100 + unicodeStr := "Chào buổi sáng, доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + unicodeStrLong := "Chào buổi sáng, đây sẽ là một đoạn văn bản dài. доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + sqlStm := `INSERT INTO "%s" VALUE {'id': ?, 'dataChar': ?, 'dataVchar': ?, 'dataBinchar': ?, 'dataText': ?, 'dataUchar': ?, 'dataUvchar': ?, 'dataUtext': ?, 'dataClob': ?, 'dataUclob': ?, 'dataBlob': ?}` + sqlStm = fmt.Sprintf(sqlStm, tblTestTemp) + for i := 1; i < numRows; i++ { + id := fmt.Sprintf("%03d", i) + row := Row{ + id: id, + dataChar: "CHAR " + id, + dataVchar: "VCHAR " + id, + dataBinchar: []byte("BINCHAR " + id), + dataText: strings.Repeat("This is supposed to be a long text ", i*2), + dataUchar: unicodeStr, + dataUvchar: unicodeStr, + dataUtext: strings.Repeat(unicodeStr, i*2), + dataClob: strings.Repeat("This is supposed to be a long text ", i*10), + dataUclob: strings.Repeat(unicodeStrLong, i*10), + dataBlob: []byte(strings.Repeat("This is supposed to be a long text ", i*10)), } - rows, err := _fetchAllRows(dbrows) + rowArr = append(rowArr, row) + params := []interface{}{row.id, row.dataChar, + row.dataVchar, row.dataBinchar, row.dataText, row.dataUchar, row.dataUvchar, row.dataUtext, + row.dataClob, row.dataUclob, row.dataBlob} + _, err := db.Exec(sqlStm, params...) if err != nil { - t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) - } - if len(rows) != len(rowArr) { - t.Fatalf("%s failed: expected %d rows but received %d", testName, len(rowArr), len(rows)) + t.Fatalf("%s failed: %s", testName+"/insert", err) } } - { - dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s WHERE id>'012'`, tblTestTemp)) - if err != nil { - t.Fatalf("%s failed: %s", testName+"/select", err) + dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s WHERE id>'012'`, tblTestTemp)) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/select", err) + } + rows, err := _fetchAllRows(dbrows) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) + } + if len(rows) != len(rowArr)-12 { + t.Fatalf("%s failed: expected %d rows but received %d", testName, len(rowArr)-12, len(rows)) + } +} + +func Test_BigTable_withLIMIT(t *testing.T) { + testName := "Test_BigTable_withLIMIT" + db := _openDb(t, testName) + defer func() { _ = db.Close() }() + _initTest(db) + + if _, err := db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=id:string WITH rcu=3 WITH wcu=5`, tblTestTemp)); err != nil { + t.Fatalf("%s failed: %s", testName+"/create_table", err) + } + type Row struct { + id string + dataChar string + dataVchar string + dataBinchar []byte + dataText string + dataUchar string + dataUvchar string + dataUtext string + dataClob string + dataUclob string + dataBlob []byte + } + rowArr := make([]Row, 0) + numRows := 100 + unicodeStr := "Chào buổi sáng, доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + unicodeStrLong := "Chào buổi sáng, đây sẽ là một đoạn văn bản dài. доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + sqlStm := `INSERT INTO "%s" VALUE {'id': ?, 'dataChar': ?, 'dataVchar': ?, 'dataBinchar': ?, 'dataText': ?, 'dataUchar': ?, 'dataUvchar': ?, 'dataUtext': ?, 'dataClob': ?, 'dataUclob': ?, 'dataBlob': ?}` + sqlStm = fmt.Sprintf(sqlStm, tblTestTemp) + for i := 1; i < numRows; i++ { + id := fmt.Sprintf("%03d", i) + row := Row{ + id: id, + dataChar: "CHAR " + id, + dataVchar: "VCHAR " + id, + dataBinchar: []byte("BINCHAR " + id), + dataText: strings.Repeat("This is supposed to be a long text ", i*2), + dataUchar: unicodeStr, + dataUvchar: unicodeStr, + dataUtext: strings.Repeat(unicodeStr, i*2), + dataClob: strings.Repeat("This is supposed to be a long text ", i*10), + dataUclob: strings.Repeat(unicodeStrLong, i*10), + dataBlob: []byte(strings.Repeat("This is supposed to be a long text ", i*10)), } - rows, err := _fetchAllRows(dbrows) + rowArr = append(rowArr, row) + params := []interface{}{row.id, row.dataChar, + row.dataVchar, row.dataBinchar, row.dataText, row.dataUchar, row.dataUvchar, row.dataUtext, + row.dataClob, row.dataUclob, row.dataBlob} + _, err := db.Exec(sqlStm, params...) if err != nil { - t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) + t.Fatalf("%s failed: %s", testName+"/insert", err) } - if len(rows) != len(rowArr)-12 { - t.Fatalf("%s failed: expected %d rows but received %d", testName, len(rowArr)-12, len(rows)) + } + + limit := 13 + dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s LIMIT %d`, tblTestTemp, limit)) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/select", err) + } + rows, err := _fetchAllRows(dbrows) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) + } + if len(rows) != limit { + t.Fatalf("%s failed: expected %d rows but received %d", testName, limit, len(rows)) + } +} + +func Test_BigTable_withORDERBY(t *testing.T) { + testName := "Test_BigTable_withORDERBY" + db := _openDb(t, testName) + defer func() { _ = db.Close() }() + _initTest(db) + + if _, err := db.Exec(fmt.Sprintf(`CREATE TABLE %s WITH pk=category:string WITH sk=id:string WITH rcu=11 WITH wcu=11`, tblTestTemp)); err != nil { + t.Fatalf("%s failed: %s", testName+"/create_table", err) + } + catList := []string{"PC", "Laptop", "Tablet", "Other"} + rand.Shuffle(len(catList), func(i, j int) { catList[i], catList[j] = catList[j], catList[i] }) + catCount := map[string]int{"PC": 0, "Laptop": 0, "Tablet": 0, "Other": 0} + type Row struct { + id string + category string + dataChar string + dataVchar string + dataBinchar []byte + dataText string + dataUchar string + dataUvchar string + dataUtext string + dataClob string + dataUclob string + dataBlob []byte + } + rowArr := make([]Row, 0) + numRows := 100 + unicodeStr := "Chào buổi sáng, доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + unicodeStrLong := "Chào buổi sáng, đây sẽ là một đoạn văn bản dài. доброе утро, ສະ​ບາຍ​ດີ​ຕອນ​ເຊົ້າ, สวัสดีตอนเช้า" + sqlStm := `INSERT INTO "%s" VALUE {'id': ?, 'category': ?, 'dataChar': ?, 'dataVchar': ?, 'dataBinchar': ?, 'dataText': ?, 'dataUchar': ?, 'dataUvchar': ?, 'dataUtext': ?, 'dataClob': ?, 'dataUclob': ?, 'dataBlob': ?}` + sqlStm = fmt.Sprintf(sqlStm, tblTestTemp) + for i := 1; i < numRows; i++ { + id := fmt.Sprintf("%03d", i) + cat := catList[rand.Intn(len(catList))] + catCount[cat]++ + row := Row{ + id: id, + category: cat, + dataChar: "CHAR " + id, + dataVchar: "VCHAR " + id, + dataBinchar: []byte("BINCHAR " + id), + dataText: strings.Repeat("This is supposed to be a long text ", i*2), + dataUchar: unicodeStr, + dataUvchar: unicodeStr, + dataUtext: strings.Repeat(unicodeStr, i*2), + dataClob: strings.Repeat("This is supposed to be a long text ", i*10), + dataUclob: strings.Repeat(unicodeStrLong, i*10), + dataBlob: []byte(strings.Repeat("This is supposed to be a long text ", i*10)), + } + rowArr = append(rowArr, row) + params := []interface{}{row.id, row.category, + row.dataChar, row.dataVchar, row.dataBinchar, row.dataText, row.dataUchar, row.dataUvchar, row.dataUtext, + row.dataClob, row.dataUclob, row.dataBlob} + _, err := db.Exec(sqlStm, params...) + if err != nil { + t.Fatalf("%s failed: %s", testName+"/insert", err) } } - { - limit := 13 - dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s LIMIT %d`, tblTestTemp, limit)) + for _, cat := range catList { + dbrows, err := db.Query(fmt.Sprintf(`SELECT * FROM %s WHERE category=? ORDER BY id DESC`, tblTestTemp), cat) if err != nil { t.Fatalf("%s failed: %s", testName+"/select", err) } @@ -97,8 +275,19 @@ func Test_BigTable(t *testing.T) { if err != nil { t.Fatalf("%s failed: %s", testName+"/fetchAllRows", err) } - if len(rows) != limit { - t.Fatalf("%s failed: expected %d rows but received %d", testName, limit, len(rows)) + if catCount[cat] != len(rows) { + t.Fatalf("%s failed: expected %d rows but received %d", testName, catCount[cat], len(rows)) + } + for i, row := range rows { + fmt.Printf("[DEBUG] %2d: %5s - %#v\n", i, row["category"], row["id"]) + if row["category"] != cat { + t.Fatalf("%s failed: expected category %s but received %s", testName, cat, row["category"]) + } + if i > 0 { + if row["id"].(string) > rows[i-1]["id"].(string) { + t.Fatalf("%s failed: expected id %s < %s", testName, row["id"], rows[i-1]["id"]) + } + } } } }