Pebble done

.github/CODEOWNERS

@@ -1,3 +1,3 @@
 # CODEOWNERS: https://help.github.com/articles/about-codeowners/
 
-*       @alexanderbez @cmwaters @creachadair @marbar3778 @tychoish @williambanfield
+*       @alexanderbez @cmwaters @marbar3778 @tychoish @williambanfield

.gitpod.yml

@@ -0,0 +1,7 @@
+# This configuration file was automatically generated by Gitpod.
+# Please adjust to your needs (see https://www.gitpod.io/docs/config-gitpod-file)
+# and commit this file to your remote git repository to share the goodness with others.
+
+image: tendermintdev/docker-tm-db-testing
+
+# this means that there's a one-click known good environment available to developers.

.golangci.yml

@@ -1,45 +1,40 @@
 linters:
   disable-all: true
   enable:
-    - bodyclose
     - deadcode
     - depguard
     - dogsled
     - dupl
     - errcheck
-    # - funlen
-    # - gochecknoglobals
-    # - gochecknoinits
+    - exportloopref
     - goconst
     - gocritic
-    # - gocyclo
-    # - godox
-    - gofmt
-    - goimports
+    - gofumpt
     - revive
     - gosec
     - gosimple
     - govet
     - ineffassign
-    # - interfacer
     - lll
     - misspell
     - nakedret
     - prealloc
-    # - scopelint
     - staticcheck
-    - structcheck
     - stylecheck
     - typecheck
+    - revive
     - unconvert
-    # - unparam
     - unused
     - varcheck
-    # - whitespace
-    # - wsl
-    # - gocognit
     - nolintlint
 
+run:
+  build-tags:
+    - cleveldb
+    - rocksdb
+    - badgerdb
+    - boltdb
+
 issues:
   exclude-rules:
     - path: _test\.go

CHANGELOG.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## Unreleased
+
+- remove mutex from prefixdb
+- [#284](https://github.com/tendermint/tm-db/pull/284): Add pebbledb, a rocks-like database that doesn't need to use cgo.
+
 ## 0.6.7
 
 **2022-2-21**

backend_test.go

@@ -315,7 +315,6 @@ func testDBIterator(t *testing.T, backend BackendType) {
 	ritr, err = db2.ReverseIterator(nil, nil)
 	require.NoError(t, err)
 	verifyIterator(t, ritr, nil, "reverse iterator with empty db")
-
 }
 
 func verifyIterator(t *testing.T, itr Iterator, expected []int64, msg string) {

badger_db.go

@@ -25,7 +25,7 @@ func NewBadgerDB(dbName, dir string) (*BadgerDB, error) {
 	// the final directory to use for the database.
 	path := filepath.Join(dir, dbName)
 
-	if err := os.MkdirAll(path, 0755); err != nil {
+	if err := os.MkdirAll(path, 0o755); err != nil {
 		return nil, err
 	}
 	opts := badger.DefaultOptions(path)

boltdb.go

@@ -1,3 +1,4 @@
+//go:build boltdb
 // +build boltdb
 
 package db
@@ -11,14 +12,10 @@ import (
 	"go.etcd.io/bbolt"
 )
 
-var (
-	bucket = []byte("tm")
-)
+var bucket = []byte("tm")
 
 func init() {
-	registerDBCreator(BoltDBBackend, func(name, dir string) (DB, error) {
-		return NewBoltDB(name, dir)
-	}, false)
+	registerDBCreator(BoltDBBackend, NewBoltDB, false)
 }
 
 // BoltDB is a wrapper around etcd's fork of bolt (https://github.com/etcd-io/bbolt).
@@ -139,6 +136,7 @@ func (bdb *BoltDB) Close() error {
 }
 
 // Print implements DB.
+// nolint: errcheck
 func (bdb *BoltDB) Print() error {
 	stats := bdb.db.Stats()
 	fmt.Printf("%v\n", stats)

boltdb_batch.go

@@ -1,3 +1,4 @@
+//go:build boltdb
 // +build boltdb
 
 package db

boltdb_iterator.go

@@ -1,3 +1,4 @@
+//go:build boltdb
 // +build boltdb
 
 package db

boltdb_test.go

@@ -1,10 +1,12 @@
+//go:build boltdb
 // +build boltdb
 
 package db
 
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -20,6 +22,16 @@ func TestBoltDBNewBoltDB(t *testing.T) {
 	db.Close()
 }
 
+func TestWithBoltDB(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "boltdb")
+
+	db, err := NewBoltDB(path, "")
+	require.NoError(t, err)
+
+	t.Run("BoltDB", func(t *testing.T) { Run(t, db) })
+}
+
 func BenchmarkBoltDBRandomReadsWrites(b *testing.B) {
 	name := fmt.Sprintf("test_%x", randStr(12))
 	db, err := NewBoltDB(name, "")

cleveldb.go

@@ -1,3 +1,4 @@
+//go:build cleveldb
 // +build cleveldb
 
 package db
@@ -153,14 +154,10 @@ func (db *CLevelDB) Print() error {
 // Stats implements DB.
 func (db *CLevelDB) Stats() map[string]string {
 	keys := []string{
-		"leveldb.aliveiters",
-		"leveldb.alivesnaps",
-		"leveldb.blockpool",
-		"leveldb.cachedblock",
 		"leveldb.num-files-at-level{n}",
-		"leveldb.openedtables",
 		"leveldb.sstables",
 		"leveldb.stats",
+		"leveldb.approximate-memory-usage",
 	}
 
 	stats := make(map[string]string, len(keys))

cleveldb_batch.go

@@ -1,3 +1,4 @@
+//go:build cleveldb
 // +build cleveldb
 
 package db

cleveldb_iterator.go

@@ -1,3 +1,4 @@
+//go:build cleveldb
 // +build cleveldb
 
 package db
@@ -20,7 +21,7 @@ var _ Iterator = (*cLevelDBIterator)(nil)
 
 func newCLevelDBIterator(source *levigo.Iterator, start, end []byte, isReverse bool) *cLevelDBIterator {
 	if isReverse {
-		if end == nil || len(end) == 0 {
+		if len(end) == 0 {
 			source.SeekToLast()
 		} else {
 			source.Seek(end)
@@ -34,7 +35,7 @@ func newCLevelDBIterator(source *levigo.Iterator, start, end []byte, isReverse b
 			}
 		}
 	} else {
-		if start == nil || len(start) == 0 {
+		if len(start) == 0 {
 			source.SeekToFirst()
 		} else {
 			source.Seek(start)
@@ -56,36 +57,31 @@ func (itr cLevelDBIterator) Domain() ([]byte, []byte) {
 
 // Valid implements Iterator.
 func (itr cLevelDBIterator) Valid() bool {
-
 	// Once invalid, forever invalid.
 	if itr.isInvalid {
 		return false
 	}
 
 	// If source errors, invalid.
 	if itr.source.GetError() != nil {
-		itr.isInvalid = true
 		return false
 	}
 
 	// If source is invalid, invalid.
 	if !itr.source.Valid() {
-		itr.isInvalid = true
 		return false
 	}
 
 	// If key is end or past it, invalid.
-	var start = itr.start
-	var end = itr.end
-	var key = itr.source.Key()
+	start := itr.start
+	end := itr.end
+	key := itr.source.Key()
 	if itr.isReverse {
 		if start != nil && bytes.Compare(key, start) < 0 {
-			itr.isInvalid = true
 			return false
 		}
 	} else {
 		if end != nil && bytes.Compare(end, key) <= 0 {
-			itr.isInvalid = true
 			return false
 		}
 	}

cleveldb_test.go

@@ -1,3 +1,4 @@
+//go:build cleveldb
 // +build cleveldb
 
 package db
@@ -7,12 +8,24 @@ import (
 	"fmt"
 	"math/rand"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
 
+func TestWithClevelDB(t *testing.T) {
+	dir := t.TempDir()
+	path := filepath.Join(dir, "cleveldb")
+
+	db, err := NewCLevelDB(path, "")
+	require.NoError(t, err)
+
+	t.Run("ClevelDB", func(t *testing.T) { Run(t, db) })
+}
+
+//nolint: errcheck
 func BenchmarkRandomReadsWrites2(b *testing.B) {
 	b.StopTimer()
 
@@ -36,9 +49,8 @@ func BenchmarkRandomReadsWrites2(b *testing.B) {
 			idx := (int64(rand.Int()) % numItems)
 			internal[idx]++
 			val := internal[idx]
-			idxBytes := int642Bytes(int64(idx))
-			valBytes := int642Bytes(int64(val))
-			//fmt.Printf("Set %X -> %X\n", idxBytes, valBytes)
+			idxBytes := int642Bytes(idx)
+			valBytes := int642Bytes(val)
 			db.Set(
 				idxBytes,
 				valBytes,
@@ -48,12 +60,12 @@ func BenchmarkRandomReadsWrites2(b *testing.B) {
 		{
 			idx := (int64(rand.Int()) % numItems)
 			val := internal[idx]
-			idxBytes := int642Bytes(int64(idx))
+			idxBytes := int642Bytes(idx)
 			valBytes, err := db.Get(idxBytes)
 			if err != nil {
 				b.Error(err)
 			}
-			//fmt.Printf("Get %X -> %X\n", idxBytes, valBytes)
+			// fmt.Printf("Get %X -> %X\n", idxBytes, valBytes)
 			if val == 0 {
 				if !bytes.Equal(valBytes, nil) {
 					b.Errorf("Expected %v for %v, got %X",

db.go

@@ -28,13 +28,19 @@ const (
 	//   - may be faster is some use-cases (random reads - indexer)
 	//   - use boltdb build tag (go build -tags boltdb)
 	BoltDBBackend BackendType = "boltdb"
-	// RocksDBBackend represents rocksdb (uses github.com/cosmos/gorocksdb)
+	// RocksDBBackend represents rocksdb (uses github.com/linxGnu/grocksdb)
 	//   - EXPERIMENTAL
 	//   - requires gcc
 	//   - use rocksdb build tag (go build -tags rocksdb)
 	RocksDBBackend BackendType = "rocksdb"
-
+	// BadgerDBBackend represents badgerdb (uses github.com/dgraph-io/badger)
+	//   - EXPERIMENTAL
+	//   - use badgerdb build tag (go build -tags badgerdb)
 	BadgerDBBackend BackendType = "badgerdb"
+	// PebbleDBDBBackend represents pebble (uses github.com/cockroachdb/pebble)
+	//   - EXPERIMENTAL
+	//   - use pebble build tag (go build -tags pebbledb)
+	PebbleDBBackend BackendType = "pebbledb"
 )
 
 type dbCreator func(name string, dir string) (DB, error)

go.mod

@@ -4,32 +4,46 @@ go 1.17
 
 require (
 	github.com/cosmos/gorocksdb v1.2.0
+	github.com/dgraph-io/badger/v3 v3.2103.2
 	github.com/gogo/protobuf v1.3.2
-	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/google/btree v1.1.2
 	github.com/jmhodges/levigo v1.0.0
 	github.com/stretchr/testify v1.8.0
 	github.com/syndtr/goleveldb v1.0.1-0.20200815110645-5c35d600f0ca
 	go.etcd.io/bbolt v1.3.6
 	google.golang.org/grpc v1.48.0
+  github.com/cockroachdb/pebble v0.0.0-20220726134658-7b78c71e4055
+	github.com/dgraph-io/badger/v3 v3.2103.2
 )
 
-require github.com/dgraph-io/badger/v3 v3.2103.2
+
+
+
 
 require (
+	github.com/DataDog/zstd v1.4.5 // indirect
+	github.com/HdrHistogram/hdrhistogram-go v1.1.2 // indirect
 	github.com/cespare/xxhash v1.1.0 // indirect
 	github.com/cespare/xxhash/v2 v2.1.1 // indirect
+	github.com/cockroachdb/errors v1.8.1 // indirect
+	github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect
+	github.com/cockroachdb/redact v1.0.8 // indirect
+	github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/dgraph-io/ristretto v0.1.0 // indirect
 	github.com/dustin/go-humanize v1.0.0 // indirect
 	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
 	github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/google/flatbuffers v1.12.1 // indirect
 	github.com/klauspost/compress v1.12.3 // indirect
+	github.com/kr/pretty v0.1.0 // indirect
+	github.com/kr/text v0.2.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.opencensus.io v0.22.5 // indirect
+	golang.org/x/exp v0.0.0-20200513190911-00229845015e // indirect
 	golang.org/x/net v0.0.0-20201021035429-f5854403a974 // indirect
 	golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9 // indirect
 	golang.org/x/text v0.3.3 // indirect