Merge pull request #879 from iotaledger/develop

Merge v0.3.4 changes to master

CHANGELOG.md

+# v0.3.4 - 2020-12-11
+* Revert Pebble to Badger.
+* **Breaking**: bumps network and database versions
+
 # v0.3.3 - 2020-12-10
 * Fix sync issue.
 * Fix pkger issue.

go.mod

@@ -5,7 +5,7 @@ go 1.14
 require (
 	github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect
 	github.com/beevik/ntp v0.3.0
-	github.com/cockroachdb/pebble v0.0.0-20201130172119-f19faf8529d6
+	github.com/dgraph-io/badger/v2 v2.0.3
 	github.com/drand/drand v1.1.1
 	github.com/drand/kyber v1.1.2
 	github.com/gin-gonic/gin v1.6.3

packages/database/pebbledb.go → packages/database/badgerdb.go

@@ -5,14 +5,16 @@ import (
 	"os"
 	"runtime"
 
-	pebbledb "github.com/cockroachdb/pebble"
-	"github.com/cockroachdb/pebble/bloom"
+	"github.com/dgraph-io/badger/v2"
+	"github.com/dgraph-io/badger/v2/options"
 	"github.com/iotaledger/hive.go/kvstore"
-	"github.com/iotaledger/hive.go/kvstore/pebble"
+	badgerstore "github.com/iotaledger/hive.go/kvstore/badger"
 )
 
-type pebbleDB struct {
-	*pebbledb.DB
+const valueLogGCDiscardRatio = 0.1
+
+type badgerDB struct {
+	*badger.DB
 }
 
 // NewDB returns a new persisting DB object.
@@ -23,61 +25,50 @@ func NewDB(dirname string) (DB, error) {
 		return nil, fmt.Errorf("could not create DB directory: %w", err)
 	}
 
-	cache := pebbledb.NewCache(50000000)
-	defer cache.Unref()
-
-	opts := &pebbledb.Options{
-		Cache:                       cache,
-		DisableWAL:                  false,
-		L0CompactionThreshold:       2,
-		L0StopWritesThreshold:       1000,
-		LBaseMaxBytes:               64 << 20, // 64 MB
-		Levels:                      make([]pebbledb.LevelOptions, 7),
-		MaxConcurrentCompactions:    3,
-		MaxOpenFiles:                16384,
-		MemTableSize:                64 << 20,
-		MemTableStopWritesThreshold: 4,
-	}
+	opts := badger.DefaultOptions(dirname)
 
-	for i := 0; i < len(opts.Levels); i++ {
-		l := &opts.Levels[i]
-		l.BlockSize = 32 << 10       // 32 KB
-		l.IndexBlockSize = 256 << 10 // 256 KB
-		l.FilterPolicy = bloom.FilterPolicy(10)
-		l.FilterType = pebbledb.TableFilter
-		if i > 0 {
-			l.TargetFileSize = opts.Levels[i-1].TargetFileSize * 2
-		}
-		l.EnsureDefaults()
-	}
-	opts.Levels[6].FilterPolicy = nil
+	opts.Logger = nil
+	opts.SyncWrites = false
+	opts.TableLoadingMode = options.MemoryMap
+	opts.ValueLogLoadingMode = options.MemoryMap
+	opts.CompactL0OnClose = false
+	opts.KeepL0InMemory = false
+	opts.VerifyValueChecksum = false
+	opts.ZSTDCompressionLevel = 1
+	opts.Compression = options.None
+	opts.MaxCacheSize = 50000000
+	opts.EventLogging = false
 
-	opts.EnsureDefaults()
+	if runtime.GOOS == "windows" {
+		opts = opts.WithTruncate(true)
+	}
 
-	db, err := pebble.CreateDB(dirname, opts)
+	db, err := badger.Open(opts)
 	if err != nil {
 		return nil, fmt.Errorf("could not open DB: %w", err)
 	}
-	return &pebbleDB{DB: db}, nil
+
+	return &badgerDB{DB: db}, nil
 }
 
-func (d *pebbleDB) NewStore() kvstore.KVStore {
-	return pebble.New(d.DB)
+func (db *badgerDB) NewStore() kvstore.KVStore {
+	return badgerstore.New(db.DB)
 }
 
 // Close closes a DB. It's crucial to call it to ensure all the pending updates make their way to disk.
-func (d *pebbleDB) Close() error {
-	if err := d.DB.Close(); err != nil {
-		return err
-	}
-	return nil
+func (db *badgerDB) Close() error {
+	return db.DB.Close()
 }
 
-func (d *pebbleDB) RequiresGC() bool {
+func (db *badgerDB) RequiresGC() bool {
 	return true
 }
 
-func (d *pebbleDB) GC() error {
+func (db *badgerDB) GC() error {
+	err := db.RunValueLogGC(valueLogGCDiscardRatio)
+	if err != nil {
+		return err
+	}
 	// trigger the go garbage collector to release the used memory
 	runtime.GC()
 	return nil

packages/tangle/tangle_test.go

@@ -96,8 +96,8 @@ func TestTangle_MissingMessages(t *testing.T) {
 		attachDelay  = 5 * time.Millisecond
 	)
 
-	// create pebble store
-	pebbleDB, err := testutil.PebbleDB(t)
+	// create badger store
+	badgerDB, err := testutil.BadgerDB(t)
 	require.NoError(t, err)
 
 	// map to keep track of the tips
@@ -106,7 +106,7 @@ func TestTangle_MissingMessages(t *testing.T) {
 
 	// setup the message factory
 	msgFactory := NewMessageFactory(
-		pebbleDB,
+		badgerDB,
 		[]byte("sequenceKey"),
 		identity.GenerateLocalIdentity(),
 		TipSelectorFunc(func(count int) []MessageID {
@@ -143,7 +143,7 @@ func TestTangle_MissingMessages(t *testing.T) {
 	}
 
 	// create the tangle
-	tangle := New(pebbleDB)
+	tangle := New(badgerDB)
 	defer tangle.Shutdown()
 	require.NoError(t, tangle.Prune())
 

plugins/autopeering/parameters.go

@@ -13,5 +13,5 @@ const (
 
 func init() {
 	flag.StringSlice(CfgEntryNodes, []string{"2PV5487xMw5rasGBXXWeqSi4hLz7r19YBt8Y1TGAsQbj@ressims.iota.cafe:15626", "5EDH4uY78EA6wrBkHHAVBWBMDt7EcksRq6pjzipoW15B@entrynode.alphanet.tanglebay.org:14656"}, "list of trusted entry nodes for auto peering")
-	flag.Int(CfgNetworkVersion, 11, "autopeering network version")
+	flag.Int(CfgNetworkVersion, 12, "autopeering network version")
 }

plugins/banner/plugin.go

@@ -17,7 +17,7 @@ var (
 	once   sync.Once
 
 	// AppVersion version number
-	AppVersion = "v0.3.3"
+	AppVersion = "v0.3.4"
 	// SimplifiedAppVersion is the version number without commit hash
 	SimplifiedAppVersion = simplifiedVersion(AppVersion)
 )

plugins/database/versioning.go

@@ -10,7 +10,7 @@ import (
 const (
 	// DBVersion defines the version of the database schema this version of GoShimmer supports.
 	// Every time there's a breaking change regarding the stored data, this version flag should be adjusted.
-	DBVersion = 13
+	DBVersion = 14
 )
 
 var (