diff --git a/go.mod b/go.mod
index 0564d83750563587252c4c190bc10c030a072a58..d0e1df79114c59810d2c315e1ffc1275bfec78a0 100644
--- a/go.mod
+++ b/go.mod
@@ -3,8 +3,7 @@ module github.com/iotaledger/goshimmer
go 1.13
require (
- github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 // indirect
- github.com/dgraph-io/badger v1.6.0
+ github.com/dgraph-io/badger/v2 v2.0.1
github.com/dgrijalva/jwt-go v3.2.0+incompatible
github.com/dgryski/go-farm v0.0.0-20191112170834-c2139c5d712b // indirect
github.com/gdamore/tcell v1.3.0
@@ -13,7 +12,7 @@ require (
github.com/googollee/go-engine.io v1.4.3-0.20190924125625-798118fc0dd2
github.com/googollee/go-socket.io v1.4.3-0.20191204093753-683f8725b6d0
github.com/gorilla/websocket v1.4.1
- github.com/iotaledger/hive.go v0.0.0-20200120092048-f168257b6ccc
+ github.com/iotaledger/hive.go v0.0.0-20200120174440-057de3927083
github.com/iotaledger/iota.go v1.0.0-beta.14
github.com/labstack/echo v3.3.10+incompatible
github.com/labstack/gommon v0.3.0 // indirect
diff --git a/go.sum b/go.sum
index 6cba4d494385fe85e220a35a9c3824cd03edc7bd..3664b27905c74ce03f34db0ca1e2e8ad939f2f08 100644
--- a/go.sum
+++ b/go.sum
@@ -7,12 +7,12 @@ dmitri.shuralyov.com/service/change v0.0.0-20181023043359-a85b471d5412/go.mod h1
dmitri.shuralyov.com/state v0.0.0-20180228185332-28bcc343414c/go.mod h1:0PRwlb0D6DFvNNtx+9ybjezNCa8XF0xaYcETyp6rHWU=
git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg=
github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
-github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M=
-github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
+github.com/DataDog/zstd v1.4.1 h1:3oxKN3wbHibqx897utPC2LTQU4J+IHWWJO+glkAkpFM=
github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
+github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE=
github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
@@ -36,10 +36,11 @@ github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwc
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dgraph-io/badger v1.5.4 h1:gVTrpUTbbr/T24uvoCaqY2KSHfNLVGm0w+hbee2HMeg=
github.com/dgraph-io/badger v1.5.4/go.mod h1:VZxzAIRPHRVNRKRo6AXrX9BJegn6il06VMTZVJYCIjQ=
-github.com/dgraph-io/badger v1.6.0 h1:DshxFxZWXUcO0xX476VJC07Xsr6ZCBVRHKZ93Oh7Evo=
-github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4=
-github.com/dgraph-io/badger/v2 v2.0.0/go.mod h1:YoRSIp1LmAJ7zH7tZwRvjNMUYLxB4wl3ebYkaIruZ04=
+github.com/dgraph-io/badger/v2 v2.0.1 h1:+D6dhIqC6jIeCclnxMHqk4HPuXgrRN5UfBsLR4dNQ3A=
+github.com/dgraph-io/badger/v2 v2.0.1/go.mod h1:YoRSIp1LmAJ7zH7tZwRvjNMUYLxB4wl3ebYkaIruZ04=
+github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e h1:aeUNgwup7PnDOBAD1BOKAqzb/W/NksOj6r3dwKKuqfg=
github.com/dgraph-io/ristretto v0.0.0-20191025175511-c1f00be0418e/go.mod h1:edzKIzGvqUCMzhTVWbiTSe75zD9Xxq0GtSBtFmaUTZs=
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
@@ -79,6 +80,7 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs=
github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
github.com/gomodule/redigo v2.0.0+incompatible/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4=
github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
@@ -116,8 +118,8 @@ github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
-github.com/iotaledger/hive.go v0.0.0-20200120092048-f168257b6ccc h1:7SZQ3+4EvMd/iSOONrvD7Sa7qCm712KqTTKh8CK/9TU=
-github.com/iotaledger/hive.go v0.0.0-20200120092048-f168257b6ccc/go.mod h1:obs07gqna53/Yw1ltzLsQzJBMyA6lGu7Fb/ltjqWMnQ=
+github.com/iotaledger/hive.go v0.0.0-20200120174440-057de3927083 h1:dQx6NHouYh3KBStOuYrgMm6wYNnf4L+grKWQV4iBxNI=
+github.com/iotaledger/hive.go v0.0.0-20200120174440-057de3927083/go.mod h1:S+v90R3u4Rqe4VoOg4DhiZrAKlKZhz2UFKuK/Neqa2o=
github.com/iotaledger/iota.go v1.0.0-beta.9/go.mod h1:F6WBmYd98mVjAmmPVYhnxg8NNIWCjjH8VWT9qvv3Rc8=
github.com/iotaledger/iota.go v1.0.0-beta.14 h1:Oeb28MfBuJEeXcGrLhTCJFtbsnc8y1u7xidsAmiOD5A=
github.com/iotaledger/iota.go v1.0.0-beta.14/go.mod h1:F6WBmYd98mVjAmmPVYhnxg8NNIWCjjH8VWT9qvv3Rc8=
@@ -239,6 +241,7 @@ github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4k
github.com/sourcegraph/annotate v0.0.0-20160123013949-f4cad6c6324d/go.mod h1:UdhH50NIW0fCiwBSr0co2m7BnFLdv4fQTgdqdJTHFeE=
github.com/sourcegraph/syntaxhighlight v0.0.0-20170531221838-bd320f5d308e/go.mod h1:HuIsMU8RRBOtsCgI77wP899iHVBQpCmg4ErYMZB+2IA=
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=
github.com/spf13/afero v1.2.2 h1:5jhuqJyZCZf2JRofRvN/nIFgIWNzPa3/Vz8mYylgbWc=
diff --git a/packages/autopeering/peer/peerdb.go b/packages/autopeering/peer/peerdb.go
index def6344b4f270d140c121e50a5504cb0101b5aba..b20fcd052b304c5dc9b4fac5b09a3ca31dce8fa5 100644
--- a/packages/autopeering/peer/peerdb.go
+++ b/packages/autopeering/peer/peerdb.go
@@ -74,7 +74,7 @@ const (
// NewPersistentDB creates a new persistent DB.
func NewPersistentDB(log *logger.Logger) DB {
- db, err := database.Get("peer")
+ db, err := database.Get(database.DBPrefixAutoPeering, database.GetBadgerInstance())
if err != nil {
panic(err)
}
@@ -157,26 +157,26 @@ func parseInt64(blob []byte) int64 {
// getInt64 retrieves an integer associated with a particular key.
func (db *persistentDB) getInt64(key []byte) int64 {
- blob, err := db.db.Get(key)
+ entry, err := db.db.Get(key)
if err != nil {
return 0
}
- return parseInt64(blob)
+ return parseInt64(entry.Value)
}
// setInt64 stores an integer in the given key.
func (db *persistentDB) setInt64(key []byte, n int64) error {
blob := make([]byte, binary.MaxVarintLen64)
blob = blob[:binary.PutVarint(blob, n)]
- return db.db.SetWithTTL(key, blob, peerExpiration)
+ return db.db.Set(database.Entry{Key: key, Value: blob, TTL: peerExpiration})
}
// LocalPrivateKey returns the private key stored in the database or creates a new one.
func (db *persistentDB) LocalPrivateKey() (PrivateKey, error) {
- key, err := db.db.Get(localFieldKey(dbLocalKey))
+ entry, err := db.db.Get(localFieldKey(dbLocalKey))
if err == database.ErrKeyNotFound {
- key, err = generatePrivateKey()
- if err == nil {
+ key, genErr := generatePrivateKey()
+ if genErr == nil {
err = db.UpdateLocalPrivateKey(key)
}
return key, err
@@ -184,13 +184,12 @@ func (db *persistentDB) LocalPrivateKey() (PrivateKey, error) {
if err != nil {
return nil, err
}
-
- return key, nil
+ return PrivateKey(entry.Value), nil
}
// UpdateLocalPrivateKey stores the provided key in the database.
func (db *persistentDB) UpdateLocalPrivateKey(key PrivateKey) error {
- return db.db.Set(localFieldKey(dbLocalKey), key)
+ return db.db.Set(database.Entry{Key: localFieldKey(dbLocalKey), Value: []byte(key)})
}
// LastPing returns that property for the given peer ID and address.
@@ -218,7 +217,7 @@ func (db *persistentDB) setPeerWithTTL(p *Peer, ttl time.Duration) error {
if err != nil {
return err
}
- return db.db.SetWithTTL(nodeKey(p.ID()), data, ttl)
+ return db.db.Set(database.Entry{Key: nodeKey(p.ID()), Value: data, TTL: ttl})
}
func (db *persistentDB) UpdatePeer(p *Peer) error {
@@ -238,7 +237,7 @@ func (db *persistentDB) Peer(id ID) *Peer {
if err != nil {
return nil
}
- return parsePeer(data)
+ return parsePeer(data.Value)
}
func randomSubset(peers []*Peer, m int) []*Peer {
@@ -259,21 +258,22 @@ func (db *persistentDB) getPeers(maxAge time.Duration) []*Peer {
peers := make([]*Peer, 0)
now := time.Now()
- err := db.db.ForEachWithPrefix([]byte(dbNodePrefix), func(key []byte, value []byte) {
- id, rest := splitNodeKey(key)
+ err := db.db.StreamForEachPrefix([]byte(dbNodePrefix), func(entry database.Entry) error {
+ id, rest := splitNodeKey(entry.Key)
if len(rest) > 0 {
- return
+ return nil
}
- p := parsePeer(value)
+ p := parsePeer(entry.Value)
if p == nil || p.ID() != id {
- return
+ return nil
}
if maxAge > 0 && now.Sub(db.LastPong(p.ID(), p.Address())) > maxAge {
- return
+ return nil
}
peers = append(peers, p)
+ return nil
})
if err != nil {
return []*Peer{}
diff --git a/packages/database/badger_instance.go b/packages/database/badger_instance.go
deleted file mode 100644
index 439aa4727db7bd7f56c29bbc46896cbba7ae7b7e..0000000000000000000000000000000000000000
--- a/packages/database/badger_instance.go
+++ /dev/null
@@ -1,69 +0,0 @@
-package database
-
-import (
- "fmt"
- "os"
- "sync"
-
- "github.com/dgraph-io/badger"
- "github.com/dgraph-io/badger/options"
- "github.com/iotaledger/goshimmer/packages/parameter"
-)
-
-var instance *badger.DB
-var once sync.Once
-
-// Returns whether the given file or directory exists.
-func exists(path string) (bool, error) {
- _, err := os.Stat(path)
- if err == nil {
- return true, nil
- }
- if os.IsNotExist(err) {
- return false, nil
- }
- return false, err
-}
-
-func checkDir(dir string) error {
- exists, err := exists(dir)
- if err != nil {
- return err
- }
-
- if !exists {
- return os.Mkdir(dir, 0700)
- }
- return nil
-}
-
-func createDB() (*badger.DB, error) {
- directory := parameter.NodeConfig.GetString(CFG_DIRECTORY)
- if err := checkDir(directory); err != nil {
- return nil, fmt.Errorf("could not check directory: %w", err)
- }
-
- opts := badger.DefaultOptions(directory)
- opts.Logger = &logger{}
- opts.Truncate = true
- opts.TableLoadingMode = options.MemoryMap
-
- db, err := badger.Open(opts)
- if err != nil {
- return nil, fmt.Errorf("could not open new DB: %w", err)
- }
-
- return db, nil
-}
-
-func GetBadgerInstance() *badger.DB {
- once.Do(func() {
- db, err := createDB()
- if err != nil {
- // errors should cause a panic to avoid singleton deadlocks
- panic(err)
- }
- instance = db
- })
- return instance
-}
diff --git a/packages/database/database.go b/packages/database/database.go
index 913ffc3f91589637dddc22272a4e8ee45cb56214..4da5818cd93cde477656d97f9d069267d0f2ccba 100644
--- a/packages/database/database.go
+++ b/packages/database/database.go
@@ -1,134 +1,78 @@
package database
import (
+ "io/ioutil"
+ "os"
+ "runtime"
"sync"
- "time"
- "github.com/dgraph-io/badger"
+ "github.com/dgraph-io/badger/v2"
+ "github.com/iotaledger/goshimmer/packages/parameter"
+ "github.com/iotaledger/hive.go/database"
+ "github.com/iotaledger/hive.go/logger"
)
var (
- ErrKeyNotFound = badger.ErrKeyNotFound
-
- dbMap = make(map[string]*prefixDb)
- mu sync.Mutex
+ instance *badger.DB
+ once sync.Once
+ ErrKeyNotFound = database.ErrKeyNotFound
)
-type prefixDb struct {
- db *badger.DB
- name string
- prefix []byte
-}
-
-func getPrefix(name string) []byte {
- return []byte(name + "_")
-}
-
-func Get(name string) (Database, error) {
- mu.Lock()
- defer mu.Unlock()
-
- if db, exists := dbMap[name]; exists {
- return db, nil
- }
-
- badger := GetBadgerInstance()
- db := &prefixDb{
- db: badger,
- name: name,
- prefix: getPrefix(name),
- }
-
- dbMap[name] = db
-
- return db, nil
-}
-
-func (this *prefixDb) setEntry(e *badger.Entry) error {
- err := this.db.Update(func(txn *badger.Txn) error {
- return txn.SetEntry(e)
- })
- return err
-}
-
-func (this *prefixDb) Set(key []byte, value []byte) error {
- e := badger.NewEntry(append(this.prefix, key...), value)
- return this.setEntry(e)
-}
+type (
+ Database = database.Database
+ Entry = database.Entry
+ KeyOnlyEntry = database.KeyOnlyEntry
+ KeyPrefix = database.KeyPrefix
+ Value = database.Value
+)
-func (this *prefixDb) SetWithTTL(key []byte, value []byte, ttl time.Duration) error {
- e := badger.NewEntry(append(this.prefix, key...), value).WithTTL(ttl)
- return this.setEntry(e)
+func Get(dbPrefix byte, optionalBadger ...*badger.DB) (Database, error) {
+ return database.Get(dbPrefix, optionalBadger...)
}
-func (this *prefixDb) Contains(key []byte) (bool, error) {
- err := this.db.View(func(txn *badger.Txn) error {
- _, err := txn.Get(append(this.prefix, key...))
- return err
- })
-
- if err == ErrKeyNotFound {
- return false, nil
- } else {
- return err == nil, err
- }
-}
+func GetBadgerInstance() *badger.DB {
+ once.Do(func() {
+ dbDir := parameter.NodeConfig.GetString(CFG_DIRECTORY)
-func (this *prefixDb) Get(key []byte) ([]byte, error) {
- var result []byte = nil
-
- err := this.db.View(func(txn *badger.Txn) error {
- item, err := txn.Get(append(this.prefix, key...))
+ var dbDirClear bool
+ // check whether the database is new, by checking whether any file exists within
+ // the database directory
+ fileInfos, err := ioutil.ReadDir(dbDir)
if err != nil {
- return err
+ // panic on other errors, for example permission related
+ if !os.IsNotExist(err) {
+ panic(err)
+ }
+ dbDirClear = true
+ }
+ if len(fileInfos) == 0 {
+ dbDirClear = true
}
- return item.Value(func(val []byte) error {
- result = append([]byte{}, val...)
-
- return nil
- })
- })
-
- return result, err
-}
-
-func (this *prefixDb) Delete(key []byte) error {
- err := this.db.Update(func(txn *badger.Txn) error {
- return txn.Delete(append(this.prefix, key...))
- })
- return err
-}
-
-func (this *prefixDb) forEach(prefix []byte, consumer func([]byte, []byte)) error {
- err := this.db.View(func(txn *badger.Txn) error {
- iteratorOptions := badger.DefaultIteratorOptions
- iteratorOptions.Prefix = prefix // filter by prefix
-
- // create an iterator the default options
- it := txn.NewIterator(iteratorOptions)
- defer it.Close()
-
- // loop through every key-value-pair and call the function
- for it.Rewind(); it.Valid(); it.Next() {
- item := it.Item()
-
- value, err := item.ValueCopy(nil)
- if err != nil {
- return err
- }
+ opts := badger.DefaultOptions(dbDir)
+ opts.Logger = nil
+ if runtime.GOOS == "windows" {
+ opts = opts.WithTruncate(true)
+ }
- consumer(item.Key()[len(this.prefix):], value)
+ db, err := database.CreateDB(dbDir, opts)
+ if err != nil {
+ // errors should cause a panic to avoid singleton deadlocks
+ panic(err)
}
- return nil
- })
- return err
-}
+ instance = db
-func (this *prefixDb) ForEachWithPrefix(prefix []byte, consumer func([]byte, []byte)) error {
- return this.forEach(append(this.prefix, prefix...), consumer)
+ // up on the first caller, check whether the version of the database is compatible
+ checkDatabaseVersion(dbDirClear)
+ })
+ return instance
}
-func (this *prefixDb) ForEach(consumer func([]byte, []byte)) error {
- return this.forEach(this.prefix, consumer)
+func CleanupBadgerInstance(log *logger.Logger) {
+ db := GetBadgerInstance()
+ log.Info("Running Badger database garbage collection")
+ var err error
+ for err == nil {
+ err = db.RunValueLogGC(0.7)
+ }
}
diff --git a/packages/database/interfaces.go b/packages/database/interfaces.go
deleted file mode 100644
index 56d2b9573836153e1b487802510ce0d7953c13ca..0000000000000000000000000000000000000000
--- a/packages/database/interfaces.go
+++ /dev/null
@@ -1,13 +0,0 @@
-package database
-
-import "time"
-
-type Database interface {
- Set(key []byte, value []byte) error
- SetWithTTL(key []byte, value []byte, ttl time.Duration) error
- Contains(key []byte) (bool, error)
- Get(key []byte) ([]byte, error)
- ForEach(consumer func(key []byte, value []byte)) error
- ForEachWithPrefix(prefix []byte, consumer func(key []byte, value []byte)) error
- Delete(key []byte) error
-}
diff --git a/packages/database/logger.go b/packages/database/logger.go
deleted file mode 100644
index ac1de091452794ddb04857acd94df5ff87df72d7..0000000000000000000000000000000000000000
--- a/packages/database/logger.go
+++ /dev/null
@@ -1,19 +0,0 @@
-package database
-
-type logger struct{}
-
-func (this *logger) Errorf(string, ...interface{}) {
- // disable logging
-}
-
-func (this *logger) Infof(string, ...interface{}) {
- // disable logging
-}
-
-func (this *logger) Warningf(string, ...interface{}) {
- // disable logging
-}
-
-func (this *logger) Debugf(string, ...interface{}) {
- // disable logging
-}
diff --git a/packages/database/prefixes.go b/packages/database/prefixes.go
new file mode 100644
index 0000000000000000000000000000000000000000..c9dc488d40b5fbe557467e1be275e6aa56052aec
--- /dev/null
+++ b/packages/database/prefixes.go
@@ -0,0 +1,11 @@
+package database
+
+const (
+ DBPrefixApprovers byte = iota
+ DBPrefixTransaction
+ DBPrefixBundle
+ DBPrefixTransactionMetadata
+ DBPrefixAddressTransactions
+ DBPrefixAutoPeering
+ DBPrefixDatabaseVersion
+)
diff --git a/packages/database/versioning.go b/packages/database/versioning.go
new file mode 100644
index 0000000000000000000000000000000000000000..a848eb0e508a7ecbf3b9c61773700fdfdaa79b1c
--- /dev/null
+++ b/packages/database/versioning.go
@@ -0,0 +1,47 @@
+package database
+
+import (
+ "errors"
+ "fmt"
+)
+
+const (
+ // DBVersion defines the version of the database schema this version of GoShimmer supports.
+ // everytime there's a breaking change regarding the stored data, this version flag should be adjusted.
+ DBVersion = 1
+)
+
+var (
+ ErrDBVersionIncompatible = errors.New("database version is not compatible. please delete your database folder and restart")
+ // the key under which the database is stored
+ dbVersionKey = []byte{0}
+)
+
+// checks whether the database is compatible with the current schema version.
+// also automatically sets the version if the database is new.
+func checkDatabaseVersion(dbIsNew bool) {
+ dbInstance, err := Get(DBPrefixDatabaseVersion, instance)
+ if err != nil {
+ panic(err)
+ }
+
+ if dbIsNew {
+ // store db version for the first time in the new database
+ if err = dbInstance.Set(Entry{Key: dbVersionKey, Value: []byte{DBVersion}}); err != nil {
+ panic(fmt.Sprintf("unable to persist db version number: %s", err.Error()))
+ }
+ return
+ }
+
+ // db version must be available
+ entry, err := dbInstance.Get(dbVersionKey)
+ if err != nil {
+ if err == ErrKeyNotFound {
+ panic(err)
+ }
+ panic(fmt.Errorf("%w: no database version was persisted", ErrDBVersionIncompatible))
+ }
+ if entry.Value[0] != DBVersion {
+ panic(fmt.Errorf("%w: supported version: %d, version of database: %d", ErrDBVersionIncompatible, DBVersion, entry.Value[0]))
+ }
+}
diff --git a/packages/shutdown/order.go b/packages/shutdown/order.go
index b16bc91e04b45b2e9744555ae229ab359d9cb18e..51ed31ec8d11251d947628842d80542e7f05d7dd 100644
--- a/packages/shutdown/order.go
+++ b/packages/shutdown/order.go
@@ -14,5 +14,6 @@ const (
ShutdownPriorityUI
ShutdownPriorityDashboard
ShutdownPriorityTangleSpammer
+ ShutdownPriorityBadgerGarbageCollection
ShutdownPriorityStatusScreen
)
diff --git a/plugins/bundleprocessor/bundleprocessor_test.go b/plugins/bundleprocessor/bundleprocessor_test.go
index 6505583f451516b7db36c7d5a0c20d90d56fa490..e7b90366bd676bf6a0ba3d284ba6e241a368b3f9 100644
--- a/plugins/bundleprocessor/bundleprocessor_test.go
+++ b/plugins/bundleprocessor/bundleprocessor_test.go
@@ -1,10 +1,13 @@
package bundleprocessor
import (
+ "io/ioutil"
+ "os"
"sync"
"testing"
"github.com/iotaledger/goshimmer/packages/client"
+ "github.com/iotaledger/goshimmer/packages/database"
"github.com/iotaledger/goshimmer/packages/model/bundle"
"github.com/iotaledger/goshimmer/packages/model/value_transaction"
"github.com/iotaledger/goshimmer/packages/parameter"
@@ -16,6 +19,7 @@ import (
"github.com/iotaledger/iota.go/consts"
"github.com/magiconair/properties/assert"
"github.com/spf13/viper"
+ "github.com/stretchr/testify/require"
)
var seed = client.NewSeed("YFHQWAUPCXC9S9DSHP9NDF9RLNPMZVCMSJKUKQP9SWUSUCPRQXCMDVDVZ9SHHESHIQNCXWBJF9UJSWE9Z", consts.SecurityLevelMedium)
@@ -69,6 +73,12 @@ func TestValidateSignatures(t *testing.T) {
}
func TestProcessSolidBundleHead(t *testing.T) {
+ dir, err := ioutil.TempDir("", t.Name())
+ require.NoError(t, err)
+ defer os.Remove(dir)
+ // use the tempdir for the database
+ parameter.NodeConfig.Set(database.CFG_DIRECTORY, dir)
+
// start a test node
node.Start(node.Plugins(tangle.PLUGIN, PLUGIN))
defer node.Shutdown()
diff --git a/plugins/tangle/approvers.go b/plugins/tangle/approvers.go
index 10073c39435df28a04fefc943518312582ab5eee..3e50c29caec97397832490d911ca3989ebe3be6b 100644
--- a/plugins/tangle/approvers.go
+++ b/plugins/tangle/approvers.go
@@ -82,7 +82,7 @@ const (
var approversDatabase database.Database
func configureApproversDatabase() {
- if db, err := database.Get("approvers"); err != nil {
+ if db, err := database.Get(database.DBPrefixApprovers, database.GetBadgerInstance()); err != nil {
panic(err)
} else {
approversDatabase = db
@@ -91,10 +91,9 @@ func configureApproversDatabase() {
func storeApproversInDatabase(approvers *approvers.Approvers) error {
if approvers.GetModified() {
- if err := approversDatabase.Set(typeutils.StringToBytes(approvers.GetHash()), approvers.Marshal()); err != nil {
+ if err := approversDatabase.Set(database.Entry{Key: typeutils.StringToBytes(approvers.GetHash()), Value: approvers.Marshal()}); err != nil {
return fmt.Errorf("%w: failed to store approvers: %s", ErrDatabaseError, err)
}
-
approvers.SetModified(false)
}
@@ -111,7 +110,7 @@ func getApproversFromDatabase(transactionHash trinary.Trytes) (*approvers.Approv
}
var result approvers.Approvers
- if err = result.Unmarshal(approversData); err != nil {
+ if err = result.Unmarshal(approversData.Value); err != nil {
panic(err)
}
diff --git a/plugins/tangle/bundle.go b/plugins/tangle/bundle.go
index 55411764a2eacc04b73122a86b119352bc0d1cbd..2cb1fb896ae5cd5a8d6152a0e2398ef4651f4a87 100644
--- a/plugins/tangle/bundle.go
+++ b/plugins/tangle/bundle.go
@@ -86,7 +86,7 @@ const (
var bundleDatabase database.Database
func configureBundleDatabase() {
- if db, err := database.Get("bundle"); err != nil {
+ if db, err := database.Get(database.DBPrefixBundle, database.GetBadgerInstance()); err != nil {
panic(err)
} else {
bundleDatabase = db
@@ -95,10 +95,9 @@ func configureBundleDatabase() {
func storeBundleInDatabase(bundle *bundle.Bundle) error {
if bundle.GetModified() {
- if err := bundleDatabase.Set(typeutils.StringToBytes(bundle.GetHash()), bundle.Marshal()); err != nil {
+ if err := bundleDatabase.Set(database.Entry{Key: typeutils.StringToBytes(bundle.GetHash()), Value: bundle.Marshal()}); err != nil {
return fmt.Errorf("%w: failed to store bundle: %s", ErrDatabaseError, err)
}
-
bundle.SetModified(false)
}
@@ -116,7 +115,7 @@ func getBundleFromDatabase(transactionHash trinary.Trytes) (*bundle.Bundle, erro
}
var result bundle.Bundle
- if err = result.Unmarshal(bundleData); err != nil {
+ if err = result.Unmarshal(bundleData.Value); err != nil {
panic(err)
}
diff --git a/plugins/tangle/plugin.go b/plugins/tangle/plugin.go
index 252c2b987da1db3774e0def97e65582f879a8c6c..6c6bc09015f462d137784c7c9c5fad2a26c59929 100644
--- a/plugins/tangle/plugin.go
+++ b/plugins/tangle/plugin.go
@@ -1,11 +1,14 @@
package tangle
import (
+ "time"
+
"github.com/iotaledger/goshimmer/packages/database"
"github.com/iotaledger/goshimmer/packages/shutdown"
"github.com/iotaledger/hive.go/daemon"
"github.com/iotaledger/hive.go/logger"
"github.com/iotaledger/hive.go/node"
+ "github.com/iotaledger/hive.go/timeutil"
"github.com/iotaledger/iota.go/trinary"
)
@@ -42,6 +45,13 @@ func configure(*node.Plugin) {
}
func run(*node.Plugin) {
+
+ daemon.BackgroundWorker("Badger garbage collection", func(shutdownSignal <-chan struct{}) {
+ timeutil.Ticker(func() {
+ database.CleanupBadgerInstance(log)
+ }, 5*time.Minute, shutdownSignal)
+ }, shutdown.ShutdownPriorityBadgerGarbageCollection)
+
runSolidifier()
}
diff --git a/plugins/tangle/solidifier_test.go b/plugins/tangle/solidifier_test.go
index 850f87501e8dc43fccc6fddb55a74cf834e9ee8d..28a63bc25a1fdcd79f16fb2f82d2b66380088ae5 100644
--- a/plugins/tangle/solidifier_test.go
+++ b/plugins/tangle/solidifier_test.go
@@ -33,7 +33,7 @@ func init() {
}
func TestTangle(t *testing.T) {
- dir, err := ioutil.TempDir("", "example")
+ dir, err := ioutil.TempDir("", t.Name())
require.NoError(t, err)
defer os.Remove(dir)
// use the tempdir for the database
diff --git a/plugins/tangle/transaction.go b/plugins/tangle/transaction.go
index fc2e2f02dd7cd3b18739eeb232f6b27b4bd791ae..bff7b6b726d8bda3b4f3fdc4278f0a3bbe80a254 100644
--- a/plugins/tangle/transaction.go
+++ b/plugins/tangle/transaction.go
@@ -83,7 +83,7 @@ const (
var transactionDatabase database.Database
func configureTransactionDatabase() {
- if db, err := database.Get("transaction"); err != nil {
+ if db, err := database.Get(database.DBPrefixTransaction, database.GetBadgerInstance()); err != nil {
panic(err)
} else {
transactionDatabase = db
@@ -92,10 +92,9 @@ func configureTransactionDatabase() {
func storeTransactionInDatabase(transaction *value_transaction.ValueTransaction) error {
if transaction.GetModified() {
- if err := transactionDatabase.Set(typeutils.StringToBytes(transaction.GetHash()), transaction.MetaTransaction.GetBytes()); err != nil {
+ if err := transactionDatabase.Set(database.Entry{Key: typeutils.StringToBytes(transaction.GetHash()), Value: transaction.MetaTransaction.GetBytes()}); err != nil {
return fmt.Errorf("%w: failed to store transaction: %s", ErrDatabaseError, err.Error())
}
-
transaction.SetModified(false)
}
@@ -111,7 +110,7 @@ func getTransactionFromDatabase(transactionHash trinary.Trytes) (*value_transact
return nil, fmt.Errorf("%w: failed to retrieve transaction: %s", ErrDatabaseError, err)
}
- return value_transaction.FromBytes(txData), nil
+ return value_transaction.FromBytes(txData.Value), nil
}
func databaseContainsTransaction(transactionHash trinary.Trytes) (bool, error) {
diff --git a/plugins/tangle/transaction_metadata.go b/plugins/tangle/transaction_metadata.go
index 94d6177af989c666a75d2b1c1e75bd1c252166bc..152f0126da1f00dd741d2c9953743a8010e9f764 100644
--- a/plugins/tangle/transaction_metadata.go
+++ b/plugins/tangle/transaction_metadata.go
@@ -83,7 +83,7 @@ const (
var transactionMetadataDatabase database.Database
func configureTransactionMetaDataDatabase() {
- if db, err := database.Get("transactionMetadata"); err != nil {
+ if db, err := database.Get(database.DBPrefixTransactionMetadata, database.GetBadgerInstance()); err != nil {
panic(err)
} else {
transactionMetadataDatabase = db
@@ -95,7 +95,7 @@ func storeTransactionMetadataInDatabase(metadata *transactionmetadata.Transactio
if marshaledMetadata, err := metadata.Marshal(); err != nil {
return err
} else {
- if err := transactionMetadataDatabase.Set(typeutils.StringToBytes(metadata.GetHash()), marshaledMetadata); err != nil {
+ if err := transactionMetadataDatabase.Set(database.Entry{Key: typeutils.StringToBytes(metadata.GetHash()), Value: marshaledMetadata}); err != nil {
return fmt.Errorf("%w: failed to store transaction metadata: %s", ErrDatabaseError, err)
}
@@ -116,7 +116,7 @@ func getTransactionMetadataFromDatabase(transactionHash trinary.Trytes) (*transa
}
var result transactionmetadata.TransactionMetadata
- if err := result.Unmarshal(txMetadata); err != nil {
+ if err := result.Unmarshal(txMetadata.Value); err != nil {
panic(err)
}
diff --git a/plugins/tangle/tx_per_address.go b/plugins/tangle/tx_per_address.go
index 1ab4950e6c79405e99e4acf3242f4ebc6591eb94..092a75248419251137c6205be0abf108b9b495df 100644
--- a/plugins/tangle/tx_per_address.go
+++ b/plugins/tangle/tx_per_address.go
@@ -13,7 +13,7 @@ var (
)
func configureTransactionHashesForAddressDatabase() {
- if db, err := database.Get("transactionsHashesForAddress"); err != nil {
+ if db, err := database.Get(database.DBPrefixAddressTransactions, database.GetBadgerInstance()); err != nil {
panic(err)
} else {
transactionsHashesForAddressDatabase = db
@@ -26,10 +26,10 @@ type TxHashForAddress struct {
}
func StoreTransactionHashForAddressInDatabase(address *TxHashForAddress) error {
- if err := transactionsHashesForAddressDatabase.Set(
- databaseKeyForHashPrefixedHash(address.Address, address.TxHash),
- []byte{},
- ); err != nil {
+ if err := transactionsHashesForAddressDatabase.Set(database.Entry{
+ Key: databaseKeyForHashPrefixedHash(address.Address, address.TxHash),
+ Value: []byte{},
+ }); err != nil {
return fmt.Errorf("%w: failed to store tx for address in database: %s", ErrDatabaseError, err)
}
return nil
@@ -47,11 +47,9 @@ func DeleteTransactionHashForAddressInDatabase(address *TxHashForAddress) error
func ReadTransactionHashesForAddressFromDatabase(address trinary.Hash) ([]trinary.Hash, error) {
var transactionHashes []trinary.Hash
- err := transactionsHashesForAddressDatabase.ForEachWithPrefix(databaseKeyForHashPrefix(address), func(key []byte, value []byte) {
- k := typeutils.BytesToString(key)
- if len(k) > 81 {
- transactionHashes = append(transactionHashes, k[81:])
- }
+ err := transactionsHashesForAddressDatabase.StreamForEachPrefixKeyOnly(databaseKeyForHashPrefix(address), func(key database.KeyOnlyEntry) error {
+ transactionHashes = append(transactionHashes, typeutils.BytesToString(key.Key))
+ return nil
})
if err != nil {