From 1266bcf878a5a233d580dd3faaae8a1168eb799d Mon Sep 17 00:00:00 2001
From: Hans Moog <hm@mkjc.net>
Date: Tue, 20 Aug 2019 22:17:22 +0200
Subject: [PATCH] Feat: added a prefix (radix) trie to compress hashes in votes
To be able to make the vote packages smaller, we have introduced a hash compression mechanism. While hashes can usually not be "compressed" due to limits in information theory, nodes in the IOTA network already know about relevant hashes upfront. We therefore only need to be able to exchange which hash we mean, when we exchange votes. This commit introduces a novel data structure, that uses radix trie like mechanisms, to create a very compact representation of a tx hash that can be understood by others. It allows us to compress the hashes from 49 bytes to 2-3 bytes which is a 94% compression rate.
---
go.mod | 2 +-
go.sum | 2 -
packages/ca/heartbeat/serialization_test.go | 4 +-
packages/datastructure/prefix_trie.go | 188 ++++++++++++++++++
packages/datastructure/prefix_trie_test.go | 142 +++++++++++++
packages/model/approvers/approvers_test.go | 2 +-
packages/model/bundle/bundle_test.go | 2 +-
.../meta_transaction/meta_transaction_test.go | 2 +-
.../value_transaction_test.go | 2 +-
plugins/autopeering/types/peer/peer_test.go | 2 +-
.../bundleprocessor/bundleprocessor_test.go | 2 +-
11 files changed, 339 insertions(+), 11 deletions(-)
create mode 100644 packages/datastructure/prefix_trie.go
create mode 100644 packages/datastructure/prefix_trie_test.go
diff --git a/go.mod b/go.mod
index 879290b9..31b7140a 100644
--- a/go.mod
+++ b/go.mod
@@ -14,10 +14,10 @@ require (
github.com/kr/pretty v0.1.0 // indirect
github.com/labstack/echo v3.3.10+incompatible
github.com/labstack/gommon v0.2.9 // indirect
- github.com/magiconair/properties v1.8.1
github.com/pkg/errors v0.8.1
github.com/rivo/tview v0.0.0-20190721135419-23dc8a0944e4
github.com/rivo/uniseg v0.1.0 // indirect
+ github.com/stretchr/testify v1.3.0
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e // indirect
diff --git a/go.sum b/go.sum
index 331a8ca2..3190f6cf 100644
--- a/go.sum
+++ b/go.sum
@@ -63,8 +63,6 @@ github.com/labstack/gommon v0.2.9/go.mod h1:E8ZTmW9vw5az5/ZyHWCp0Lw4OH2ecsaBP1C/
github.com/lucasb-eyer/go-colorful v1.0.2 h1:mCMFu6PgSozg9tDNMMK3g18oJBX7oYGrC09mS6CXfO4=
github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s=
github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
-github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
-github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU=
github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE=
diff --git a/packages/ca/heartbeat/serialization_test.go b/packages/ca/heartbeat/serialization_test.go
index 39226bc5..c98c532b 100644
--- a/packages/ca/heartbeat/serialization_test.go
+++ b/packages/ca/heartbeat/serialization_test.go
@@ -15,11 +15,11 @@ import (
func TestMarshal(t *testing.T) {
ownNodeId := identity.GenerateRandomIdentity().Identifier
- toggledTransactions := make([]*heartbeatproto.ToggledTransaction, 1000)
+ toggledTransactions := make([]*heartbeatproto.ToggledTransaction, 10000)
for i := 0; i < len(toggledTransactions); i++ {
toggledTransactions[i] = &heartbeatproto.ToggledTransaction{
- TransactionId: make([]byte, 32),
+ TransactionId: make([]byte, 3),
ToggleReason: 0,
}
}
diff --git a/packages/datastructure/prefix_trie.go b/packages/datastructure/prefix_trie.go
new file mode 100644
index 00000000..a7e712a6
--- /dev/null
+++ b/packages/datastructure/prefix_trie.go
@@ -0,0 +1,188 @@
+package datastructure
+
+import (
+ "bytes"
+ "fmt"
+ "sync"
+)
+
+type PrefixTrie struct {
+ value []byte
+ children map[byte]*PrefixTrie
+ size int
+ mutex sync.RWMutex
+}
+
+func NewPrefixTree() *PrefixTrie {
+ return &PrefixTrie{
+ value: nil,
+ children: make(map[byte]*PrefixTrie),
+ }
+}
+
+func (prefixTrie *PrefixTrie) Get(byteSequenceOrPrefix []byte) (result [][]byte) {
+ prefixTrie.mutex.RLock()
+ defer prefixTrie.mutex.RUnlock()
+
+ result = make([][]byte, 0)
+
+ currentNode := prefixTrie
+
+ for currentLevel := 0; currentLevel < len(byteSequenceOrPrefix); currentLevel++ {
+ if currentNode.value != nil && bytes.HasPrefix(currentNode.value, byteSequenceOrPrefix) {
+ result = append(result, currentNode.value)
+
+ return
+ }
+
+ if existingNode, exists := currentNode.children[byteSequenceOrPrefix[currentLevel]]; exists {
+ currentNode = existingNode
+ } else {
+ // error tried to inflate non-existing entry
+ return
+ }
+ }
+
+ if currentNode.value != nil {
+ result = append(result, currentNode.value)
+ } else {
+ // traverse child elements
+ if false {
+ fmt.Println("WAS")
+ }
+ }
+
+ return
+}
+
+func (prefixTrie *PrefixTrie) GetPrefix(insertedBytes []byte) []byte {
+ prefixTrie.mutex.RLock()
+ defer prefixTrie.mutex.RUnlock()
+
+ currentNode := prefixTrie
+ currentLevel := 0
+
+ for {
+ // if we have reached our target
+ if bytes.Equal(currentNode.value, insertedBytes) {
+ return insertedBytes[:currentLevel]
+ }
+
+ // if we have arrived at the wrong node: return nil
+ if currentNode.value != nil {
+ return nil
+ }
+
+ if childNode, exists := currentNode.children[insertedBytes[currentLevel]]; !exists {
+ return nil
+ } else {
+ currentNode = childNode
+ }
+
+ // increase level counter
+ currentLevel++
+ }
+}
+
+func (prefixTrie *PrefixTrie) Insert(byteSequence []byte) bool {
+ prefixTrie.mutex.Lock()
+ defer prefixTrie.mutex.Unlock()
+
+ currentNode := prefixTrie
+ currentLevel := 0
+
+ for {
+ // if we have reached our target node: insert value
+ if currentNode.value == nil && len(currentNode.children) == 0 {
+ currentNode.value = byteSequence
+
+ prefixTrie.size++
+
+ return true
+ }
+
+ // if we have reached a previous leaf
+ if currentNode.value != nil {
+ // return if same element
+ if bytes.Equal(currentNode.value, byteSequence) {
+ return false
+ }
+
+ // move current value to correct sub element
+ currentNode.children[currentNode.value[currentLevel]] = &PrefixTrie{
+ value: currentNode.value,
+ children: make(map[byte]*PrefixTrie),
+ }
+
+ // set the value to nil
+ currentNode.value = nil
+ }
+
+ // traverse or create correct child element
+ if existingChildNode, exists := currentNode.children[byteSequence[currentLevel]]; exists {
+ currentNode = existingChildNode
+ } else {
+ newNode := &PrefixTrie{
+ children: make(map[byte]*PrefixTrie),
+ }
+
+ currentNode.children[byteSequence[currentLevel]] = newNode
+
+ currentNode = newNode
+ }
+
+ // increase level counter
+ currentLevel++
+ }
+}
+
+func (prefixTrie *PrefixTrie) Delete(byteSequence []byte) bool {
+ prefixTrie.mutex.Lock()
+ defer prefixTrie.mutex.Unlock()
+
+ currentNode := prefixTrie
+
+ // trivial case: delete from root
+ if bytes.Equal(currentNode.value, byteSequence) {
+ currentNode.value = nil
+
+ prefixTrie.size--
+
+ return true
+ }
+
+ // non-trivial case: delete leaf
+ for currentLevel := 0; currentLevel < len(byteSequence); currentLevel++ {
+ if existingNode, exists := currentNode.children[byteSequence[currentLevel]]; exists {
+ if bytes.Equal(existingNode.value, byteSequence) {
+ delete(currentNode.children, byteSequence[currentLevel])
+
+ if len(currentNode.children) == 1 {
+ for index, currentChild := range currentNode.children {
+ if currentChild.value != nil {
+ currentNode.value = currentChild.value
+ delete(currentNode.children, index)
+ }
+ }
+ }
+
+ prefixTrie.size--
+
+ return true
+ }
+
+ currentNode = existingNode
+ } else {
+ return false
+ }
+ }
+
+ return false
+}
+
+func (prefixTrie *PrefixTrie) GetSize() int {
+ prefixTrie.mutex.RLock()
+ defer prefixTrie.mutex.RUnlock()
+
+ return prefixTrie.size
+}
diff --git a/packages/datastructure/prefix_trie_test.go b/packages/datastructure/prefix_trie_test.go
new file mode 100644
index 00000000..34eb6e4f
--- /dev/null
+++ b/packages/datastructure/prefix_trie_test.go
@@ -0,0 +1,142 @@
+package datastructure
+
+import (
+ "bytes"
+ "math/rand"
+ "testing"
+
+ "github.com/iotaledger/iota.go/trinary"
+ "github.com/stretchr/testify/assert"
+)
+
+func BenchmarkByteTrie_Insert(b *testing.B) {
+ trie := &PrefixTrie{
+ value: nil,
+ children: make(map[byte]*PrefixTrie),
+ }
+
+ b.ResetTimer()
+
+ for i := 0; i < b.N; i++ {
+ token := make([]byte, 50)
+ rand.Read(token)
+
+ trie.Insert(token)
+ }
+}
+
+func TestPrefixTrie_GetPrefix(t *testing.T) {
+ trie := &PrefixTrie{
+ value: nil,
+ children: make(map[byte]*PrefixTrie),
+ }
+
+ var token []byte
+ for i := 0; i < 64000; i++ {
+ token = make([]byte, 50)
+ rand.Read(token)
+
+ trie.Insert(token)
+ }
+
+ assert.True(t, len(trie.GetPrefix(token)) <= 3)
+}
+
+func TestPrefixTrie_Insert(t *testing.T) {
+ trie := NewPrefixTree()
+
+ tx1Hash := trinary.MustTrytesToBytes("NDMXEXQFVRJKHVRGYURKMRMUUYUNPCREQHEZKNSEHK9SWSIQDU9IF9IHXTOIVVHWXGLJHHUR9NIGMAUQC")
+ tx2Hash := trinary.MustTrytesToBytes("HWTXYVLGPKUUFXGBBGKLMRCI9VOW9MEJRCJPRMUGCKHOVCCMFSZNAFEWOOVYUEHGYDWPWBEKWJPAZ9999")
+ tx3Hash := trinary.MustTrytesToBytes("IPVMHNESBZUKAIYFJCYEVXDBICITTLDUQIOVZODAISWCNEMBLWHVBDTEHNWRENIDEGVVODLXHTXGA9999")
+
+ assert.Equal(t, 0, trie.GetSize())
+
+ assert.Equal(t, true, trie.Insert(tx1Hash))
+ assert.Equal(t, true, trie.Insert(tx2Hash))
+ assert.Equal(t, true, trie.Insert(tx3Hash))
+ assert.Equal(t, false, trie.Insert(tx1Hash))
+
+ assert.Equal(t, 3, trie.GetSize())
+
+ txFound := false
+ for _, hash := range trie.Get(trie.GetPrefix(tx1Hash)) {
+ if bytes.Equal(hash, tx1Hash) {
+ txFound = true
+ }
+ }
+ assert.True(t, txFound)
+
+ assert.Equal(t, true, trie.Delete(tx1Hash))
+ assert.Equal(t, false, trie.Delete(tx1Hash))
+ assert.Equal(t, true, trie.Delete(tx2Hash))
+
+ assert.Equal(t, 1, trie.GetSize())
+}
+
+func TestPrefixTrie_Get(t *testing.T) {
+ trie := NewPrefixTree()
+
+ tx1Hash := trinary.MustTrytesToBytes("NDMXEXQFVRJKHVRGYURKMRMUUYUNPCREQHEZKNSEHK9SWSIQDU9IF9IHXTOIVVHWXGLJHHUR9NIGMAUQC")
+ tx2Hash := trinary.MustTrytesToBytes("HWTXYVLGPKUUFXGBBGKLMRCI9VOW9MEJRCJPRMUGCKHOVCCMFSZNAFEWOOVYUEHGYDWPWBEKWJPAZ9999")
+
+ trie.Insert(tx1Hash)
+ trie.Insert(tx2Hash)
+
+ prefix := trie.GetPrefix(tx1Hash)
+
+ resultsByPrefix := trie.Get(prefix)
+ resultsByFullHash := trie.Get(tx1Hash)
+
+ assert.Equal(t, len(resultsByPrefix), len(resultsByFullHash))
+ for i, foundHashCandidate := range resultsByPrefix {
+ assert.True(t, bytes.Equal(foundHashCandidate, resultsByFullHash[i]))
+ }
+}
+
+func TestPrefixTrie_GetSize(t *testing.T) {
+ trie := NewPrefixTree()
+
+ assert.Equal(t, 0, trie.GetSize())
+
+ tx1Hash := trinary.MustTrytesToBytes("NDMXEXQFVRJKHVRGYURKMRMUUYUNPCREQHEZKNSEHK9SWSIQDU9IF9IHXTOIVVHWXGLJHHUR9NIGMAUQC")
+ tx2Hash := trinary.MustTrytesToBytes("HWTXYVLGPKUUFXGBBGKLMRCI9VOW9MEJRCJPRMUGCKHOVCCMFSZNAFEWOOVYUEHGYDWPWBEKWJPAZ9999")
+ tx3Hash := trinary.MustTrytesToBytes("IPVMHNESBZUKAIYFJCYEVXDBICITTLDUQIOVZODAISWCNEMBLWHVBDTEHNWRENIDEGVVODLXHTXGA9999")
+
+ trie.Insert(tx1Hash)
+ trie.Insert(tx2Hash)
+ assert.Equal(t, 2, trie.GetSize())
+
+ trie.Insert(tx3Hash)
+ assert.Equal(t, 3, trie.GetSize())
+
+ trie.Delete(tx1Hash)
+ assert.Equal(t, 2, trie.GetSize())
+
+ trie.Delete(tx2Hash)
+ assert.Equal(t, 1, trie.GetSize())
+
+ trie.Delete(tx3Hash)
+ assert.Equal(t, 0, trie.GetSize())
+
+ trie.Delete(tx3Hash)
+ assert.Equal(t, 0, trie.GetSize())
+}
+
+func TestPrefixTrie_Delete(t *testing.T) {
+ trie := NewPrefixTree()
+
+ tx1Hash := trinary.MustTrytesToBytes("NDMXEXQFVRJKHVRGYURKMRMUUYUNPCREQHEZKNSEHK9SWSIQDU9IF9IHXTOIVVHWXGLJHHUR9NIGMAUQC")
+ tx2Hash := trinary.MustTrytesToBytes("HWTXYVLGPKUUFXGBBGKLMRCI9VOW9MEJRCJPRMUGCKHOVCCMFSZNAFEWOOVYUEHGYDWPWBEKWJPAZ9999")
+ tx3Hash := trinary.MustTrytesToBytes("IPVMHNESBZUKAIYFJCYEVXDBICITTLDUQIOVZODAISWCNEMBLWHVBDTEHNWRENIDEGVVODLXHTXGA9999")
+
+ trie.Insert(tx1Hash)
+ trie.Insert(tx2Hash)
+
+ assert.Equal(t, true, trie.Delete(tx1Hash))
+ assert.Equal(t, false, trie.Delete(tx1Hash))
+
+ assert.Equal(t, true, trie.Delete(tx2Hash))
+ assert.Equal(t, false, trie.Delete(tx3Hash))
+
+ assert.Equal(t, 0, trie.GetSize())
+}
diff --git a/packages/model/approvers/approvers_test.go b/packages/model/approvers/approvers_test.go
index 926c5c07..71ec8042 100644
--- a/packages/model/approvers/approvers_test.go
+++ b/packages/model/approvers/approvers_test.go
@@ -5,7 +5,7 @@ import (
"testing"
"github.com/iotaledger/iota.go/trinary"
- "github.com/magiconair/properties/assert"
+ "github.com/stretchr/testify/assert"
)
func TestApprovers_SettersGetters(t *testing.T) {
diff --git a/packages/model/bundle/bundle_test.go b/packages/model/bundle/bundle_test.go
index d00bb6ec..46a171ac 100644
--- a/packages/model/bundle/bundle_test.go
+++ b/packages/model/bundle/bundle_test.go
@@ -4,7 +4,7 @@ import (
"testing"
"github.com/iotaledger/iota.go/trinary"
- "github.com/magiconair/properties/assert"
+ "github.com/stretchr/testify/assert"
)
func TestBundle_SettersGetters(t *testing.T) {
diff --git a/packages/model/meta_transaction/meta_transaction_test.go b/packages/model/meta_transaction/meta_transaction_test.go
index 2ba055b7..2cd47e58 100644
--- a/packages/model/meta_transaction/meta_transaction_test.go
+++ b/packages/model/meta_transaction/meta_transaction_test.go
@@ -6,7 +6,7 @@ import (
"testing"
"github.com/iotaledger/iota.go/trinary"
- "github.com/magiconair/properties/assert"
+ "github.com/stretchr/testify/assert"
)
func TestMetaTransaction_SettersGetters(t *testing.T) {
diff --git a/packages/model/value_transaction/value_transaction_test.go b/packages/model/value_transaction/value_transaction_test.go
index c1f732d6..71fa3bd5 100644
--- a/packages/model/value_transaction/value_transaction_test.go
+++ b/packages/model/value_transaction/value_transaction_test.go
@@ -5,7 +5,7 @@ import (
"testing"
"github.com/iotaledger/iota.go/trinary"
- "github.com/magiconair/properties/assert"
+ "github.com/stretchr/testify/assert"
)
func TestValueTransaction_SettersGetters(t *testing.T) {
diff --git a/plugins/autopeering/types/peer/peer_test.go b/plugins/autopeering/types/peer/peer_test.go
index 8d73ab17..c5406501 100644
--- a/plugins/autopeering/types/peer/peer_test.go
+++ b/plugins/autopeering/types/peer/peer_test.go
@@ -8,7 +8,7 @@ import (
"github.com/iotaledger/goshimmer/plugins/autopeering/types/salt"
"github.com/iotaledger/goshimmer/packages/identity"
- "github.com/magiconair/properties/assert"
+ "github.com/stretchr/testify/assert"
)
func TestPeer_MarshalUnmarshal(t *testing.T) {
diff --git a/plugins/bundleprocessor/bundleprocessor_test.go b/plugins/bundleprocessor/bundleprocessor_test.go
index 14834c67..f3aef46e 100644
--- a/plugins/bundleprocessor/bundleprocessor_test.go
+++ b/plugins/bundleprocessor/bundleprocessor_test.go
@@ -16,7 +16,7 @@ import (
"github.com/iotaledger/goshimmer/plugins/tangle"
"github.com/iotaledger/iota.go/consts"
- "github.com/magiconair/properties/assert"
+ "github.com/stretchr/testify/assert"
)
var seed = client.NewSeed("YFHQWAUPCXC9S9DSHP9NDF9RLNPMZVCMSJKUKQP9SWUSUCPRQXCMDVDVZ9SHHESHIQNCXWBJF9UJSWE9Z", consts.SecurityLevelMedium)
--
GitLab