Skip to content
Snippets Groups Projects
Commit 1266bcf8 authored by Hans Moog's avatar Hans Moog
Browse files

Feat: added a prefix (radix) trie to compress hashes in votes

To be able to make the vote packages smaller, we have introduced a hash compression mechanism. While hashes can usually not be "compressed" due to limits in information theory, nodes in the IOTA network already know about relevant hashes upfront. We therefore only need to be able to exchange which hash we mean, when we exchange votes. This commit introduces a novel data structure, that uses radix trie like mechanisms, to create a very compact representation of a tx hash that can be understood by others. It allows us to compress the hashes from 49 bytes to 2-3 bytes which is a 94% compression rate.
parent 369f8cfc
No related branches found
No related tags found
No related merge requests found
......@@ -14,10 +14,10 @@ require (
github.com/kr/pretty v0.1.0 // indirect
github.com/labstack/echo v3.3.10+incompatible
github.com/labstack/gommon v0.2.9 // indirect
github.com/magiconair/properties v1.8.1
github.com/pkg/errors v0.8.1
github.com/rivo/tview v0.0.0-20190721135419-23dc8a0944e4
github.com/rivo/uniseg v0.1.0 // indirect
github.com/stretchr/testify v1.3.0
golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4
golang.org/x/net v0.0.0-20190724013045-ca1201d0de80
golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e // indirect
......
......@@ -15,11 +15,11 @@ import (
func TestMarshal(t *testing.T) {
ownNodeId := identity.GenerateRandomIdentity().Identifier
toggledTransactions := make([]*heartbeatproto.ToggledTransaction, 1000)
toggledTransactions := make([]*heartbeatproto.ToggledTransaction, 10000)
for i := 0; i < len(toggledTransactions); i++ {
toggledTransactions[i] = &heartbeatproto.ToggledTransaction{
TransactionId: make([]byte, 32),
TransactionId: make([]byte, 3),
ToggleReason: 0,
}
}
......
package datastructure
import (
"bytes"
"fmt"
"sync"
)
type PrefixTrie struct {
value []byte
children map[byte]*PrefixTrie
size int
mutex sync.RWMutex
}
func NewPrefixTree() *PrefixTrie {
return &PrefixTrie{
value: nil,
children: make(map[byte]*PrefixTrie),
}
}
func (prefixTrie *PrefixTrie) Get(byteSequenceOrPrefix []byte) (result [][]byte) {
prefixTrie.mutex.RLock()
defer prefixTrie.mutex.RUnlock()
result = make([][]byte, 0)
currentNode := prefixTrie
for currentLevel := 0; currentLevel < len(byteSequenceOrPrefix); currentLevel++ {
if currentNode.value != nil && bytes.HasPrefix(currentNode.value, byteSequenceOrPrefix) {
result = append(result, currentNode.value)
return
}
if existingNode, exists := currentNode.children[byteSequenceOrPrefix[currentLevel]]; exists {
currentNode = existingNode
} else {
// error tried to inflate non-existing entry
return
}
}
if currentNode.value != nil {
result = append(result, currentNode.value)
} else {
// traverse child elements
if false {
fmt.Println("WAS")
}
}
return
}
func (prefixTrie *PrefixTrie) GetPrefix(insertedBytes []byte) []byte {
prefixTrie.mutex.RLock()
defer prefixTrie.mutex.RUnlock()
currentNode := prefixTrie
currentLevel := 0
for {
// if we have reached our target
if bytes.Equal(currentNode.value, insertedBytes) {
return insertedBytes[:currentLevel]
}
// if we have arrived at the wrong node: return nil
if currentNode.value != nil {
return nil
}
if childNode, exists := currentNode.children[insertedBytes[currentLevel]]; !exists {
return nil
} else {
currentNode = childNode
}
// increase level counter
currentLevel++
}
}
func (prefixTrie *PrefixTrie) Insert(byteSequence []byte) bool {
prefixTrie.mutex.Lock()
defer prefixTrie.mutex.Unlock()
currentNode := prefixTrie
currentLevel := 0
for {
// if we have reached our target node: insert value
if currentNode.value == nil && len(currentNode.children) == 0 {
currentNode.value = byteSequence
prefixTrie.size++
return true
}
// if we have reached a previous leaf
if currentNode.value != nil {
// return if same element
if bytes.Equal(currentNode.value, byteSequence) {
return false
}
// move current value to correct sub element
currentNode.children[currentNode.value[currentLevel]] = &PrefixTrie{
value: currentNode.value,
children: make(map[byte]*PrefixTrie),
}
// set the value to nil
currentNode.value = nil
}
// traverse or create correct child element
if existingChildNode, exists := currentNode.children[byteSequence[currentLevel]]; exists {
currentNode = existingChildNode
} else {
newNode := &PrefixTrie{
children: make(map[byte]*PrefixTrie),
}
currentNode.children[byteSequence[currentLevel]] = newNode
currentNode = newNode
}
// increase level counter
currentLevel++
}
}
func (prefixTrie *PrefixTrie) Delete(byteSequence []byte) bool {
prefixTrie.mutex.Lock()
defer prefixTrie.mutex.Unlock()
currentNode := prefixTrie
// trivial case: delete from root
if bytes.Equal(currentNode.value, byteSequence) {
currentNode.value = nil
prefixTrie.size--
return true
}
// non-trivial case: delete leaf
for currentLevel := 0; currentLevel < len(byteSequence); currentLevel++ {
if existingNode, exists := currentNode.children[byteSequence[currentLevel]]; exists {
if bytes.Equal(existingNode.value, byteSequence) {
delete(currentNode.children, byteSequence[currentLevel])
if len(currentNode.children) == 1 {
for index, currentChild := range currentNode.children {
if currentChild.value != nil {
currentNode.value = currentChild.value
delete(currentNode.children, index)
}
}
}
prefixTrie.size--
return true
}
currentNode = existingNode
} else {
return false
}
}
return false
}
func (prefixTrie *PrefixTrie) GetSize() int {
prefixTrie.mutex.RLock()
defer prefixTrie.mutex.RUnlock()
return prefixTrie.size
}
package datastructure
import (
"bytes"
"math/rand"
"testing"
"github.com/iotaledger/iota.go/trinary"
"github.com/stretchr/testify/assert"
)
func BenchmarkByteTrie_Insert(b *testing.B) {
trie := &PrefixTrie{
value: nil,
children: make(map[byte]*PrefixTrie),
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
token := make([]byte, 50)
rand.Read(token)
trie.Insert(token)
}
}
func TestPrefixTrie_GetPrefix(t *testing.T) {
trie := &PrefixTrie{
value: nil,
children: make(map[byte]*PrefixTrie),
}
var token []byte
for i := 0; i < 64000; i++ {
token = make([]byte, 50)
rand.Read(token)
trie.Insert(token)
}
assert.True(t, len(trie.GetPrefix(token)) <= 3)
}
func TestPrefixTrie_Insert(t *testing.T) {
trie := NewPrefixTree()
tx1Hash := trinary.MustTrytesToBytes("NDMXEXQFVRJKHVRGYURKMRMUUYUNPCREQHEZKNSEHK9SWSIQDU9IF9IHXTOIVVHWXGLJHHUR9NIGMAUQC")
tx2Hash := trinary.MustTrytesToBytes("HWTXYVLGPKUUFXGBBGKLMRCI9VOW9MEJRCJPRMUGCKHOVCCMFSZNAFEWOOVYUEHGYDWPWBEKWJPAZ9999")
tx3Hash := trinary.MustTrytesToBytes("IPVMHNESBZUKAIYFJCYEVXDBICITTLDUQIOVZODAISWCNEMBLWHVBDTEHNWRENIDEGVVODLXHTXGA9999")
assert.Equal(t, 0, trie.GetSize())
assert.Equal(t, true, trie.Insert(tx1Hash))
assert.Equal(t, true, trie.Insert(tx2Hash))
assert.Equal(t, true, trie.Insert(tx3Hash))
assert.Equal(t, false, trie.Insert(tx1Hash))
assert.Equal(t, 3, trie.GetSize())
txFound := false
for _, hash := range trie.Get(trie.GetPrefix(tx1Hash)) {
if bytes.Equal(hash, tx1Hash) {
txFound = true
}
}
assert.True(t, txFound)
assert.Equal(t, true, trie.Delete(tx1Hash))
assert.Equal(t, false, trie.Delete(tx1Hash))
assert.Equal(t, true, trie.Delete(tx2Hash))
assert.Equal(t, 1, trie.GetSize())
}
func TestPrefixTrie_Get(t *testing.T) {
trie := NewPrefixTree()
tx1Hash := trinary.MustTrytesToBytes("NDMXEXQFVRJKHVRGYURKMRMUUYUNPCREQHEZKNSEHK9SWSIQDU9IF9IHXTOIVVHWXGLJHHUR9NIGMAUQC")
tx2Hash := trinary.MustTrytesToBytes("HWTXYVLGPKUUFXGBBGKLMRCI9VOW9MEJRCJPRMUGCKHOVCCMFSZNAFEWOOVYUEHGYDWPWBEKWJPAZ9999")
trie.Insert(tx1Hash)
trie.Insert(tx2Hash)
prefix := trie.GetPrefix(tx1Hash)
resultsByPrefix := trie.Get(prefix)
resultsByFullHash := trie.Get(tx1Hash)
assert.Equal(t, len(resultsByPrefix), len(resultsByFullHash))
for i, foundHashCandidate := range resultsByPrefix {
assert.True(t, bytes.Equal(foundHashCandidate, resultsByFullHash[i]))
}
}
func TestPrefixTrie_GetSize(t *testing.T) {
trie := NewPrefixTree()
assert.Equal(t, 0, trie.GetSize())
tx1Hash := trinary.MustTrytesToBytes("NDMXEXQFVRJKHVRGYURKMRMUUYUNPCREQHEZKNSEHK9SWSIQDU9IF9IHXTOIVVHWXGLJHHUR9NIGMAUQC")
tx2Hash := trinary.MustTrytesToBytes("HWTXYVLGPKUUFXGBBGKLMRCI9VOW9MEJRCJPRMUGCKHOVCCMFSZNAFEWOOVYUEHGYDWPWBEKWJPAZ9999")
tx3Hash := trinary.MustTrytesToBytes("IPVMHNESBZUKAIYFJCYEVXDBICITTLDUQIOVZODAISWCNEMBLWHVBDTEHNWRENIDEGVVODLXHTXGA9999")
trie.Insert(tx1Hash)
trie.Insert(tx2Hash)
assert.Equal(t, 2, trie.GetSize())
trie.Insert(tx3Hash)
assert.Equal(t, 3, trie.GetSize())
trie.Delete(tx1Hash)
assert.Equal(t, 2, trie.GetSize())
trie.Delete(tx2Hash)
assert.Equal(t, 1, trie.GetSize())
trie.Delete(tx3Hash)
assert.Equal(t, 0, trie.GetSize())
trie.Delete(tx3Hash)
assert.Equal(t, 0, trie.GetSize())
}
func TestPrefixTrie_Delete(t *testing.T) {
trie := NewPrefixTree()
tx1Hash := trinary.MustTrytesToBytes("NDMXEXQFVRJKHVRGYURKMRMUUYUNPCREQHEZKNSEHK9SWSIQDU9IF9IHXTOIVVHWXGLJHHUR9NIGMAUQC")
tx2Hash := trinary.MustTrytesToBytes("HWTXYVLGPKUUFXGBBGKLMRCI9VOW9MEJRCJPRMUGCKHOVCCMFSZNAFEWOOVYUEHGYDWPWBEKWJPAZ9999")
tx3Hash := trinary.MustTrytesToBytes("IPVMHNESBZUKAIYFJCYEVXDBICITTLDUQIOVZODAISWCNEMBLWHVBDTEHNWRENIDEGVVODLXHTXGA9999")
trie.Insert(tx1Hash)
trie.Insert(tx2Hash)
assert.Equal(t, true, trie.Delete(tx1Hash))
assert.Equal(t, false, trie.Delete(tx1Hash))
assert.Equal(t, true, trie.Delete(tx2Hash))
assert.Equal(t, false, trie.Delete(tx3Hash))
assert.Equal(t, 0, trie.GetSize())
}
......@@ -5,7 +5,7 @@ import (
"testing"
"github.com/iotaledger/iota.go/trinary"
"github.com/magiconair/properties/assert"
"github.com/stretchr/testify/assert"
)
func TestApprovers_SettersGetters(t *testing.T) {
......
......@@ -4,7 +4,7 @@ import (
"testing"
"github.com/iotaledger/iota.go/trinary"
"github.com/magiconair/properties/assert"
"github.com/stretchr/testify/assert"
)
func TestBundle_SettersGetters(t *testing.T) {
......
......@@ -6,7 +6,7 @@ import (
"testing"
"github.com/iotaledger/iota.go/trinary"
"github.com/magiconair/properties/assert"
"github.com/stretchr/testify/assert"
)
func TestMetaTransaction_SettersGetters(t *testing.T) {
......
......@@ -5,7 +5,7 @@ import (
"testing"
"github.com/iotaledger/iota.go/trinary"
"github.com/magiconair/properties/assert"
"github.com/stretchr/testify/assert"
)
func TestValueTransaction_SettersGetters(t *testing.T) {
......
......@@ -8,7 +8,7 @@ import (
"github.com/iotaledger/goshimmer/plugins/autopeering/types/salt"
"github.com/iotaledger/goshimmer/packages/identity"
"github.com/magiconair/properties/assert"
"github.com/stretchr/testify/assert"
)
func TestPeer_MarshalUnmarshal(t *testing.T) {
......
......@@ -16,7 +16,7 @@ import (
"github.com/iotaledger/goshimmer/plugins/tangle"
"github.com/iotaledger/iota.go/consts"
"github.com/magiconair/properties/assert"
"github.com/stretchr/testify/assert"
)
var seed = client.NewSeed("YFHQWAUPCXC9S9DSHP9NDF9RLNPMZVCMSJKUKQP9SWUSUCPRQXCMDVDVZ9SHHESHIQNCXWBJF9UJSWE9Z", consts.SecurityLevelMedium)
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment