Skip to content
Snippets Groups Projects
Unverified Commit 18ca0dae authored by jonastheis's avatar jonastheis
Browse files

Start refactoring codebase to use new message layout - WIP

parent 95789c60
Branches
No related tags found
No related merge requests found
Showing with 137 additions and 107 deletions
......@@ -46,11 +46,8 @@ func ExamplePayload() {
// 3. build actual transaction (the base layer creates this and wraps the ontology provided payload)
tx := tangle.NewMessage(
// parent1 in "network tangle" ontology (filled by tipSelector)
tangle.EmptyMessageID,
// parent2 in "network tangle" ontology (filled by tipSelector)
tangle.EmptyMessageID,
[]tangle.MessageID{tangle.EmptyMessageID},
[]tangle.MessageID{},
// the time when the transaction was created
time.Now(),
......
......
......@@ -14,12 +14,15 @@ import (
"github.com/iotaledger/hive.go/marshalutil"
"github.com/iotaledger/hive.go/objectstorage"
"github.com/iotaledger/hive.go/stringify"
"github.com/iotaledger/hive.go/types"
"github.com/mr-tron/base58"
"golang.org/x/crypto/blake2b"
"golang.org/x/xerrors"
)
const (
MessageVersion uint8 = 1
// MaxMessageSize defines the maximum size of a message.
MaxMessageSize = 64 * 1024
......@@ -50,6 +53,7 @@ var EmptyMessageID = MessageID{}
// NewMessageID creates a new message id.
func NewMessageID(base58EncodedString string) (result MessageID, err error) {
// TODO: rename to avoid collision with imported package
bytes, err := base58.Decode(base58EncodedString)
if err != nil {
err = fmt.Errorf("failed to decode base58 encoded string '%s': %w", base58EncodedString, err)
......@@ -140,13 +144,21 @@ type Message struct {
}
// NewMessage creates a new message with the details provided by the issuer.
func NewMessage(parent1ID MessageID, parent2ID MessageID, issuingTime time.Time, issuerPublicKey ed25519.PublicKey, sequenceNumber uint64, payload payload.Payload, nonce uint64, signature ed25519.Signature) (result *Message) {
// TODO: do syntactical validation
func NewMessage(strongParents []MessageID, weakParents []MessageID, issuingTime time.Time, issuerPublicKey ed25519.PublicKey, sequenceNumber uint64, payload payload.Payload, nonce uint64, signature ed25519.Signature) (result *Message) {
// syntactical validation
parentsCount := len(strongParents) + len(weakParents)
if parentsCount < MinParentsCount || parentsCount > MaxParentsCount {
panic(fmt.Sprintf("amount of parents (%d) not in valid range (%d-%d)", parentsCount, MinParentsCount, MaxParentsCount))
}
if len(strongParents) < MinStrongParentsCount {
panic(fmt.Sprintf("amount of strong parents (%d) failed to reach MinStrongParentsCount (%d)", len(strongParents), MinStrongParentsCount))
}
return &Message{
// TODO: copy slices for strong/weak parents?
//parent1ID: parent1ID,
//parent2ID: parent2ID,
version: MessageVersion,
strongParents: sortParents(strongParents),
weakParents: sortParents(weakParents),
issuerPublicKey: issuerPublicKey,
issuingTime: issuingTime,
sequenceNumber: sequenceNumber,
......@@ -156,6 +168,28 @@ func NewMessage(parent1ID MessageID, parent2ID MessageID, issuingTime time.Time,
}
}
// filters and sorts given parents and returns a new slice with sorted parents
func sortParents(parents []MessageID) (sorted []MessageID) {
seen := make(map[MessageID]types.Empty)
sorted = make([]MessageID, 0, len(parents))
// filter duplicates
for _, parent := range parents {
if _, seenAlready := seen[parent]; seenAlready {
continue
}
seen[parent] = types.Void
sorted = append(sorted, parent)
}
// sort parents
sort.Slice(parents, func(i, j int) bool {
return bytes.Compare(parents[i].Bytes(), parents[j].Bytes()) < 0
})
return
}
// MessageFromBytes parses the given bytes into a message.
func MessageFromBytes(bytes []byte) (result *Message, consumedBytes int, err error) {
marshalUtil := marshalutil.New(bytes)
......
......
......@@ -16,13 +16,13 @@ func TestMessage_VerifySignature(t *testing.T) {
keyPair := ed25519.GenerateKeyPair()
pl := payload.NewGenericDataPayload([]byte("test"))
unsigned := NewMessage(EmptyMessageID, EmptyMessageID, time.Time{}, keyPair.PublicKey, 0, pl, 0, ed25519.Signature{})
unsigned := NewMessage([]MessageID{EmptyMessageID}, []MessageID{}, time.Time{}, keyPair.PublicKey, 0, pl, 0, ed25519.Signature{})
assert.False(t, unsigned.VerifySignature())
unsignedBytes := unsigned.Bytes()
signature := keyPair.PrivateKey.Sign(unsignedBytes[:len(unsignedBytes)-ed25519.SignatureSize])
signed := NewMessage(EmptyMessageID, EmptyMessageID, time.Time{}, keyPair.PublicKey, 0, pl, 0, signature)
signed := NewMessage([]MessageID{EmptyMessageID}, []MessageID{}, time.Time{}, keyPair.PublicKey, 0, pl, 0, signature)
assert.True(t, signed.VerifySignature())
}
......@@ -39,8 +39,8 @@ func TestMessage_MarshalUnmarshal(t *testing.T) {
restoredMessage, _, err := MessageFromBytes(testMessage.Bytes())
if assert.NoError(t, err, err) {
assert.Equal(t, testMessage.ID(), restoredMessage.ID())
assert.Equal(t, testMessage.Parent1ID(), restoredMessage.Parent1ID())
assert.Equal(t, testMessage.Parent2ID(), restoredMessage.Parent2ID())
assert.ElementsMatch(t, testMessage.StrongParents(), restoredMessage.StrongParents())
assert.ElementsMatch(t, testMessage.WeakParents(), restoredMessage.WeakParents())
assert.Equal(t, testMessage.IssuerPublicKey(), restoredMessage.IssuerPublicKey())
assert.Equal(t, testMessage.IssuingTime().Round(time.Second), restoredMessage.IssuingTime().Round(time.Second))
assert.Equal(t, testMessage.SequenceNumber(), restoredMessage.SequenceNumber())
......
......
......@@ -20,7 +20,7 @@ var (
// A TipSelector selects two tips, parent2 and parent1, for a new message to attach to.
type TipSelector interface {
Tips() (parent1 MessageID, parent2 MessageID)
Tips(count int) (parents []MessageID)
}
// A Worker performs the PoW for the provided message in serialized byte form.
......@@ -83,12 +83,16 @@ func (f *MessageFactory) IssuePayload(p payload.Payload) (*Message, error) {
return nil, err
}
parent1ID, parent2ID := f.selector.Tips()
// TODO: change hardcoded amount of parents
strongParents := f.selector.Tips(2)
// TODO: approval switch: select weak parents
weakParents := make([]MessageID, 0)
issuingTime := time.Now()
issuerPublicKey := f.localIdentity.PublicKey()
// do the PoW
nonce, err := f.doPOW(parent1ID, parent2ID, issuingTime, issuerPublicKey, sequenceNumber, p)
nonce, err := f.doPOW(strongParents, weakParents, issuingTime, issuerPublicKey, sequenceNumber, p)
if err != nil {
err = fmt.Errorf("pow failed: %w", err)
f.Events.Error.Trigger(err)
......@@ -96,11 +100,11 @@ func (f *MessageFactory) IssuePayload(p payload.Payload) (*Message, error) {
}
// create the signature
signature := f.sign(parent1ID, parent2ID, issuingTime, issuerPublicKey, sequenceNumber, p, nonce)
signature := f.sign(strongParents, weakParents, issuingTime, issuerPublicKey, sequenceNumber, p, nonce)
msg := NewMessage(
parent1ID,
parent2ID,
strongParents,
weakParents,
issuingTime,
issuerPublicKey,
sequenceNumber,
......@@ -119,18 +123,18 @@ func (f *MessageFactory) Shutdown() {
}
}
func (f *MessageFactory) doPOW(parent1ID MessageID, parent2ID MessageID, issuingTime time.Time, key ed25519.PublicKey, seq uint64, payload payload.Payload) (uint64, error) {
func (f *MessageFactory) doPOW(strongParents []MessageID, weakParents []MessageID, issuingTime time.Time, key ed25519.PublicKey, seq uint64, payload payload.Payload) (uint64, error) {
// create a dummy message to simplify marshaling
dummy := NewMessage(parent1ID, parent2ID, issuingTime, key, seq, payload, 0, ed25519.EmptySignature).Bytes()
dummy := NewMessage(strongParents, weakParents, issuingTime, key, seq, payload, 0, ed25519.EmptySignature).Bytes()
f.workerMutex.RLock()
defer f.workerMutex.RUnlock()
return f.worker.DoPOW(dummy)
}
func (f *MessageFactory) sign(parent1ID MessageID, parent2ID MessageID, issuingTime time.Time, key ed25519.PublicKey, seq uint64, payload payload.Payload, nonce uint64) ed25519.Signature {
func (f *MessageFactory) sign(strongParents []MessageID, weakParents []MessageID, issuingTime time.Time, key ed25519.PublicKey, seq uint64, payload payload.Payload, nonce uint64) ed25519.Signature {
// create a dummy message to simplify marshaling
dummy := NewMessage(parent1ID, parent2ID, issuingTime, key, seq, payload, nonce, ed25519.EmptySignature)
dummy := NewMessage(strongParents, weakParents, issuingTime, key, seq, payload, nonce, ed25519.EmptySignature)
dummyBytes := dummy.Bytes()
contentLength := len(dummyBytes) - len(dummy.Signature())
......@@ -138,11 +142,11 @@ func (f *MessageFactory) sign(parent1ID MessageID, parent2ID MessageID, issuingT
}
// The TipSelectorFunc type is an adapter to allow the use of ordinary functions as tip selectors.
type TipSelectorFunc func() (MessageID, MessageID)
type TipSelectorFunc func(count int) (parents []MessageID)
// Tips calls f().
func (f TipSelectorFunc) Tips() (MessageID, MessageID) {
return f()
func (f TipSelectorFunc) Tips(count int) (parents []MessageID) {
return f(count)
}
// The WorkerFunc type is an adapter to allow the use of ordinary functions as a PoW performer.
......
......
......@@ -30,7 +30,7 @@ func TestMessageFactory_BuildMessage(t *testing.T) {
mapdb.NewMapDB(),
[]byte(sequenceKey),
identity.GenerateLocalIdentity(),
TipSelectorFunc(func() (MessageID, MessageID) { return EmptyMessageID, EmptyMessageID }),
TipSelectorFunc(func(count int) []MessageID { return []MessageID{EmptyMessageID} }),
)
defer msgFactory.Shutdown()
......@@ -48,8 +48,8 @@ func TestMessageFactory_BuildMessage(t *testing.T) {
msg, err := msgFactory.IssuePayload(p)
require.NoError(t, err)
assert.NotNil(t, msg.Parent1ID())
assert.NotNil(t, msg.Parent2ID())
// TODO: approval switch: make test case with weak parents
assert.NotEmpty(t, msg.StrongParents())
// time in range of 0.1 seconds
assert.InDelta(t, time.Now().UnixNano(), msg.IssuingTime().UnixNano(), 100000000)
......@@ -74,8 +74,8 @@ func TestMessageFactory_BuildMessage(t *testing.T) {
msg, err := msgFactory.IssuePayload(p)
require.NoError(t, err)
assert.NotNil(t, msg.Parent1ID())
assert.NotNil(t, msg.Parent2ID())
// TODO: approval switch: make test case with weak parents
assert.NotEmpty(t, msg.StrongParents())
// time in range of 0.1 seconds
assert.InDelta(t, time.Now().UnixNano(), msg.IssuingTime().UnixNano(), 100000000)
......@@ -116,7 +116,7 @@ func TestMessageFactory_POW(t *testing.T) {
mapdb.NewMapDB(),
[]byte(sequenceKey),
identity.GenerateLocalIdentity(),
TipSelectorFunc(func() (MessageID, MessageID) { return EmptyMessageID, EmptyMessageID }),
TipSelectorFunc(func(count int) []MessageID { return []MessageID{EmptyMessageID} }),
)
defer msgFactory.Shutdown()
......@@ -143,7 +143,7 @@ func TestWorkerFunc_PayloadSize(t *testing.T) {
mapdb.NewMapDB(),
[]byte(sequenceKey),
identity.GenerateLocalIdentity(),
TipSelectorFunc(func() (MessageID, MessageID) { return EmptyMessageID, EmptyMessageID }),
TipSelectorFunc(func(count int) []MessageID { return []MessageID{EmptyMessageID} }),
)
defer msgFactory.Shutdown()
......
......
......@@ -18,7 +18,7 @@ func BenchmarkVerifyDataMessages(b *testing.B) {
var pool async.WorkerPool
pool.Tune(runtime.GOMAXPROCS(0))
factory := NewMessageFactory(mapdb.NewMapDB(), []byte(DBSequenceNumber), identity.GenerateLocalIdentity(), TipSelectorFunc(func() (MessageID, MessageID) { return EmptyMessageID, EmptyMessageID }))
factory := NewMessageFactory(mapdb.NewMapDB(), []byte(DBSequenceNumber), identity.GenerateLocalIdentity(), TipSelectorFunc(func(count int) []MessageID { return []MessageID{EmptyMessageID} }))
messages := make([][]byte, b.N)
for i := 0; i < b.N; i++ {
......@@ -46,7 +46,7 @@ func BenchmarkVerifyDataMessages(b *testing.B) {
func BenchmarkVerifySignature(b *testing.B) {
pool, _ := ants.NewPool(80, ants.WithNonblocking(false))
factory := NewMessageFactory(mapdb.NewMapDB(), []byte(DBSequenceNumber), identity.GenerateLocalIdentity(), TipSelectorFunc(func() (MessageID, MessageID) { return EmptyMessageID, EmptyMessageID }))
factory := NewMessageFactory(mapdb.NewMapDB(), []byte(DBSequenceNumber), identity.GenerateLocalIdentity(), TipSelectorFunc(func(count int) []MessageID { return []MessageID{EmptyMessageID} }))
messages := make([]*Message, b.N)
for i := 0; i < b.N; i++ {
......
......
......@@ -90,13 +90,11 @@ func (t *Tangle) Approvers(messageID MessageID) CachedApprovers {
// message as an approver.
func (t *Tangle) DeleteMessage(messageID MessageID) {
t.Message(messageID).Consume(func(currentMsg *Message) {
parent1MsgID := currentMsg.Parent1ID()
t.deleteApprover(parent1MsgID, messageID)
parent2MsgID := currentMsg.Parent2ID()
if parent2MsgID != parent1MsgID {
t.deleteApprover(parent2MsgID, messageID)
}
// TODO: reconsider behavior with approval switch
currentMsg.ForEachParent(func(parent Parent) {
t.deleteApprover(parent.ID, messageID)
})
t.messageMetadataStorage.Delete(messageID[:])
t.messageStorage.Delete(messageID[:])
......@@ -196,14 +194,11 @@ func (t *Tangle) storeMessageWorker(msg *Message) {
messageID := msg.ID()
cachedMsgMetadata := &CachedMessageMetadata{CachedObject: t.messageMetadataStorage.Store(NewMessageMetadata(messageID))}
// store parent1 approver
parent1MsgID := msg.Parent1ID()
t.approverStorage.Store(NewApprover(parent1MsgID, messageID)).Release()
// store parent2 approver
if parent2MsgID := msg.Parent2ID(); parent2MsgID != parent1MsgID {
t.approverStorage.Store(NewApprover(parent2MsgID, messageID)).Release()
}
// TODO: approval switch: we probably need to introduce approver types
// store approvers
msg.ForEachStrongParent(func(parent MessageID) {
t.approverStorage.Store(NewApprover(parent, messageID)).Release()
})
// trigger events
if t.missingMessageStorage.DeleteIfPresent(messageID[:]) {
......@@ -269,9 +264,13 @@ func (t *Tangle) isMessageSolid(msg *Message, msgMetadata *MessageMetadata) bool
}
// as missing messages are requested in isMessageMarkedAsSolid, we want to prevent short-circuit evaluation
parent1Solid := t.isMessageMarkedAsSolid(msg.Parent1ID())
parent2Solid := t.isMessageMarkedAsSolid(msg.Parent2ID())
return parent1Solid && parent2Solid
solid := true
msg.ForEachParent(func(parent Parent) {
solid = solid && t.isMessageMarkedAsSolid(parent.ID)
})
return solid
}
// builds up a stack from the given message and tries to solidify into the present.
......
......
......@@ -126,11 +126,8 @@ func TestTangle_MissingMessages(t *testing.T) {
// remove a tip if the width of the tangle is reached
if tips.Size() >= widthOfTheTangle {
if rand.Intn(1000) < 500 {
tips.Delete(msg.Parent2ID())
} else {
tips.Delete(msg.Parent1ID())
}
index := rand.Intn(len(msg.StrongParents()))
tips.Delete(msg.StrongParents()[index])
}
// add current message as a tip
......
......
......@@ -8,13 +8,13 @@ import (
)
func newTestNonceMessage(nonce uint64) *Message {
return NewMessage(EmptyMessageID, EmptyMessageID, time.Time{}, ed25519.PublicKey{}, 0, payload.NewGenericDataPayload([]byte("test")), nonce, ed25519.Signature{})
return NewMessage([]MessageID{EmptyMessageID}, []MessageID{}, time.Time{}, ed25519.PublicKey{}, 0, payload.NewGenericDataPayload([]byte("test")), nonce, ed25519.Signature{})
}
func newTestDataMessage(payloadString string) *Message {
return NewMessage(EmptyMessageID, EmptyMessageID, time.Now(), ed25519.PublicKey{}, 0, payload.NewGenericDataPayload([]byte(payloadString)), 0, ed25519.Signature{})
return NewMessage([]MessageID{EmptyMessageID}, []MessageID{}, time.Now(), ed25519.PublicKey{}, 0, payload.NewGenericDataPayload([]byte(payloadString)), 0, ed25519.Signature{})
}
func newTestParentsDataMessage(payloadString string, parent1, parent2 MessageID) *Message {
return NewMessage(parent1, parent2, time.Now(), ed25519.PublicKey{}, 0, payload.NewGenericDataPayload([]byte(payloadString)), 0, ed25519.Signature{})
func newTestParentsDataMessage(payloadString string, strongParents, weakParents []MessageID) *Message {
return NewMessage(strongParents, weakParents, time.Now(), ed25519.PublicKey{}, 0, payload.NewGenericDataPayload([]byte(payloadString)), 0, ed25519.Signature{})
}
......@@ -38,34 +38,28 @@ func (t *MessageTipSelector) AddTip(msg *Message) {
t.Events.TipAdded.Trigger(messageID)
}
parent1MessageID := msg.Parent1ID()
if _, deleted := t.tips.Delete(parent1MessageID); deleted {
t.Events.TipRemoved.Trigger(parent1MessageID)
}
parent2MessageID := msg.Parent2ID()
if _, deleted := t.tips.Delete(parent2MessageID); deleted {
t.Events.TipRemoved.Trigger(parent2MessageID)
msg.ForEachStrongParent(func(parent MessageID) {
if _, deleted := t.tips.Delete(parent); deleted {
t.Events.TipRemoved.Trigger(parent)
}
})
}
// Tips returns two tips.
func (t *MessageTipSelector) Tips() (parent1MessageID, parent2MessageID MessageID) {
func (t *MessageTipSelector) Tips(count int) (parents []MessageID) {
parents = make([]MessageID, 0, count)
tip := t.tips.RandomEntry()
if tip == nil {
parent1MessageID = EmptyMessageID
parent2MessageID = EmptyMessageID
parents = append(parents, EmptyMessageID)
return
}
parent2MessageID = tip.(MessageID)
if t.tips.Size() == 1 {
parent1MessageID = parent2MessageID
return
}
tipMessageID := tip.(MessageID)
parents = append(parents, tipMessageID)
// TODO: adjust tip selection to select as many tips as count
// it is a bit tricky to not cause a deadlock if we don't allow duplicates
parent1MessageID = t.tips.RandomEntry().(MessageID)
for parent1MessageID == parent2MessageID && t.tips.Size() > 1 {
parent1MessageID = t.tips.RandomEntry().(MessageID)
......
......
......@@ -52,8 +52,8 @@ func createExplorerMessage(msg *tangle.Message) (*ExplorerMessage, error) {
IssuerPublicKey: msg.IssuerPublicKey().String(),
Signature: msg.Signature().String(),
SequenceNumber: msg.SequenceNumber(),
Parent1MessageID: msg.Parent1ID().String(),
Parent2MessageID: msg.Parent2ID().String(),
//Parent1MessageID: msg.Parent1ID().String(),
//Parent2MessageID: msg.Parent2ID().String(),
Solid: cachedMessageMetadata.Unwrap().IsSolid(),
PayloadType: uint32(msg.Payload().Type()),
Payload: ProcessPayload(msg.Payload()),
......
......
......@@ -53,8 +53,8 @@ func sendVertex(cachedMessage *tangle.CachedMessage, cachedMessageMetadata *tang
}
broadcastWsMessage(&wsmsg{MsgTypeVertex, &vertex{
ID: msg.ID().String(),
Parent1ID: msg.Parent1ID().String(),
Parent2ID: msg.Parent2ID().String(),
//Parent1ID: msg.Parent1ID().String(),
//Parent2ID: msg.Parent2ID().String(),
IsSolid: cachedMessageMetadata.Unwrap().IsSolid(),
}}, true)
}
......
......
......@@ -47,8 +47,9 @@ func findByIDHandler(c echo.Context) error {
SolidificationTime: msgMetadata.SolidificationTime().Unix(),
},
ID: msg.ID().String(),
Parent1ID: msg.Parent1ID().String(),
Parent2ID: msg.Parent2ID().String(),
//TODO: adjust
//Parent1ID: msg.Parent1ID().String(),
//Parent2ID: msg.Parent2ID().String(),
IssuerPublicKey: msg.IssuerPublicKey().String(),
IssuingTime: msg.IssuingTime().Unix(),
SequenceNumber: msg.SequenceNumber(),
......
......
......@@ -48,26 +48,30 @@ func PastconeHandler(c echo.Context) error {
// get parent1 and parent2
msg := msgObject.Unwrap()
parent2ID := msg.Parent2ID()
parent1ID := msg.Parent1ID()
onlyGenesis := true
msg.ForEachParent(func(parent tangle.Parent) {
onlyGenesis = onlyGenesis && (parent.ID == tangle.EmptyMessageID)
})
if onlyGenesis {
// release objects
msgObject.Release()
msgMetadataObject.Release()
if parent2ID == tangle.EmptyMessageID && msg.Parent1ID() == tangle.EmptyMessageID {
// msg only attaches to genesis
continue
} else {
if !submitted[parent2ID] && parent2ID != tangle.EmptyMessageID {
stack.PushBack(parent2ID)
submitted[parent2ID] = true
}
if !submitted[parent1ID] && parent1ID != tangle.EmptyMessageID {
stack.PushBack(parent1ID)
submitted[parent1ID] = true
msg.ForEachParent(func(parent tangle.Parent) {
if !submitted[parent.ID] && parent.ID != tangle.EmptyMessageID {
stack.PushBack(parent.ID)
submitted[parent.ID] = true
}
})
}
// release objects
msgObject.Release()
msgMetadataObject.Release()
}
return c.JSON(http.StatusOK, PastconeResponse{Exist: true, PastConeSize: checkedMessageCount})
}
......
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please to comment