diff --git a/go.mod b/go.mod index 688761a4..5d3e1766 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,7 @@ require ( github.com/dgraph-io/badger/v4 v4.2.0 github.com/dominikbraun/graph v0.23.0 github.com/gammazero/deque v0.2.1 + github.com/gorilla/websocket v1.5.1 github.com/libp2p/go-libp2p v0.35.1 github.com/libp2p/go-libp2p-kad-dht v0.25.2 github.com/lunfardo314/easyfl v0.0.0-20241220192652-d6fac839a94a @@ -55,7 +56,6 @@ require ( github.com/google/gopacket v1.1.19 // indirect github.com/google/pprof v0.0.0-20240207164012-fb44976bdcd5 // indirect github.com/google/uuid v1.4.0 // indirect - github.com/gorilla/websocket v1.5.1 // indirect github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/golang-lru v0.5.4 // indirect diff --git a/sequencer/task/proposer_endorse1.go b/sequencer/task/proposer_endorse1.go index 7416e1e3..63dc706b 100644 --- a/sequencer/task/proposer_endorse1.go +++ b/sequencer/task/proposer_endorse1.go @@ -31,16 +31,25 @@ func endorse1ProposeGenerator(p *Proposer) (*attacher.IncrementalAttacher, bool) // use pair with new outputs return true } - pair := extendEndorsePair{ - extend: extend, - endorse: endorse, - } - if !p.Task.slotData.alreadyCheckedE1.Contains(pair) { - // it is new pair. Use it and save it as already checked -> next time will be filtered out - p.Task.slotData.alreadyCheckedE1.Insert(pair) - return true - } - return false + return p.Task.slotData.checkCombination(extend, endorse) + + //combHash := extendEndorseCombinationHash(extend, endorse) + //if !p.Task.slotData.alreadyCheckedExtendEndorseCombination.Contains(combHash) { + // // it is new pair. Use it and save it as already checked -> next time will be filtered out + // p.Task.slotData.alreadyCheckedExtendEndorseCombination.Insert(combHash) + // return true + //} + // + //pair := extendEndorsePair{ + // extend: extend, + // endorse: endorse, + //} + //if !p.Task.slotData.alreadyCheckedE1.Contains(pair) { + // // it is new pair. Use it and save it as already checked -> next time will be filtered out + // p.Task.slotData.alreadyCheckedE1.Insert(pair) + // return true + //} + //return false }) if a == nil { diff --git a/sequencer/task/proposer_endorse2.go b/sequencer/task/proposer_endorse2.go index e2bc2a4a..86fa875a 100644 --- a/sequencer/task/proposer_endorse2.go +++ b/sequencer/task/proposer_endorse2.go @@ -52,37 +52,39 @@ func endorse2ProposeGenerator(p *Proposer) (*attacher.IncrementalAttacher, bool) if endorsementCandidate == endorsing0 { continue } - - triplet := extendEndorseTriplet{ - extend: extending, - endorse1: endorsing, - endorse2: endorsementCandidate, - } if !newOutputArrived { - checkedInThePast := false - p.slotData.withWriteLock(func() { - // optimization: skipping repeating triplets if new outputs didn't arrive meanwhile - checkedInThePast = p.slotData.alreadyCheckedTriplets.Contains(triplet) - if !checkedInThePast { - // assume invariance wrt order of endorsements - // check triplet with swapped endorsements - checkedInThePast = p.slotData.alreadyCheckedTriplets.Contains(extendEndorseTriplet{ - extend: extending, - endorse1: endorsementCandidate, - endorse2: endorsing, - }) - } - }) - if checkedInThePast { - continue - } + continue } + if !p.Task.slotData.checkCombination(extending, endorsing, endorsementCandidate) { + continue + } + + //triplet := extendEndorseTriplet{ + // extend: extending, + // endorse1: endorsing, + // endorse2: endorsementCandidate, + //} + //if !newOutputArrived { + // checkedInThePast := false + // p.slotData.withWriteLock(func() { + // // optimization: skipping repeating triplets if new outputs didn't arrive meanwhile + // checkedInThePast = p.slotData.alreadyCheckedTriplets.Contains(triplet) + // if !checkedInThePast { + // // assume invariance wrt order of endorsements + // // check triplet with swapped endorsements + // checkedInThePast = p.slotData.alreadyCheckedTriplets.Contains(extendEndorseTriplet{ + // extend: extending, + // endorse1: endorsementCandidate, + // endorse2: endorsing, + // }) + // } + // }) + // if checkedInThePast { + // continue + // } + //} if err := a.InsertEndorsement(endorsementCandidate); err == nil { - // remember triplet for the next check, same list for e2 and r2 - p.slotData.withWriteLock(func() { - p.slotData.alreadyCheckedTriplets.Insert(triplet) - }) addedSecond = true break //>>>> return attacher } diff --git a/sequencer/task/proposer_endorse2rnd.go b/sequencer/task/proposer_endorse2rnd.go index c1d5e841..23aca450 100644 --- a/sequencer/task/proposer_endorse2rnd.go +++ b/sequencer/task/proposer_endorse2rnd.go @@ -53,37 +53,43 @@ func endorse2RndProposeGenerator(p *Proposer) (*attacher.IncrementalAttacher, bo if endorsementCandidate == endorsing0 { continue } - - triplet := extendEndorseTriplet{ - extend: extending, - endorse1: endorsing, - endorse2: endorsementCandidate, - } if !newOutputArrived { - checkedInThePast := false - p.slotData.withWriteLock(func() { - // optimization: skipping repeating triplets if new outputs didn't arrive meanwhile - checkedInThePast = p.slotData.alreadyCheckedTriplets.Contains(triplet) - if !checkedInThePast { - // assume invariance wrt order of endorsements - // check triplet with swapped endorsements - checkedInThePast = p.slotData.alreadyCheckedTriplets.Contains(extendEndorseTriplet{ - extend: extending, - endorse1: endorsementCandidate, - endorse2: endorsing, - }) - } - }) - if checkedInThePast { - continue - } + continue + } + if !p.slotData.checkCombination(extending, endorsing, endorsementCandidate) { + continue } + // + //triplet := extendEndorseTriplet{ + // extend: extending, + // endorse1: endorsing, + // endorse2: endorsementCandidate, + //} + //if !newOutputArrived { + // checkedInThePast := false + // p.slotData.withWriteLock(func() { + // // optimization: skipping repeating triplets if new outputs didn't arrive meanwhile + // checkedInThePast = p.slotData.alreadyCheckedTriplets.Contains(triplet) + // if !checkedInThePast { + // // assume invariance wrt order of endorsements + // // check triplet with swapped endorsements + // checkedInThePast = p.slotData.alreadyCheckedTriplets.Contains(extendEndorseTriplet{ + // extend: extending, + // endorse1: endorsementCandidate, + // endorse2: endorsing, + // }) + // } + // }) + // if checkedInThePast { + // continue + // } + //} if err := a.InsertEndorsement(endorsementCandidate); err == nil { // remember triplet for the next check, same list for e2 and r2 - p.slotData.withWriteLock(func() { - p.slotData.alreadyCheckedTriplets.Insert(triplet) - }) + //p.slotData.withWriteLock(func() { + // p.slotData.alreadyCheckedTriplets.Insert(triplet) + //}) addedSecond = true break //>>>> return attacher } diff --git a/sequencer/task/proposer_endorse3.go b/sequencer/task/proposer_endorse3.go new file mode 100644 index 00000000..b1cba004 --- /dev/null +++ b/sequencer/task/proposer_endorse3.go @@ -0,0 +1,87 @@ +package task + +import ( + "time" + + "github.com/lunfardo314/proxima/core/attacher" +) + +const TraceTagEndorse3Proposer = "propose-endorse3" + +// TODO WIP + +//func init() { +// registerProposerStrategy(&Strategy{ +// Name: "endorse3", +// ShortName: "e3", +// GenerateProposal: endorse3ProposeGenerator, +// }) +//} + +func endorse3ProposeGenerator(p *Proposer) (*attacher.IncrementalAttacher, bool) { + if p.targetTs.IsSlotBoundary() { + // the proposer does not generate branch transactions + return nil, true + } + + // Check all pairs, in descending order + a := p.ChooseFirstExtendEndorsePair(false, nil) + if a == nil { + p.Tracef(TraceTagEndorse3Proposer, "propose: ChooseFirstExtendEndorsePair returned nil") + return nil, false + } + endorsing := a.Endorsing()[0] + extending := a.Extending() + if !a.Completed() { + a.Close() + p.Tracef(TraceTagEndorse3Proposer, "proposal [extend=%s, endorsing=%s] not complete 1", extending.IDShortString, endorsing.IDShortString) + return nil, false + } + + newOutputArrived := p.Backlog().ArrivedOutputsSince(p.slotData.lastTimeBacklogCheckedE2) + p.slotData.lastTimeBacklogCheckedE2 = time.Now() + + // then try to add one endorsement more + addedSecond := false + endorsing0 := a.Endorsing()[0] + for _, endorsementCandidate := range p.Backlog().CandidatesToEndorseSorted(p.targetTs) { + select { + case <-p.ctx.Done(): + a.Close() + return nil, true + default: + } + if endorsementCandidate == endorsing0 { + continue + } + if !newOutputArrived { + continue + } + if !p.Task.slotData.checkCombination(extending, endorsing, endorsementCandidate) { + continue + } + + if err := a.InsertEndorsement(endorsementCandidate); err == nil { + addedSecond = true + break //>>>> return attacher + } + p.Tracef(TraceTagEndorse3Proposer, "failed to include endorsement target %s", endorsementCandidate.IDShortString) + } + if !addedSecond { + // no need to repeat job of endorse1 + a.Close() + return nil, false + } + + p.insertInputs(a) + + if !a.Completed() { + a.Close() + endorsing = a.Endorsing()[0] + extending = a.Extending() + p.Tracef(TraceTagEndorse3Proposer, "proposal [extend=%s, endorsing=%s] not complete 2", extending.IDShortString, endorsing.IDShortString) + return nil, false + } + + return a, false +} diff --git a/sequencer/task/slot_data.go b/sequencer/task/slot_data.go index c235c4ac..c6f430aa 100644 --- a/sequencer/task/slot_data.go +++ b/sequencer/task/slot_data.go @@ -1,6 +1,9 @@ package task import ( + "bytes" + "slices" + "sort" "sync" "time" @@ -9,6 +12,7 @@ import ( "github.com/lunfardo314/proxima/util" "github.com/lunfardo314/proxima/util/lines" "github.com/lunfardo314/proxima/util/set" + "golang.org/x/crypto/blake2b" ) // SlotData collect values of sequencer during one slot @@ -33,8 +37,11 @@ type ( lastTimeBacklogCheckedE2 time.Time lastTimeBacklogCheckedR2 time.Time alreadyCheckedTriplets set.Set[extendEndorseTriplet] //shared by e2 and r2 + // extend proposers optimization + alreadyCheckedExtendEndorseCombination set.Set[combinationHash] } + combinationHash [8]byte extendEndorsePair struct { extend vertex.WrappedOutput endorse *vertex.WrappedTx @@ -49,11 +56,12 @@ type ( func NewSlotData(slot ledger.Slot) *SlotData { return &SlotData{ - slot: slot, - seqTxSubmitted: make([]ledger.TransactionID, 0), - proposalsByProposer: make(map[string]int), - alreadyCheckedE1: set.New[extendEndorsePair](), - alreadyCheckedTriplets: set.New[extendEndorseTriplet](), + slot: slot, + seqTxSubmitted: make([]ledger.TransactionID, 0), + proposalsByProposer: make(map[string]int), + alreadyCheckedE1: set.New[extendEndorsePair](), + alreadyCheckedTriplets: set.New[extendEndorseTriplet](), + alreadyCheckedExtendEndorseCombination: set.New[combinationHash](), } } @@ -122,3 +130,32 @@ func (s *SlotData) withWriteLock(fun func()) { fun() s.mutex.Unlock() } + +func extendEndorseCombinationHash(extend vertex.WrappedOutput, endorse ...*vertex.WrappedTx) (ret combinationHash) { + endorseSorted := slices.Clone(endorse) + sort.Slice(endorseSorted, func(i, j int) bool { + return ledger.LessTxID(endorseSorted[i].ID, endorseSorted[j].ID) + }) + + var buf bytes.Buffer + + buf.Write(extend.VID.ID[:]) + buf.WriteByte(extend.Index) + for i := range endorseSorted { + buf.Write(endorseSorted[i].ID[:]) + } + retSlice := blake2b.Sum256(buf.Bytes()) + copy(ret[:], retSlice[:]) + return +} + +// checkCombination checks combination and inserts into the list. Returns true if it is new combination +func (s *SlotData) checkCombination(extend vertex.WrappedOutput, endorse ...*vertex.WrappedTx) (ret bool) { + combHash := extendEndorseCombinationHash(extend, endorse...) + s.withWriteLock(func() { + if ret = !s.alreadyCheckedExtendEndorseCombination.Contains(combHash); ret { + s.alreadyCheckedExtendEndorseCombination.Insert(combHash) + } + }) + return +}