From a7ebb6d0d10bc792a913680cdd50b4601ef54793 Mon Sep 17 00:00:00 2001 From: ia Date: Mon, 23 Oct 2017 13:10:46 -0500 Subject: [PATCH 01/18] problem: unimplemented debug methods in api solution: remove methods from web3ext resolves #364 --- internal/web3ext/web3ext.go | 97 ------------------------------------- 1 file changed, 97 deletions(-) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 6e6930e52..e1aa668c5 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -169,26 +169,6 @@ web3._extend({ call: 'debug_setHead', params: 1 }), - new web3._extend.Method({ - name: 'traceBlock', - call: 'debug_traceBlock', - params: 1 - }), - new web3._extend.Method({ - name: 'traceBlockByFile', - call: 'debug_traceBlockByFile', - params: 1 - }), - new web3._extend.Method({ - name: 'traceBlockByNumber', - call: 'debug_traceBlockByNumber', - params: 1 - }), - new web3._extend.Method({ - name: 'traceBlockByHash', - call: 'debug_traceBlockByHash', - params: 1 - }), new web3._extend.Method({ name: 'seedHash', call: 'debug_seedHash', @@ -199,12 +179,6 @@ web3._extend({ call: 'debug_dumpBlock', params: 1 }), - new web3._extend.Method({ - name: 'chaindbProperty', - call: 'debug_chaindbProperty', - params: 1, - outputFormatter: console.log - }), new web3._extend.Method({ name: 'metrics', call: 'debug_metrics', @@ -223,77 +197,6 @@ web3._extend({ params: 1, inputFormatter: [web3._extend.formatters.inputOptionalStringFormatter] }), - new web3._extend.Method({ - name: 'backtraceAt', - call: 'debug_backtraceAt', - params: 1, - }), - new web3._extend.Method({ - name: 'stacks', - call: 'debug_stacks', - params: 0, - outputFormatter: console.log - }), - new web3._extend.Method({ - name: 'memStats', - call: 'debug_memStats', - params: 0, - }), - new web3._extend.Method({ - name: 'gcStats', - call: 'debug_gcStats', - params: 0, - }), - new web3._extend.Method({ - name: 'cpuProfile', - call: 'debug_cpuProfile', - params: 2 - }), - new web3._extend.Method({ - name: 'startCPUProfile', - call: 'debug_startCPUProfile', - params: 1 - }), - new web3._extend.Method({ - name: 'stopCPUProfile', - call: 'debug_stopCPUProfile', - params: 0 - }), - new web3._extend.Method({ - name: 'goTrace', - call: 'debug_goTrace', - params: 2 - }), - new web3._extend.Method({ - name: 'startGoTrace', - call: 'debug_startGoTrace', - params: 1 - }), - new web3._extend.Method({ - name: 'stopGoTrace', - call: 'debug_stopGoTrace', - params: 0 - }), - new web3._extend.Method({ - name: 'blockProfile', - call: 'debug_blockProfile', - params: 2 - }), - new web3._extend.Method({ - name: 'setBlockProfileRate', - call: 'debug_setBlockProfileRate', - params: 1 - }), - new web3._extend.Method({ - name: 'writeBlockProfile', - call: 'debug_writeBlockProfile', - params: 1 - }), - new web3._extend.Method({ - name: 'writeMemProfile', - call: 'debug_writeMemProfile', - params: 1 - }), new web3._extend.Method({ name: 'traceTransaction', call: 'debug_traceTransaction', From bfcdc4686d60163ab85f7eb4ab092cecd74db432 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Zdyba=C5=82?= Date: Thu, 23 Nov 2017 13:43:23 +0100 Subject: [PATCH 02/18] More information about building from source Added sections about building specific tag from source and using release tarball. --- README.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/README.md b/README.md index ca3cd1532..56f1e2cda 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,27 @@ To install... Executables built from source will, by default, be installed in `$GOPATH/bin/`. +#### Building specific release +All the above commands results with building binaries from `HEAD`. To use a specific release/tag, use the following: +``` +$ go get -d github.com/ethereumproject/go-ethereum/... +$ cd $GOPATH/src/github.com/ethereumproject/go-ethereum +$ git checkout +$ go install -ldflags "-X main.Version="`git describe --tags` ./cmd/... +``` + +#### Using release source code tarball +Because of strict Go directory structure, tarball needs to be extracted into proper subdirectory under `$GOPATH`. +Following commands are example of building the v4.1.1 release: +``` +$ mkdir -p $GOPATH/src/github.com/ethereumproject +$ cd $GOPATH/src/github.com/ethereumproject +$ tar xzf /path/to/go-ethereum-4.1.1.tar.gz +$ mv go-ethereum-4.1.1 go-ethereum +$ cd go-ethereum +$ go install -ldflags "-X main.Version=v4.1.1" ./cmd/... +``` + ## Executables This repository includes several wrappers/executables found in the `cmd` directory. From 4f1a4e9d0cd543781c229e921fe065d1fc9e3ecc Mon Sep 17 00:00:00 2001 From: ia Date: Fri, 17 Nov 2017 19:43:39 +0700 Subject: [PATCH 03/18] problem: sync head tracking stalling solution: calculate local and peer TD based on current block (parent of announced) at NewBlockMsg (eth/handler.go#L646) yields: - some additional mlog logs and components ---- Below a complete history of my failures and successes along the way.... think about splitting addresses into ip,port ... but it can be ipv4 OR ipv6, so ain't as simple as split on : Revert "think about splitting addresses into ip,port" This reverts commit a7b1f7734a7cba619fe0b2e1a19557ef3b7d0ebe. add uncles count and received at to block.write mlog add fetcher.discard.announcement to mlog fix block fetcher logging quite abbreviating hashes, it is annoying problem: client stalling sporadically around importing block with unknown parent solution(attempt): try forgetting hash when importing block with unknown parent think about using forgetBlock instead of forgetHash check for block/header existence before queuing check for block/header existence before queuing use fetching check for expiration to inject unfetched block if not already have use fetching map w/ reschedule instead of nil-pointery injection problem: reorg conditional should use block number solution: remove fetcher resets and modify reorg conditional in blockchain insertchain problem: inconsistent and abbreviated hex logging solution: use hash.Hex standaard and remove dead commented code problem: still some inconsistent hash logging solution: use hex() add mlog event for blockchain reorg problem: relying on logstash to aggregate block time diffs is a pain in the ass solution: calculate block time diffs on write block for mlog and create field BLOCK.DIFF_PARENT_TIME problem: nil pointer on parentTimeDiff because I'm a dumbass solution: fix random notes in fetcher hack hack a fake announcement notifaction capitalize hack remove unused imports problem: hack is whack; forked chain not being followed solution: don't forget announced block w/o parent, leave it in queue for fetcher to gap fill problem: ChainSplitTy is never used solution: remove it log whole hashes, not abbreviations problem: fetcher stalled because of unannounced parent block for best chain head solution: check for local existence of trueHead when handling Block message from peer, if we don't have trueHead for block, get it log more about peer best TD and trueHead mgmt and catch to make sure test syncronising Revert "log more about peer best TD and trueHead mgmt and catch" This reverts commit 390a487910a3a57525d98d835bb923b88f39b100. Revert "problem: fetcher stalled because of unannounced parent block for best chain head" This reverts commit a50ece541716eaa36b9d9bc593afd0a4a08e151d. calcualte trueTD before setting peer head add logs for debugging around peerTd just in case add downloader.peer.un/register mlogs display full tx hash when logging detail broadcast tx to peers add mlog for downloader.tune.qos remove a measly space --- core/blockchain.go | 61 ++++++++++++++++++-------- core/mlog.go | 17 ++++++++ eth/downloader/downloader.go | 43 +++++++++++++++--- eth/downloader/mlog.go | 49 +++++++++++++++++++++ eth/fetcher/fetcher.go | 85 +++++++++++++++++++++++++++++------- eth/fetcher/mlog.go | 22 ++++++++++ eth/handler.go | 15 ++++--- 7 files changed, 248 insertions(+), 44 deletions(-) create mode 100644 eth/downloader/mlog.go create mode 100644 eth/fetcher/mlog.go diff --git a/core/blockchain.go b/core/blockchain.go index 29c305626..059944f65 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -150,7 +150,7 @@ func NewBlockChain(chainDb ethdb.Database, config *ChainConfig, pow pow.PoW, mux // Check the current state of the block hashes and make sure that we do not have any of the bad blocks in our chain for i := range config.BadHashes { if header := bc.GetHeader(config.BadHashes[i].Hash); header != nil && header.Number.Cmp(config.BadHashes[i].Block) == 0 { - glog.V(logger.Error).Infof("Found bad hash, rewinding chain to block #%d [%x…]", header.Number, header.ParentHash[:4]) + glog.V(logger.Error).Infof("Found bad hash, rewinding chain to block #%d [%s]", header.Number, header.ParentHash.Hex()) bc.SetHead(header.Number.Uint64() - 1) glog.V(logger.Error).Infoln("Chain rewind was successful, resuming normal operation") } @@ -198,7 +198,7 @@ func NewBlockChainDryrun(chainDb ethdb.Database, config *ChainConfig, pow pow.Po // Check the current state of the block hashes and make sure that we do not have any of the bad blocks in our chain for i := range config.BadHashes { if header := bc.GetHeader(config.BadHashes[i].Hash); header != nil && header.Number.Cmp(config.BadHashes[i].Block) == 0 { - glog.V(logger.Error).Infof("Found bad hash, rewinding chain to block #%d [%x…]", header.Number, header.ParentHash[:4]) + glog.V(logger.Error).Infof("Found bad hash, rewinding chain to block #%d [%s]", header.Number, header.ParentHash.Hex()) bc.SetHead(header.Number.Uint64() - 1) glog.V(logger.Error).Infoln("Chain rewind was successful, resuming normal operation") } @@ -612,7 +612,7 @@ func (self *BlockChain) LoadLastState(dryrun bool) error { glog.V(logger.Info).Infof("Validating currentFastBlock: %v", self.currentFastBlock.Number()) if e := self.blockIsInvalid(self.currentFastBlock); e != nil { if !dryrun { - glog.V(logger.Warn).Infof("WARNING: Found unhealthy head fast block #%d (%x): %v \nAttempting chain reset with recovery.", self.currentFastBlock.Number(), self.currentFastBlock.Hash(), e) + glog.V(logger.Warn).Infof("WARNING: Found unhealthy head fast block #%d [%x]: %v \nAttempting chain reset with recovery.", self.currentFastBlock.Number(), self.currentFastBlock.Hash(), e) return recoverOrReset() } return fmt.Errorf("invalid currentFastBlock: %v", e) @@ -1082,7 +1082,6 @@ type WriteStatus byte const ( NonStatTy WriteStatus = iota CanonStatTy - SplitStatTy SideStatTy ) @@ -1272,6 +1271,11 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err case SideStatTy: mlogWriteStatus = "SIDE" } + parent := self.GetBlock(block.ParentHash()) + parentTimeDiff := new(big.Int) + if parent != nil { + parentTimeDiff = new(big.Int).Sub(block.Time(), parent.Time()) + } mlogBlockchain.Send(mlogBlockchainWriteBlock.SetDetailValues( mlogWriteStatus, err, @@ -1282,6 +1286,10 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err block.GasUsed(), block.Coinbase().Hex(), block.Time(), + block.Difficulty(), + len(block.Uncles()), + block.ReceivedAt, + parentTimeDiff, )) }() } @@ -1302,9 +1310,21 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err externTd := new(big.Int).Add(block.Difficulty(), ptd) // If the total difficulty is higher than our known, add it to the canonical chain - // Second clause in the if statement reduces the vulnerability to selfish mining. - // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf - if externTd.Cmp(localTd) > 0 || (externTd.Cmp(localTd) == 0 && mrand.Float64() < 0.5) { + // Compare local vs external difficulties + tdCompare := externTd.Cmp(localTd) + + // Initialize reorg if incoming td is greater than local. + reorg := tdCompare > 0 + + // If difficulties are the same, check block numbers. + if tdCompare == 0 { + nCompare := block.Number().Cmp(self.currentBlock.Number()) + // Prefer earlier block number or randomize. + // Second clause in the statement reduces the vulnerability to selfish mining. + // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf + reorg = nCompare < 0 || (nCompare == 0 && mrand.Float64() < 0.5) + } + if reorg { // Reorganise the chain if the parent is not the head block if block.ParentHash() != self.currentBlock.Hash() { if err := self.reorg(self.currentBlock, block); err != nil { @@ -1468,7 +1488,7 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (chainIndex int, err err switch status { case CanonStatTy: if glog.V(logger.Debug) { - glog.Infof("[%v] inserted block #%d (%d TXs %v G %d UNCs) (%x...). Took %v\n", time.Now().UnixNano(), block.Number(), len(block.Transactions()), block.GasUsed(), len(block.Uncles()), block.Hash().Bytes()[0:4], time.Since(bstart)) + glog.Infof("[%v] inserted block #%d (%d TXs %v G %d UNCs) [%s]. Took %v\n", time.Now().UnixNano(), block.Number(), len(block.Transactions()), block.GasUsed(), len(block.Uncles()), block.Hash().Hex(), time.Since(bstart)) } events = append(events, ChainEvent{block, block.Hash(), logs}) @@ -1486,12 +1506,9 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (chainIndex int, err err } case SideStatTy: if glog.V(logger.Detail) { - glog.Infof("inserted forked block #%d (TD=%v) (%d TXs %d UNCs) (%x...). Took %v\n", block.Number(), block.Difficulty(), len(block.Transactions()), len(block.Uncles()), block.Hash().Bytes()[0:4], time.Since(bstart)) + glog.Infof("inserted forked block #%d (TD=%v) (%d TXs %d UNCs) [%s]. Took %v\n", block.Number(), block.Difficulty(), len(block.Transactions()), len(block.Uncles()), block.Hash().Hex(), time.Since(bstart)) } events = append(events, ChainSideEvent{block, logs}) - - case SplitStatTy: - events = append(events, ChainSplitEvent{block, logs}) } stats.processed++ } @@ -1511,15 +1528,15 @@ func (self *BlockChain) InsertChain(chain types.Blocks) (chainIndex int, err err tend, )) } - glog.V(logger.Info).Infof("imported %d block(s) (%d queued %d ignored) including %d txs in %v. #%v [%x / %x]\n", + glog.V(logger.Info).Infof("imported %d block(s) (%d queued %d ignored) including %d txs in %v. #%v [%s / %s]\n", stats.processed, stats.queued, stats.ignored, txcount, tend, end.Number(), - start.Hash().Bytes()[:4], - end.Hash().Bytes()[:4]) + start.Hash().Hex(), + end.Hash().Hex()) } go self.postChainEvents(events, coalescedLogs) @@ -1596,9 +1613,17 @@ func (self *BlockChain) reorg(oldBlock, newBlock *types.Block) error { } } + commonHash := commonBlock.Hash() if glog.V(logger.Debug) { - commonHash := commonBlock.Hash() - glog.Infof("Chain split detected @ %x. Reorganising chain from #%v %x to %x", commonHash[:4], numSplit, oldStart.Hash().Bytes()[:4], newStart.Hash().Bytes()[:4]) + glog.Infof("Chain split detected @ [%s]. Reorganising chain from #%v %s to %s", commonHash.Hex(), numSplit, oldStart.Hash().Hex(), newStart.Hash().Hex()) + } + if logger.MlogEnabled() { + mlogBlockchain.Send(mlogBlockchainReorgBlocks.SetDetailValues( + commonHash.Hex(), + numSplit, + oldStart.Hash().Hex(), + newStart.Hash().Hex(), + )) } var addedTxs types.Transactions @@ -1689,7 +1714,7 @@ func (chain *BlockChain) update() { if len(blocks) > 0 { types.BlockBy(types.Number).Sort(blocks) if i, err := chain.InsertChain(blocks); err != nil { - log.Printf("periodic future chain update on block #%d (%x): %s", blocks[i].Number(), blocks[i].Hash(), err) + log.Printf("periodic future chain update on block #%d [%s]: %s", blocks[i].Number(), blocks[i].Hash().Hex(), err) } } } diff --git a/core/mlog.go b/core/mlog.go index 11b7e9a21..bab83a359 100644 --- a/core/mlog.go +++ b/core/mlog.go @@ -38,6 +38,10 @@ A STATUS of NONE means it was written _without_ any abnormal chain event, such a {"BLOCK", "GAS_USED", "BIGINT"}, {"BLOCK", "COINBASE", "STRING"}, {"BLOCK", "TIME", "BIGINT"}, + {"BLOCK", "DIFFICULTY", "BIGINT"}, + {"BLOCK", "UNCLES", "INT"}, + {"BLOCK", "RECEIVED_AT", "BIGINT"}, + {"BLOCK", "DIFF_PARENT_TIME", "BIGINT"}, }, } @@ -58,6 +62,19 @@ var mlogBlockchainInsertBlocks = logger.MLogT{ }, } +var mlogBlockchainReorgBlocks = logger.MLogT{ + Description: "Called when a chain split is detected and a subset of blocks are reoganized.", + Receiver: "BLOCKCHAIN", + Verb: "REORG", + Subject: "BLOCKS", + Details: []logger.MLogDetailT{ + {"REORG", "LAST_COMMON_HASH", "STRING"}, + {"REORG", "SPLIT_NUMBER", "BIGINT"}, + {"BLOCKS", "OLD_START_HASH", "STRING"}, + {"BLOCKS", "NEW_START_HASH", "STRING"}, + }, +} + var mlogTxPoolAddTx = logger.MLogT{ Description: `Called once when a valid transaction is added to tx pool. $TO.NAME will be the account address hex or '[NEW_CONTRACT]' in case of a contract.`, diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 1944ab195..2106798ad 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -266,8 +266,20 @@ func (d *Downloader) RegisterPeer(id string, version int, currentHead currentHea getRelHeaders relativeHeaderFetcherFn, getAbsHeaders absoluteHeaderFetcherFn, getBlockBodies blockBodyFetcherFn, getReceipts receiptFetcherFn, getNodeData stateFetcherFn) error { + var err error + defer func() { + if logger.MlogEnabled() { + mlogDownloader.Send(mlogDownloaderRegisterPeer.SetDetailValues( + id, + version, + err, + )) + } + }() + glog.V(logger.Detail).Infoln("Registering peer", id) - if err := d.peers.Register(newPeer(id, version, currentHead, getRelHeaders, getAbsHeaders, getBlockBodies, getReceipts, getNodeData)); err != nil { + err = d.peers.Register(newPeer(id, version, currentHead, getRelHeaders, getAbsHeaders, getBlockBodies, getReceipts, getNodeData)) + if err != nil { glog.V(logger.Error).Infoln("Register failed:", err) return err } @@ -280,9 +292,21 @@ func (d *Downloader) RegisterPeer(id string, version int, currentHead currentHea // the specified peer. An effort is also made to return any pending fetches into // the queue. func (d *Downloader) UnregisterPeer(id string) error { + + var err error + defer func() { + if logger.MlogEnabled() { + mlogDownloader.Send(mlogDownloaderUnregisterPeer.SetDetailValues( + id, + err, + )) + } + }() + // Unregister the peer from the active peer set and revoke any fetch tasks glog.V(logger.Detail).Infoln("Unregistering peer", id) - if err := d.peers.Unregister(id); err != nil { + err = d.peers.Unregister(id) + if err != nil { glog.V(logger.Error).Infoln("Unregister failed:", err) return err } @@ -685,10 +709,10 @@ func (d *Downloader) findAncestor(p *peer, height uint64) (uint64, error) { // If the head fetch already found an ancestor, return if !common.EmptyHash(hash) { if int64(number) <= floor { - glog.V(logger.Warn).Infof("%v: potential rewrite attack: #%d [%x…] <= #%d limit", p, number, hash[:4], floor) + glog.V(logger.Warn).Infof("%v: potential rewrite attack: #%d [%x…] <= #%d limit", p, number, hash, floor) return 0, errInvalidAncestor } - glog.V(logger.Debug).Infof("%v: common ancestor: #%d [%x…]", p, number, hash[:4]) + glog.V(logger.Debug).Infof("%v: common ancestor: #%d [%x…]", p, number, hash) return number, nil } // Ancestor not found, we need to binary search over our chain @@ -1479,7 +1503,16 @@ func (d *Downloader) qosTuner() { atomic.StoreUint64(&d.rttConfidence, conf) // Log the new QoS values and sleep until the next RTT - glog.V(logger.Debug).Infof("Quality of service: rtt %v, conf %.3f, ttl %v", rtt, float64(conf)/1000000.0, d.requestTTL()) + ttl := d.requestTTL() + if logger.MlogEnabled() { + mlogDownloader.Send(mlogDownloaderTuneQOS.SetDetailValues( + rtt, + float64(conf)/1000000.0, + ttl, + )) + } + glog.V(logger.Debug).Infof("Quality of service: rtt %v, conf %.3f, ttl %v", rtt, float64(conf)/1000000.0, ttl) + select { case <-d.quitCh: return diff --git a/eth/downloader/mlog.go b/eth/downloader/mlog.go new file mode 100644 index 000000000..4455815c8 --- /dev/null +++ b/eth/downloader/mlog.go @@ -0,0 +1,49 @@ +package downloader + +import "github.com/ethereumproject/go-ethereum/logger" + +var mlogDownloader = logger.MLogRegisterAvailable("downloader", mLogLines) + +var mLogLines = []logger.MLogT{ + mlogDownloaderRegisterPeer, + mlogDownloaderUnregisterPeer, + mlogDownloaderTuneQOS, +} + +var mlogDownloaderRegisterPeer = logger.MLogT{ + Description: "Called when the downloader registers a peer.", + Receiver: "DOWNLOADER", + Verb: "REGISTER", + Subject: "PEER", + Details: []logger.MLogDetailT{ + {"PEER", "ID", "STRING"}, + {"PEER", "VERSION", "INT"}, + {"REGISTER", "ERROR", "STRING_OR_NULL"}, + }, +} + +var mlogDownloaderUnregisterPeer = logger.MLogT{ + Description: "Called when the downloader unregisters a peer.", + Receiver: "DOWNLOADER", + Verb: "UNREGISTER", + Subject: "PEER", + Details: []logger.MLogDetailT{ + {"PEER", "ID", "STRING"}, + {"UNREGISTER", "ERROR", "STRING_OR_NULL"}, + }, +} + +var mlogDownloaderTuneQOS = logger.MLogT{ + Description: `Called at intervals to gather peer latency statistics. Estimates request round trip time. + +RTT reports the estimated Request Round Trip time, confidence is measures from 0-1 (1 is ultimately confidenct), +and TTL reports the Timeout Allowance for a single downloader request to finish within.`, + Receiver: "DOWNLOADER", + Verb: "TUNE", + Subject: "QOS", + Details: []logger.MLogDetailT{ + {"QOS", "RTT", "DURATION"}, + {"QOS", "CONFIDENCE", "NUMBER"}, + {"QOS", "TTL", "DURATION"}, + }, +} \ No newline at end of file diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index 78bd3cb1b..f6e031486 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -134,7 +134,7 @@ type Fetcher struct { getBlock blockRetrievalFn // Retrieves a block from the local chain validateBlock blockValidatorFn // Checks if a block's headers have a valid proof of work broadcastBlock blockBroadcasterFn // Broadcasts a block to connected peers - chainHeight chainHeightFn // Retrieves the current chain's height + chainHeight chainHeightFn // Retrieves the current local chain's height (blockchain.currentBlock.Number) insertChain chainInsertFn // Injects a batch of blocks into the chain dropPeer peerDropFn // Drops a peer for misbehaving @@ -205,7 +205,7 @@ func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time } } -// Enqueue tries to fill gaps the the fetcher's future import queue. +// Enqueue tries to fill gaps in the fetcher's future import queue. func (f *Fetcher) Enqueue(peer string, block *types.Block) error { op := &inject{ origin: peer, @@ -334,7 +334,15 @@ func (f *Fetcher) loop() { // If we have a valid block number, check that it's potentially useful if notification.number > 0 { if dist := int64(notification.number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { - glog.V(logger.Debug).Infof("[eth/62] Peer %s: discarded announcement #%d [%x…], distance %d", notification.origin, notification.number, notification.hash[:4], dist) + if logger.MlogEnabled() { + mlogFetcher.Send(mlogFetcherDiscardAnnouncement.SetDetailValues( + notification.origin, + notification.number, + notification.hash.Str(), + dist, + )) + } + glog.V(logger.Debug).Infof("[eth/62] Peer %s: discarded announcement #%d [%s], distance %d", notification.origin, notification.number, notification.hash.Hex(), dist) metrics.FetchAnnounceDrops.Mark(1) break } @@ -466,7 +474,7 @@ func (f *Fetcher) loop() { if announce := f.fetching[hash]; announce != nil && f.fetched[hash] == nil && f.completing[hash] == nil && f.queued[hash] == nil { // If the delivered header does not match the promised number, drop the announcer if header.Number.Uint64() != announce.number { - glog.V(logger.Detail).Infof("[eth/62] Peer %s: invalid block number for [%x…]: announced %d, provided %d", announce.origin, header.Hash().Bytes()[:4], announce.number, header.Number.Uint64()) + glog.V(logger.Detail).Infof("[eth/62] Peer %s: invalid block number for [%s]: announced %d, provided %d", announce.origin, header.Hash().Hex(), announce.number, header.Number.Uint64()) f.dropPeer(announce.origin) f.forgetHash(hash) continue @@ -478,7 +486,7 @@ func (f *Fetcher) loop() { // If the block is empty (header only), short circuit into the final import queue if header.TxHash == types.DeriveSha(types.Transactions{}) && header.UncleHash == types.CalcUncleHash([]*types.Header{}) { - glog.V(logger.Detail).Infof("[eth/62] Peer %s: block #%d [%x…] empty, skipping body retrieval", announce.origin, header.Number.Uint64(), header.Hash().Bytes()[:4]) + glog.V(logger.Detail).Infof("[eth/62] Peer %s: block #%d [%s] empty, skipping body retrieval", announce.origin, header.Number.Uint64(), header.Hash().Hex()) block := types.NewBlockWithHeader(header) block.ReceivedAt = task.time @@ -490,7 +498,7 @@ func (f *Fetcher) loop() { // Otherwise add to the list of blocks needing completion incomplete = append(incomplete, announce) } else { - glog.V(logger.Detail).Infof("[eth/62] Peer %s: block #%d [%x…] already imported, discarding header", announce.origin, header.Number.Uint64(), header.Hash().Bytes()[:4]) + glog.V(logger.Detail).Infof("[eth/62] Peer %s: block #%d [%s] already imported, discarding header", announce.origin, header.Number.Uint64(), header.Hash().Hex()) f.forgetHash(hash) } } else { @@ -621,14 +629,50 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) { // Ensure the peer isn't DOSing us count := f.queues[peer] + 1 if count > blockLimit { - glog.V(logger.Debug).Infof("Peer %s: discarded block #%d [%x…], exceeded allowance (%d)", peer, block.NumberU64(), hash.Bytes()[:4], blockLimit) + glog.V(logger.Debug).Infof("Peer %s: discarded block #%d [%s], exceeded allowance (%d)", peer, block.NumberU64(), hash.Hex(), blockLimit) metrics.FetchBroadcastDOS.Mark(1) f.forgetHash(hash) return } // Discard any past or too distant blocks + // eg1; AHEAD (next blocks) + // block.Number = 16 + // blockchain.Height = 15 + // 16 - 15 = 1; 1 < -7 (FALSE) || 1 > 32 (FALSE) + // eg2 + // block.Number = 55 + // blockchain.Height = 15 + // 55 - 15 = 40; 40 < -7 (FALSE) || 40 > 32 (TRUE) --> discard, drop, and forget + + // eg1; SAME (same head height) + // block.Number = 15 + // blockchain.Height = 15 + // 15 - 15 = 0; 0 < -7 (FALSE) || 0 > 32 (FALSE) + + // eg1; BEHIND (past blocks) + // block.Number = 14 + // blockchain.Height = 15 + // 14 - 15 = -1; -1 < -7 (FALSE) || -1 > 32 (FALSE) + // eg2 + // block.Number = 5 + // blockchain.Height = 15 + // 5 - 15 = -10; -10 < -6 (TRUE) || -10 > 32 (FALSE) if dist := int64(block.NumberU64()) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { - glog.V(logger.Debug).Infof("Peer %s: discarded block #%d [%x…], distance %d", peer, block.NumberU64(), hash.Bytes()[:4], dist) + if logger.MlogEnabled() { + mlogFetcher.Send(mlogFetcherDiscardAnnouncement.SetDetailValues( + peer, + block.NumberU64(), + hash.Str(), + dist, + )) + } + glog.V(logger.Debug).Infof("Peer %s: discarded block #%d [%s], distance %d", peer, block.NumberU64(), hash.Hex(), dist) + metrics.FetchBroadcastDrops.Mark(1) + f.forgetHash(hash) + return + } + if f.getBlock(hash) != nil { + glog.V(logger.Debug).Infof("Peer %s: discarded block #%d [%s], already have", peer, block.NumberU64(), hash.Hex()) metrics.FetchBroadcastDrops.Mark(1) f.forgetHash(hash) return @@ -646,26 +690,35 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) { f.queueChangeHook(op.block.Hash(), true) } if glog.V(logger.Debug) { - glog.Infof("Peer %s: queued block #%d [%x…], total %v", peer, block.NumberU64(), hash.Bytes()[:4], f.queue.Size()) + glog.Infof("Peer %s: queued block #%d [%s], total %v", peer, block.NumberU64(), hash.Hex(), f.queue.Size()) } } } // insert spawns a new goroutine to run a block insertion into the chain. If the -// block's number is at the same height as the current import phase, if updates +// block's number is at the same height as the current import phase, it updates // the phase states accordingly. func (f *Fetcher) insert(peer string, block *types.Block) { hash := block.Hash() // Run the import on a new thread - glog.V(logger.Debug).Infof("Peer %s: importing block #%d [%x…]", peer, block.NumberU64(), hash[:4]) + glog.V(logger.Debug).Infof("Peer %s: importing block #%d [%s]", peer, block.NumberU64(), hash.Hex()) go func() { - defer func() { f.done <- hash }() - // If the parent's unknown, abort insertion + haveParent := true + + defer func() { + if haveParent { + f.done <- hash + } + }() + + // If the parent's unknown, abort insertion, and don't forget the hash and block; + // use queue gap fill to get unknown parent. parent := f.getBlock(block.ParentHash()) if parent == nil { - glog.V(logger.Debug).Infof("Peer %s: parent []%x] of block #%d [%x…] unknown", block.ParentHash().Bytes()[:4], peer, block.NumberU64(), hash[:4]) + glog.V(logger.Debug).Infof("Peer %s: parent [%s] of block #%d [%s] unknown", peer, block.ParentHash().Hex(), block.NumberU64(), hash.Hex()) + haveParent = false return } // Quickly validate the header and propagate the block if it passes @@ -680,13 +733,13 @@ func (f *Fetcher) insert(peer string, block *types.Block) { default: // Something went very wrong, drop the peer - glog.V(logger.Debug).Infof("Peer %s: block #%d [%x…] verification failed: %v", peer, block.NumberU64(), hash[:4], err) + glog.V(logger.Debug).Infof("Peer %s: block #%d [%s] verification failed: %v", peer, block.NumberU64(), hash.Hex(), err) f.dropPeer(peer) return } // Run the actual import and log any issues if _, err := f.insertChain(types.Blocks{block}); err != nil { - glog.V(logger.Warn).Infof("Peer %s: block #%d [%x…] import failed: %v", peer, block.NumberU64(), hash[:4], err) + glog.V(logger.Warn).Infof("Peer %s: block #%d [%s] import failed: %v", peer, block.NumberU64(), hash.Hex(), err) return } // If import succeeded, broadcast the block diff --git a/eth/fetcher/mlog.go b/eth/fetcher/mlog.go new file mode 100644 index 000000000..a3e72fd22 --- /dev/null +++ b/eth/fetcher/mlog.go @@ -0,0 +1,22 @@ +package fetcher + +import "github.com/ethereumproject/go-ethereum/logger" + +var mlogFetcher = logger.MLogRegisterAvailable("fetcher", mLogLines) + +var mLogLines = []logger.MLogT{ + mlogFetcherDiscardAnnouncement, +} + +var mlogFetcherDiscardAnnouncement = logger.MLogT{ + Description: "Called when a block announcement is discarded.", + Receiver: "FETCHER", + Verb: "DISCARD", + Subject: "ANNOUNCEMENT", + Details: []logger.MLogDetailT{ + {"ANNOUNCEMENT", "ORIGIN", "STRING"}, + {"ANNOUNCEMENT", "NUMBER", "INT"}, + {"ANNOUNCEMENT", "HASH", "STRING"}, + {"ANNOUNCEMENT", "DISTANCE", "INT"}, + }, +} diff --git a/eth/handler.go b/eth/handler.go index 0a8f32e88..10fce788a 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -632,26 +632,31 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Mark the peer as owning the block and schedule it for import p.MarkBlock(request.Block.Hash()) - p.SetHead(request.Block.ParentHash(), request.TD) pm.fetcher.Enqueue(p.id, request.Block) // Assuming the block is importable by the peer, but possibly not yet done so, // calculate the head hash and TD that the peer truly must have. var ( trueHead = request.Block.ParentHash() - trueTD = request.TD + trueTD = new(big.Int).Sub(request.TD, request.Block.Difficulty()) ) // Update the peers total difficulty if better than the previous if _, td := p.Head(); trueTD.Cmp(td) > 0 { + glog.V(logger.Debug).Infof("Peer %s: setting head: tdWas=%v trueTD=%v", p.id, td, trueTD) p.SetHead(trueHead, trueTD) - + // Schedule a sync if above ours. Note, this will not fire a sync for a gap of // a singe block (as the true TD is below the propagated block), however this // scenario should easily be covered by the fetcher. currentBlock := pm.blockchain.CurrentBlock() - if trueTD.Cmp(pm.blockchain.GetTd(currentBlock.Hash())) > 0 { + if localTd := pm.blockchain.GetTd(currentBlock.Hash()); trueTD.Cmp(localTd) > 0 { + glog.V(logger.Debug).Infof("Peer %s: localTD=%v (<) peerTrueTD=%v, synchronising", p.id, localTd, trueTD) go pm.synchronise(p) + } else { + glog.V(logger.Detail).Infof("Peer %s: localTD=%v (>=) peerTrueTD=%v, NOT synchronising", p.id, localTd, trueTD) } + } else { + glog.V(logger.Detail).Infof("Peer %s: NOT setting head: tdWas=%v trueTD=%v", p.id, td, trueTD) } case msg.Code == TxMsg: @@ -720,7 +725,7 @@ func (pm *ProtocolManager) BroadcastTx(hash common.Hash, tx *types.Transaction) for _, peer := range peers { peer.SendTransactions(types.Transactions{tx}) } - glog.V(logger.Detail).Infoln("broadcast tx to", len(peers), "peers") + glog.V(logger.Detail).Infof("broadcast tx [%s] to %d peers", tx.Hash().Hex(), len(peers)) } // Mined broadcast loop From 9cd9a6f30c66a55d89557be1cd01cc7e949fa9d5 Mon Sep 17 00:00:00 2001 From: ia Date: Thu, 23 Nov 2017 10:54:12 +0700 Subject: [PATCH 04/18] problem: blockchain insert should check blocknumber in addition to TD re: reorg solution: modify reorg conditional in blockchain insertchain to use smaller block number or randomized block reorg based on http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf rel #346 rel #398 possibly #378 --- core/blockchain.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/core/blockchain.go b/core/blockchain.go index 059944f65..ab49469fe 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1270,6 +1270,8 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err mlogWriteStatus = "CANON" case SideStatTy: mlogWriteStatus = "SIDE" + case SplitStatTy: + mlogWriteStatus = "SPLIT" } parent := self.GetBlock(block.ParentHash()) parentTimeDiff := new(big.Int) From 57e502c78212942f9a5ff85954a492e046f24e58 Mon Sep 17 00:00:00 2001 From: ia Date: Fri, 24 Nov 2017 17:54:46 +0700 Subject: [PATCH 05/18] (nonfunctional) remove dead commented code notes --- eth/fetcher/fetcher.go | 23 ----------------------- 1 file changed, 23 deletions(-) diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index f6e031486..91f3bbd84 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -634,29 +634,6 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) { f.forgetHash(hash) return } - // Discard any past or too distant blocks - // eg1; AHEAD (next blocks) - // block.Number = 16 - // blockchain.Height = 15 - // 16 - 15 = 1; 1 < -7 (FALSE) || 1 > 32 (FALSE) - // eg2 - // block.Number = 55 - // blockchain.Height = 15 - // 55 - 15 = 40; 40 < -7 (FALSE) || 40 > 32 (TRUE) --> discard, drop, and forget - - // eg1; SAME (same head height) - // block.Number = 15 - // blockchain.Height = 15 - // 15 - 15 = 0; 0 < -7 (FALSE) || 0 > 32 (FALSE) - - // eg1; BEHIND (past blocks) - // block.Number = 14 - // blockchain.Height = 15 - // 14 - 15 = -1; -1 < -7 (FALSE) || -1 > 32 (FALSE) - // eg2 - // block.Number = 5 - // blockchain.Height = 15 - // 5 - 15 = -10; -10 < -6 (TRUE) || -10 > 32 (FALSE) if dist := int64(block.NumberU64()) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { if logger.MlogEnabled() { mlogFetcher.Send(mlogFetcherDiscardAnnouncement.SetDetailValues( From 22a3b01955299b9e54a41815f0f97c3c86e8659e Mon Sep 17 00:00:00 2001 From: ia Date: Fri, 24 Nov 2017 18:56:31 +0700 Subject: [PATCH 06/18] problem: git merge is almost perfect but it missed a removed line solution: remove removed SplitStatTy for core/blockchain --- core/blockchain.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index ab49469fe..059944f65 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1270,8 +1270,6 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err mlogWriteStatus = "CANON" case SideStatTy: mlogWriteStatus = "SIDE" - case SplitStatTy: - mlogWriteStatus = "SPLIT" } parent := self.GetBlock(block.ParentHash()) parentTimeDiff := new(big.Int) From b8dbfd9d8f5f226bdb4f9f8f5827c72185bdeca1 Mon Sep 17 00:00:00 2001 From: ia Date: Sun, 26 Nov 2017 08:33:35 +0700 Subject: [PATCH 07/18] problem: logging mismatch genesis block too verbose solution: abbreviate our own genesis block hash log --- eth/peer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/peer.go b/eth/peer.go index e26c05037..83b6b7e84 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -281,7 +281,7 @@ func (p *peer) readStatus(network int, status *statusData, genesis common.Hash) return errResp(ErrDecode, "msg %v: %v", msg, err) } if status.GenesisBlock != genesis { - return errResp(ErrGenesisBlockMismatch, "%x (!= %x)", status.GenesisBlock, genesis) + return errResp(ErrGenesisBlockMismatch, "%x (!= %x…)", status.GenesisBlock, genesis.Bytes()[:8]) } if int(status.NetworkId) != network { return errResp(ErrNetworkIdMismatch, "%d (!= %d)", status.NetworkId, network) From f4154a1ae0961c1285e3a2a1fbb91f46a060f670 Mon Sep 17 00:00:00 2001 From: ia Date: Sun, 26 Nov 2017 09:52:50 +0700 Subject: [PATCH 08/18] fix: should show 'Import' when at or above last known chain height --- cmd/geth/cmd.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/cmd.go b/cmd/geth/cmd.go index 8056f0d18..613ec7f04 100644 --- a/cmd/geth/cmd.go +++ b/cmd/geth/cmd.go @@ -864,7 +864,7 @@ func runStatusSyncLogs(e *eth.Ethereum, interval string, maxPeers int) { currentBlockHex = blockchain.CurrentFastBlock().Hash().Hex() } } - if current == height && !(current == 0 && height == 0) { + if current >= height && !(current == 0 && height == 0) { fMode = "Import " // with spaces to make same length as Discover, FastSync, FullSync fOfHeight = strings.Repeat(" ", 12) fHeightRatio = strings.Repeat(" ", 7) From a7381ce457091c5f628a9e4cd9a3b814fef8e56f Mon Sep 17 00:00:00 2001 From: ia Date: Sun, 26 Nov 2017 12:32:20 +0700 Subject: [PATCH 09/18] sketch fuck with initsync vs catchupsync, drop ttl scaling to 1:rtt --- eth/downloader/downloader.go | 9 ++++++--- eth/downloader/types.go | 3 +++ eth/handler.go | 8 +++++++- 3 files changed, 16 insertions(+), 4 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 2106798ad..acaec2b7e 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -54,7 +54,7 @@ const ( var ( rttMinEstimate = 2 * time.Second // Minimum round-trip time to target for download requests - rttMaxEstimate = 20 * time.Second // Maximum rount-trip time to target for download requests + rttMaxEstimate = 20 * time.Second // Maximum round-trip time to target for download requests rttMinConfidence = 0.1 // Worse confidence factor in our estimated RTT value ttlScaling = 3 // Constant scaling factor for RTT -> TTL conversion ttlLimit = time.Minute // Maximum TTL allowance to prevent reaching crazy timeouts @@ -72,7 +72,7 @@ var ( fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it fsPivotInterval = 512 // Number of headers out of which to randomize the pivot point fsMinFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync - fsCriticalTrials = 10 // Number of times to retry in the cricical section before bailing + fsCriticalTrials = 10 // Number of times to retry in the critical section before bailing ) var ( @@ -144,6 +144,7 @@ type Downloader struct { insertReceipts receiptChainInsertFn // Injects a batch of blocks and their receipts into the chain rollback chainRollbackFn // Removes a batch of recently added chain links dropPeer peerDropFn // Drops a peer for misbehaving + pmSynced pmSyncedFn // Checks if pm has completed initial sync // Status synchroniseMock func(id string, hash common.Hash) error // Replacement for synchronise during testing @@ -180,7 +181,7 @@ type Downloader struct { func New(stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, hasBlockAndState blockAndStateCheckFn, getHeader headerRetrievalFn, getBlock blockRetrievalFn, headHeader headHeaderRetrievalFn, headBlock headBlockRetrievalFn, headFastBlock headFastBlockRetrievalFn, commitHeadBlock headBlockCommitterFn, getTd tdRetrievalFn, insertHeaders headerChainInsertFn, - insertBlocks blockChainInsertFn, insertReceipts receiptChainInsertFn, rollback chainRollbackFn, dropPeer peerDropFn) *Downloader { + insertBlocks blockChainInsertFn, insertReceipts receiptChainInsertFn, rollback chainRollbackFn, dropPeer peerDropFn, pmIsSynced pmSyncedFn) *Downloader { dl := &Downloader{ mode: FullSync, @@ -203,6 +204,7 @@ func New(stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, ha insertReceipts: insertReceipts, rollback: rollback, dropPeer: dropPeer, + pmSynced: pmIsSynced, newPeerCh: make(chan *peer, 1), headerCh: make(chan dataPack, 1), bodyCh: make(chan dataPack, 1), @@ -330,6 +332,7 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode switch err { case nil: log.Printf("peer %q sync complete", id) + ttlScaling = 1 // Modify timeout scaling, shrinking it because now we can be less patient return true case errBusy: diff --git a/eth/downloader/types.go b/eth/downloader/types.go index f318f106a..9d1cb7e1e 100644 --- a/eth/downloader/types.go +++ b/eth/downloader/types.go @@ -66,6 +66,9 @@ type chainRollbackFn func([]common.Hash) // peerDropFn is a callback type for dropping a peer detected as malicious. type peerDropFn func(id string) +// pmIsSynced checks if the pm has finished initial sync +type pmSyncedFn func() bool + // dataPack is a data message returned by a peer for some query. type dataPack interface { PeerId() string diff --git a/eth/handler.go b/eth/handler.go index 10fce788a..dd1474b0f 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -154,7 +154,7 @@ func NewProtocolManager(config *core.ChainConfig, fastSync bool, networkId int, manager.downloader = downloader.New(chaindb, manager.eventMux, blockchain.HasHeader, blockchain.HasBlockAndState, blockchain.GetHeader, blockchain.GetBlock, blockchain.CurrentHeader, blockchain.CurrentBlock, blockchain.CurrentFastBlock, blockchain.FastSyncCommitHead, blockchain.GetTd, blockchain.InsertHeaderChain, manager.insertChain, blockchain.InsertReceiptChain, blockchain.Rollback, - manager.removePeer) + manager.removePeer, manager.isSynced) validator := func(block *types.Block, parent *types.Block) error { return core.ValidateHeader(config, pow, block.Header(), parent.Header(), true, false) @@ -183,6 +183,10 @@ func (pm *ProtocolManager) insertChain(blocks types.Blocks) (i int, err error) { return i, err } +func (pm *ProtocolManager) isSynced() bool { + return pm.synced > 0 +} + func (pm *ProtocolManager) removePeer(id string) { // Short circuit if the peer was already removed peer := pm.peers.Peer(id) @@ -651,6 +655,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { currentBlock := pm.blockchain.CurrentBlock() if localTd := pm.blockchain.GetTd(currentBlock.Hash()); trueTD.Cmp(localTd) > 0 { glog.V(logger.Debug).Infof("Peer %s: localTD=%v (<) peerTrueTD=%v, synchronising", p.id, localTd, trueTD) + + // TODO: async sync go pm.synchronise(p) } else { glog.V(logger.Detail).Infof("Peer %s: localTD=%v (>=) peerTrueTD=%v, NOT synchronising", p.id, localTd, trueTD) From b51cb3cf053399bb3d095c318612ab180634c4c8 Mon Sep 17 00:00:00 2001 From: ia Date: Mon, 27 Nov 2017 15:09:03 +0700 Subject: [PATCH 10/18] fix: update protocol test against err msg for genesis mismatch --- eth/protocol_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/protocol_test.go b/eth/protocol_test.go index 22436c514..9419e6207 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -65,7 +65,7 @@ func testStatusMsgErrors(t *testing.T, protocol int) { }, { code: StatusMsg, data: statusData{uint32(protocol), NetworkId, td, currentBlock, common.Hash{3}}, - wantError: errResp(ErrGenesisBlockMismatch, "0300000000000000000000000000000000000000000000000000000000000000 (!= %x)", genesis), + wantError: errResp(ErrGenesisBlockMismatch, "0300000000000000000000000000000000000000000000000000000000000000 (!= %x…)", genesis.Bytes()[:8]), }, } From cc2d21270c1f7cba7934e13c65072fac50efb157 Mon Sep 17 00:00:00 2001 From: ia Date: Mon, 27 Nov 2017 15:53:55 +0700 Subject: [PATCH 11/18] fix: leftover sketches in dl.New signature from git tomfoolery --- eth/downloader/downloader.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index acaec2b7e..e44679dd1 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -144,7 +144,6 @@ type Downloader struct { insertReceipts receiptChainInsertFn // Injects a batch of blocks and their receipts into the chain rollback chainRollbackFn // Removes a batch of recently added chain links dropPeer peerDropFn // Drops a peer for misbehaving - pmSynced pmSyncedFn // Checks if pm has completed initial sync // Status synchroniseMock func(id string, hash common.Hash) error // Replacement for synchronise during testing @@ -181,7 +180,7 @@ type Downloader struct { func New(stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, hasBlockAndState blockAndStateCheckFn, getHeader headerRetrievalFn, getBlock blockRetrievalFn, headHeader headHeaderRetrievalFn, headBlock headBlockRetrievalFn, headFastBlock headFastBlockRetrievalFn, commitHeadBlock headBlockCommitterFn, getTd tdRetrievalFn, insertHeaders headerChainInsertFn, - insertBlocks blockChainInsertFn, insertReceipts receiptChainInsertFn, rollback chainRollbackFn, dropPeer peerDropFn, pmIsSynced pmSyncedFn) *Downloader { + insertBlocks blockChainInsertFn, insertReceipts receiptChainInsertFn, rollback chainRollbackFn, dropPeer peerDropFn) *Downloader { dl := &Downloader{ mode: FullSync, @@ -204,7 +203,6 @@ func New(stateDb ethdb.Database, mux *event.TypeMux, hasHeader headerCheckFn, ha insertReceipts: insertReceipts, rollback: rollback, dropPeer: dropPeer, - pmSynced: pmIsSynced, newPeerCh: make(chan *peer, 1), headerCh: make(chan dataPack, 1), bodyCh: make(chan dataPack, 1), From 6ee3833d9acabb1dec4f9a08285483c0c75d8855 Mon Sep 17 00:00:00 2001 From: ia Date: Mon, 27 Nov 2017 15:55:25 +0700 Subject: [PATCH 12/18] Revert "sketch fuck with initsync vs catchupsync, drop ttl scaling to 1:rtt" This reverts commit a7381ce457091c5f628a9e4cd9a3b814fef8e56f. --- eth/downloader/downloader.go | 5 ++--- eth/downloader/types.go | 3 --- eth/handler.go | 8 +------- 3 files changed, 3 insertions(+), 13 deletions(-) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index e44679dd1..2106798ad 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -54,7 +54,7 @@ const ( var ( rttMinEstimate = 2 * time.Second // Minimum round-trip time to target for download requests - rttMaxEstimate = 20 * time.Second // Maximum round-trip time to target for download requests + rttMaxEstimate = 20 * time.Second // Maximum rount-trip time to target for download requests rttMinConfidence = 0.1 // Worse confidence factor in our estimated RTT value ttlScaling = 3 // Constant scaling factor for RTT -> TTL conversion ttlLimit = time.Minute // Maximum TTL allowance to prevent reaching crazy timeouts @@ -72,7 +72,7 @@ var ( fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it fsPivotInterval = 512 // Number of headers out of which to randomize the pivot point fsMinFullBlocks = 1024 // Number of blocks to retrieve fully even in fast sync - fsCriticalTrials = 10 // Number of times to retry in the critical section before bailing + fsCriticalTrials = 10 // Number of times to retry in the cricical section before bailing ) var ( @@ -330,7 +330,6 @@ func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode switch err { case nil: log.Printf("peer %q sync complete", id) - ttlScaling = 1 // Modify timeout scaling, shrinking it because now we can be less patient return true case errBusy: diff --git a/eth/downloader/types.go b/eth/downloader/types.go index 9d1cb7e1e..f318f106a 100644 --- a/eth/downloader/types.go +++ b/eth/downloader/types.go @@ -66,9 +66,6 @@ type chainRollbackFn func([]common.Hash) // peerDropFn is a callback type for dropping a peer detected as malicious. type peerDropFn func(id string) -// pmIsSynced checks if the pm has finished initial sync -type pmSyncedFn func() bool - // dataPack is a data message returned by a peer for some query. type dataPack interface { PeerId() string diff --git a/eth/handler.go b/eth/handler.go index dd1474b0f..10fce788a 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -154,7 +154,7 @@ func NewProtocolManager(config *core.ChainConfig, fastSync bool, networkId int, manager.downloader = downloader.New(chaindb, manager.eventMux, blockchain.HasHeader, blockchain.HasBlockAndState, blockchain.GetHeader, blockchain.GetBlock, blockchain.CurrentHeader, blockchain.CurrentBlock, blockchain.CurrentFastBlock, blockchain.FastSyncCommitHead, blockchain.GetTd, blockchain.InsertHeaderChain, manager.insertChain, blockchain.InsertReceiptChain, blockchain.Rollback, - manager.removePeer, manager.isSynced) + manager.removePeer) validator := func(block *types.Block, parent *types.Block) error { return core.ValidateHeader(config, pow, block.Header(), parent.Header(), true, false) @@ -183,10 +183,6 @@ func (pm *ProtocolManager) insertChain(blocks types.Blocks) (i int, err error) { return i, err } -func (pm *ProtocolManager) isSynced() bool { - return pm.synced > 0 -} - func (pm *ProtocolManager) removePeer(id string) { // Short circuit if the peer was already removed peer := pm.peers.Peer(id) @@ -655,8 +651,6 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { currentBlock := pm.blockchain.CurrentBlock() if localTd := pm.blockchain.GetTd(currentBlock.Hash()); trueTD.Cmp(localTd) > 0 { glog.V(logger.Debug).Infof("Peer %s: localTD=%v (<) peerTrueTD=%v, synchronising", p.id, localTd, trueTD) - - // TODO: async sync go pm.synchronise(p) } else { glog.V(logger.Detail).Infof("Peer %s: localTD=%v (>=) peerTrueTD=%v, NOT synchronising", p.id, localTd, trueTD) From 9fab3db0e8648d17ff5195ff13b6b2982726c054 Mon Sep 17 00:00:00 2001 From: ia Date: Mon, 27 Nov 2017 16:38:30 +0700 Subject: [PATCH 13/18] (nonfunctional) add comment --- eth/fetcher/fetcher.go | 1 + 1 file changed, 1 insertion(+) diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/fetcher.go index 91f3bbd84..27d65c8dc 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/fetcher.go @@ -648,6 +648,7 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) { f.forgetHash(hash) return } + // Don't queue block if we already have it. if f.getBlock(hash) != nil { glog.V(logger.Debug).Infof("Peer %s: discarded block #%d [%s], already have", peer, block.NumberU64(), hash.Hex()) metrics.FetchBroadcastDrops.Mark(1) From 3aee42c9e984110149e1190eca049cff09e712cd Mon Sep 17 00:00:00 2001 From: ia Date: Tue, 28 Nov 2017 17:31:57 +0700 Subject: [PATCH 14/18] consensus|problem: should not use block number to influence chain reorg solution: use strictly TD, randonly choosing a reorg if find equal competing TDs --- core/blockchain.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 059944f65..39584715d 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1316,13 +1316,11 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err // Initialize reorg if incoming td is greater than local. reorg := tdCompare > 0 - // If difficulties are the same, check block numbers. + // If TDs are the same, randomize. if tdCompare == 0 { - nCompare := block.Number().Cmp(self.currentBlock.Number()) - // Prefer earlier block number or randomize. - // Second clause in the statement reduces the vulnerability to selfish mining. + // Reduces the vulnerability to selfish mining. // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf - reorg = nCompare < 0 || (nCompare == 0 && mrand.Float64() < 0.5) + reorg = mrand.Float64() < 0.5 } if reorg { // Reorganise the chain if the parent is not the head block From daf81a919c67560a6def77d226ecdd971b60f795 Mon Sep 17 00:00:00 2001 From: ia Date: Tue, 28 Nov 2017 21:50:11 +0700 Subject: [PATCH 15/18] nonfunctional|fix: typo in comment rebuild CI --- core/blockchain.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/blockchain.go b/core/blockchain.go index 39584715d..252d7835f 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1313,7 +1313,7 @@ func (self *BlockChain) WriteBlock(block *types.Block) (status WriteStatus, err // Compare local vs external difficulties tdCompare := externTd.Cmp(localTd) - // Initialize reorg if incoming td is greater than local. + // Initialize reorg if incoming TD is greater than local. reorg := tdCompare > 0 // If TDs are the same, randomize. From 8202bee5a40f1b098ecf0dddb7a022092f4277ba Mon Sep 17 00:00:00 2001 From: ia Date: Thu, 30 Nov 2017 10:59:46 -0600 Subject: [PATCH 16/18] problem: CI tests getting killed solution: move a log describing when found peer with higher TD syncing from Debug->Info should reset and restart Travis --- eth/handler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/handler.go b/eth/handler.go index 10fce788a..50e010d6c 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -650,7 +650,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // scenario should easily be covered by the fetcher. currentBlock := pm.blockchain.CurrentBlock() if localTd := pm.blockchain.GetTd(currentBlock.Hash()); trueTD.Cmp(localTd) > 0 { - glog.V(logger.Debug).Infof("Peer %s: localTD=%v (<) peerTrueTD=%v, synchronising", p.id, localTd, trueTD) + glog.V(logger.Info).Infof("Peer %s: localTD=%v (<) peerTrueTD=%v, synchronising", p.id, localTd, trueTD) go pm.synchronise(p) } else { glog.V(logger.Detail).Infof("Peer %s: localTD=%v (>=) peerTrueTD=%v, NOT synchronising", p.id, localTd, trueTD) From d8c814c71692dfddfed1fe258565f267da64c690 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Zdyba=C5=82?= Date: Fri, 1 Dec 2017 16:25:35 +0100 Subject: [PATCH 17/18] Switch from go1.8 to go1.8.x --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e99711258..ced7bdc14 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ language: go go_import_path: github.com/ethereumproject/go-ethereum -go: 1.8 +go: 1.8.x os: - linux - osx From 8a39ed1fc6624374d31ec0c6c5cbc0c51379381c Mon Sep 17 00:00:00 2001 From: ia Date: Sat, 2 Dec 2017 15:14:51 -0600 Subject: [PATCH 18/18] (nonfunctional) problem: restart CI, solution: fix minor typos in README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 56f1e2cda..f47f53840 100644 --- a/README.md +++ b/README.md @@ -54,8 +54,8 @@ $ go install -ldflags "-X main.Version="`git describe --tags` ./cmd/... ``` #### Using release source code tarball -Because of strict Go directory structure, tarball needs to be extracted into proper subdirectory under `$GOPATH`. -Following commands are example of building the v4.1.1 release: +Because of strict Go directory structure, the tarball needs to be extracted into proper subdirectory under `$GOPATH`. +The following commands are an example of building the v4.1.1 release: ``` $ mkdir -p $GOPATH/src/github.com/ethereumproject $ cd $GOPATH/src/github.com/ethereumproject