Merge pull request #390 from ethereumproject/ensure-valid-bc

core: db stability and recoverability

accounts/caching.go

@@ -19,11 +19,11 @@
 package accounts
 
 import (
-	"time"
-	"github.com/ethereumproject/go-ethereum/common"
 	"fmt"
+	"github.com/ethereumproject/go-ethereum/common"
 	"os"
 	"strings"
+	"time"
 )
 
 // Minimum amount of time between cache reloads. This limit applies if the platform does
@@ -75,7 +75,6 @@ type caching interface {
 	close()
 }
 
-
 func skipKeyFile(fi os.FileInfo) bool {
 	// Skip editor backups and UNIX-style hidden files.
 	if strings.HasSuffix(fi.Name(), "~") || strings.HasPrefix(fi.Name(), ".") {
@@ -89,4 +88,4 @@ func skipKeyFile(fi os.FileInfo) bool {
 		return true
 	}
 	return false
-}
+}

cmd/geth/accountcmd.go

@@ -31,7 +31,7 @@ import (
 
 var (
 	AccountsIndexFlag = cli.BoolFlag{
-		Name: "index-accounts,indexaccounts",
+		Name:  "index-accounts,indexaccounts",
 		Usage: "Enable key-value db store for indexing large amounts of key files",
 	}
 	walletCommand = cli.Command{

cmd/geth/chaincmd.go

@@ -17,14 +17,14 @@
 package main
 
 import (
+	"bufio"
 	"fmt"
 	"log"
 	"os"
 	"path/filepath"
 	"strconv"
-	"time"
 	"strings"
-	"bufio"
+	"time"
 
 	"github.com/ethereumproject/go-ethereum/common"
 	"github.com/ethereumproject/go-ethereum/console"
@@ -104,14 +104,25 @@ var (
 	}
 	resetCommand = cli.Command{
 		Action: resetChaindata,
-		Name: "reset",
-		Usage: "Reset the chain database",
+		Name:   "reset",
+		Usage:  "Reset the chain database",
 		Description: `
 		Reset does a hard reset of the entire chain database.
 		This is a drastic and irreversible command, and should be used with caution.
 		The command will require user confirmation before any action is taken.
 		`,
 	}
+	recoverCommand = cli.Command{
+		Action: recoverChaindata,
+		Name:   "recover",
+		Usage:  "Attempt blockchain data recovery in case of data corruption",
+		Description: `
+		Recover scans and health-checks all available blockchain data in order
+		to recover all consistent and healthy block data. It will remove invalid or
+		corrupt block data that may have been caused by hard killing, system failure,
+		space limitations, or attack.
+		`,
+	}
 )
 
 func importChain(ctx *cli.Context) error {
@@ -287,7 +298,7 @@ func dump(ctx *cli.Context) error {
 			if sorted {
 				err = state.SortedDump(addresses, prefix, indent, out)
 			} else {
-				err = state.UnsortedDump(addresses,prefix,indent,out)
+				err = state.UnsortedDump(addresses, prefix, indent, out)
 			}
 
 			if err != nil {

cmd/geth/cmd.go

@@ -17,34 +17,33 @@
 package main
 
 import (
-	"fmt"
-	"io"
-	"os"
-	"os/signal"
-	"runtime"
-
-	"strings"
-
+	"bufio"
 	"errors"
-	"io/ioutil"
-	"math/big"
-	"path/filepath"
-	"strconv"
-	"syscall"
-	"time"
-
+	"fmt"
 	"github.com/ethereumproject/ethash"
 	"github.com/ethereumproject/go-ethereum/core"
 	"github.com/ethereumproject/go-ethereum/core/state"
 	"github.com/ethereumproject/go-ethereum/core/types"
 	"github.com/ethereumproject/go-ethereum/eth"
 	"github.com/ethereumproject/go-ethereum/eth/downloader"
+	"github.com/ethereumproject/go-ethereum/event"
 	"github.com/ethereumproject/go-ethereum/logger"
 	"github.com/ethereumproject/go-ethereum/logger/glog"
 	"github.com/ethereumproject/go-ethereum/node"
+	"github.com/ethereumproject/go-ethereum/pow"
 	"github.com/ethereumproject/go-ethereum/rlp"
 	"gopkg.in/urfave/cli.v1"
-	"bufio"
+	"io"
+	"io/ioutil"
+	"math/big"
+	"os"
+	"os/signal"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
 )
 
 const (
@@ -573,7 +572,9 @@ func rollback(ctx *cli.Context) error {
 
 	glog.Warning("Rolling back blockchain...")
 
-	bc.SetHead(blockIndex)
+	if err := bc.SetHead(blockIndex); err != nil {
+		glog.V(logger.Warn).Infof("error setting head: %v", err)
+	}
 
 	// Check if *neither* block nor fastblock numbers match desired head number
 	nowCurrentHead := bc.CurrentBlock().Number().Uint64()
@@ -945,6 +946,9 @@ func stringInSlice(s string, sl []string) bool {
 	return false
 }
 
+// makeMLogDocumentation creates markdown documentation text for, eg. wiki
+// That's why it uses 'fmt' instead of glog or log; output with prefixes (glog and log)
+// will break markdown.
 func makeMLogDocumentation(ctx *cli.Context) error {
 	wantComponents := ctx.Args()
 
@@ -1002,6 +1006,68 @@ func makeMLogDocumentation(ctx *cli.Context) error {
 	return nil
 }
 
+func recoverChaindata(ctx *cli.Context) error {
+
+	// Congruent to MakeChain(), but uses special NewBlockChainDryrun. Avoids a one-off function in flags.go.
+	var err error
+	sconf := mustMakeSufficientChainConfig(ctx)
+	bcdb := MakeChainDatabase(ctx)
+	defer bcdb.Close()
+
+	pow := pow.PoW(core.FakePow{})
+	if !ctx.GlobalBool(aliasableName(FakePoWFlag.Name, ctx)) {
+		pow = ethash.New()
+	} else {
+		glog.V(logger.Info).Info("Consensus: fake")
+	}
+
+	bc, err := core.NewBlockChainDryrun(bcdb, sconf.ChainConfig, pow, new(event.TypeMux))
+	if err != nil {
+		glog.Fatal("Could not start chain manager: ", err)
+	}
+
+	if blockchainLoadError := bc.LoadLastState(true); blockchainLoadError != nil {
+		glog.V(logger.Error).Infof("! Found error while loading blockchain: %v", blockchainLoadError)
+		// but do not return
+	}
+
+	header := bc.CurrentHeader()
+	currentBlock := bc.CurrentBlock()
+	currentFastBlock := bc.CurrentFastBlock()
+
+	glog.V(logger.Error).Infoln("Current status (before recovery attempt):")
+	if header != nil {
+		glog.V(logger.Error).Infof("Last header: #%d\n", header.Number.Uint64())
+		if currentBlock != nil {
+			glog.V(logger.Error).Infof("Last block: #%d\n", currentBlock.Number())
+		} else {
+			glog.V(logger.Error).Infoln("! Last block: nil")
+		}
+		if currentFastBlock != nil {
+			glog.V(logger.Error).Infof("Last fast block: #%d\n", currentFastBlock.Number())
+		} else {
+			glog.V(logger.Error).Infoln("! Last fast block: nil")
+		}
+	} else {
+		glog.V(logger.Error).Infoln("! Last header: nil")
+	}
+
+	glog.V(logger.Error).Infoln(glog.Separator("-"))
+
+	glog.V(logger.Error).Infoln("Checking db validity and recoverable data...")
+	checkpoint := bc.Recovery(1, 2048)
+	glog.V(logger.Error).Infof("Found last recoverable checkpoint=#%d\n", checkpoint)
+
+	glog.V(logger.Error).Infoln(glog.Separator("-"))
+
+	glog.V(logger.Error).Infoln("Setting blockchain db head to last safe checkpoint...")
+	if setHeadErr := bc.SetHead(checkpoint); setHeadErr != nil {
+		glog.V(logger.Error).Infof("Error setting head: %v\n", setHeadErr)
+		return setHeadErr
+	}
+	return nil
+}
+
 // https://gist.github.com/r0l1/3dcbb0c8f6cfe9c66ab8008f55f8f28b
 // askForConfirmation asks the user for confirmation. A user must type in "yes" or "no" and
 // then press enter. It has fuzzy matching, so "y", "Y", "yes", "YES", and "Yes" all count as
@@ -1050,4 +1116,4 @@ func resetChaindata(ctx *cli.Context) error {
 		glog.V(logger.Error).Infoln("Leaving chaindata untouched. As you were.")
 	}
 	return nil
-}
+}

cmd/geth/flag.go

@@ -106,7 +106,7 @@ var (
 	devModeDataDirPath = filepath.Join(os.TempDir(), "/ethereum_dev_mode")
 
 	cacheChainIdentity string
-	cacheChainConfig *core.SufficientChainConfig
+	cacheChainConfig   *core.SufficientChainConfig
 )
 
 // chainIsMorden allows either
@@ -151,7 +151,7 @@ func mustMakeChainIdentity(ctx *cli.Context) (identity string) {
 	if cacheChainIdentity != "" {
 		return cacheChainIdentity
 	}
-	defer func () {
+	defer func() {
 		cacheChainIdentity = identity
 	}()