diff --git a/pkg/net/libp2p/params.go b/pkg/net/libp2p/params.go index 77ef49bbd..acca8bf7e 100644 --- a/pkg/net/libp2p/params.go +++ b/pkg/net/libp2p/params.go @@ -30,9 +30,9 @@ func DefaultParams() Params { return Params{ ProtocolID: "/mir/0.0.1", ConnectionTTL: DefaultConnectionTTL, - ConnectionBufferSize: 128, + ConnectionBufferSize: 512, StreamWriteTimeout: 100 * time.Millisecond, - ReconnectionPeriod: time.Second, + ReconnectionPeriod: 3 * time.Second, MaxDataPerWrite: 100 * 1024, // 100 kiB MinComplainPeriod: time.Second, } diff --git a/pkg/net/libp2p/transport_test.go b/pkg/net/libp2p/transport_test.go index 769e70cf8..0494e9e68 100644 --- a/pkg/net/libp2p/transport_test.go +++ b/pkg/net/libp2p/transport_test.go @@ -263,7 +263,12 @@ func (m *mockLibp2pCommunication) testEventuallyConnected(nodeID1, nodeID2 stdty return network.Connected == n1.host.Network().Connectedness(n2.host.ID()) && m.streamExist(n1, n2) && m.streamExist(n2, n1) }, - 15*time.Second, 100*time.Millisecond) + 60*time.Second, 200*time.Millisecond) + // One minute seems like way too much, but we observed cases where connection attempts kept failing + // for over 30 seconds. + // TODO: there is something fishy about this. + // It only started occurring after updating quic-go from 0.33.0 to 0.39.4, + // as part of a libp2p update from 0.27.8 to 0.32.2. } func (m *mockLibp2pCommunication) testConnectionsEmpty() { @@ -654,9 +659,9 @@ func TestMessaging(t *testing.T) { received := 0 disconnect := make(chan struct{}) - testTimeDuration := time.Duration(15) - testTimer := time.NewTimer(testTimeDuration * time.Second) - disconnectTimer := time.NewTimer(testTimeDuration / 3 * time.Second) + testTimeDuration := 15 * time.Second + testTimer := time.NewTimer(testTimeDuration) + disconnectTimer := time.NewTimer(testTimeDuration / 3) wg.Add(1) go func() { diff --git a/pkg/trantor/testing/smr_test.go b/pkg/trantor/testing/smr_test.go index 0079c9d21..6df000bcd 100644 --- a/pkg/trantor/testing/smr_test.go +++ b/pkg/trantor/testing/smr_test.go @@ -7,6 +7,7 @@ import ( "os" "path/filepath" "strconv" + "sync" "testing" "time" @@ -37,7 +38,8 @@ import ( ) const ( - failedTestDir = "failed-test-data" + failedTestDir = "failed-test-data" + simTransportName = "sim" ) func TestIntegration(t *testing.T) { @@ -74,6 +76,7 @@ type TestConfig struct { CrashedReplicas map[int]bool CheckFunc func(tb testing.TB, deployment *deploytest.Deployment, conf *TestConfig) Logger logging.Logger + Skip bool // If set to true, test is skipped. } func testIntegrationWithISS(tt *testing.T) { @@ -117,7 +120,7 @@ func testIntegrationWithISS(tt *testing.T) { NumClients: 1, Transport: "libp2p", NumFakeTXs: 10, - Duration: 10 * time.Second, + Duration: 60 * time.Second, }}, 5: {"Submit 10 transactions with 1 node and libp2p networking", &TestConfig{ @@ -125,7 +128,7 @@ func testIntegrationWithISS(tt *testing.T) { NumClients: 1, Transport: "libp2p", NumNetTXs: 10, - Duration: 10 * time.Second, + Duration: 20 * time.Second, }}, 6: {"Submit 10 transactions with 4 nodes and libp2p networking", &TestConfig{ @@ -134,25 +137,25 @@ func testIntegrationWithISS(tt *testing.T) { NumClients: 1, Transport: "libp2p", NumNetTXs: 10, - Duration: 15 * time.Second, + Duration: 60 * time.Second, }}, 7: {"Do nothing with 1 node in simulation", &TestConfig{ NodeIDsWeight: deploytest.NewNodeIDsDefaultWeights(1), - Transport: "sim", + Transport: simTransportName, Duration: 4 * time.Second, }}, 8: {"Do nothing with 4 nodes in simulation, one of them slow", &TestConfig{ NodeIDsWeight: deploytest.NewNodeIDsDefaultWeights(4), - Transport: "sim", + Transport: simTransportName, Duration: 20 * time.Second, SlowProposeReplicas: map[int]bool{0: true}, }}, 9: {"Submit 10 fake transactions with 1 node in simulation", &TestConfig{ NodeIDsWeight: deploytest.NewNodeIDsDefaultWeights(1), - Transport: "sim", + Transport: simTransportName, NumFakeTXs: 10, Directory: "mirbft-deployment-test", Duration: 4 * time.Second, @@ -161,7 +164,7 @@ func testIntegrationWithISS(tt *testing.T) { &TestConfig{ NodeIDsWeight: deploytest.NewNodeIDsDefaultWeights(1), NumClients: 1, - Transport: "sim", + Transport: simTransportName, NumFakeTXs: 10, Directory: "mirbft-deployment-test", Duration: 4 * time.Second, @@ -170,7 +173,7 @@ func testIntegrationWithISS(tt *testing.T) { &TestConfig{ NodeIDsWeight: deploytest.NewNodeIDsDefaultWeights(1), NumClients: 0, - Transport: "sim", + Transport: simTransportName, NumFakeTXs: 100, Duration: 20 * time.Second, }}, @@ -178,7 +181,7 @@ func testIntegrationWithISS(tt *testing.T) { &TestConfig{ NodeIDsWeight: deploytest.NewNodeIDsDefaultWeights(4), NumClients: 0, - Transport: "sim", + Transport: simTransportName, NumFakeTXs: 100, Duration: 20 * time.Second, SlowProposeReplicas: map[int]bool{0: true}, @@ -190,9 +193,9 @@ func testIntegrationWithISS(tt *testing.T) { return types.VoteWeight(fmt.Sprintf("%d0000000000000000000", pow2(int(numericID)))) // ensures last 2 nodes weight is greater than twice the sum of the others' }), NumClients: 0, - Transport: "libp2p", + Transport: simTransportName, NumFakeTXs: 100, - Duration: 20 * time.Second, + Duration: 30 * time.Second, ErrorExpected: es.Errorf("no transactions were delivered"), CrashedReplicas: map[int]bool{2: true, 3: true}, CheckFunc: func(tb testing.TB, deployment *deploytest.Deployment, conf *TestConfig) { @@ -210,9 +213,9 @@ func testIntegrationWithISS(tt *testing.T) { return types.VoteWeight(fmt.Sprintf("%d0000000000000000000", pow2(int(4-numericID)))) // ensures first 2 nodes weight is greater than twice the sum of the others' }), NumClients: 0, - Transport: "libp2p", + Transport: simTransportName, NumFakeTXs: 100, - Duration: 40 * time.Second, + Duration: 60 * time.Second, ErrorExpected: es.Errorf("no transactions were delivered"), CrashedReplicas: map[int]bool{2: true, 3: true}, CheckFunc: func(tb testing.TB, deployment *deploytest.Deployment, conf *TestConfig) { @@ -228,12 +231,39 @@ func testIntegrationWithISS(tt *testing.T) { for i, test := range tests { i, test := i, test - // Create a directory for the deployment-generated files and set the test directory name. - // The directory will be automatically removed when the outer test function exits. - createDeploymentDir(tt, test.Config) - + var lock sync.Mutex tt.Run(fmt.Sprintf("%03d", i), func(t *testing.T) { - simMode := (test.Config.Transport == "sim") + + lock.Lock() + defer lock.Unlock() + + if test.Config.Skip { + t.Logf("Skipping test: %s", t.Name()) + return + } + + defer func() { + if err := recover(); err != nil || t.Failed() { + t.Logf("Test #%03d (%s) failed", i, test.Desc) + if test.Config.Transport == simTransportName { + t.Logf("Reproduce with RANDOM_SEED=%d", test.Config.RandomSeed) + } + // Save the test data. + testRelDir, err := filepath.Rel(os.TempDir(), test.Config.Directory) + require.NoError(t, err) + retainedDir := filepath.Join(failedTestDir, testRelDir) + + t.Logf("Saving deployment data to: %s\n", retainedDir) + err = copy.Copy(test.Config.Directory, retainedDir) + require.NoError(t, err) + } + }() + + // Create a directory for the deployment-generated files and set the test directory name. + // The directory will be automatically removed when the outer test function exits. + createDeploymentDir(t, test.Config) + + simMode := test.Config.Transport == simTransportName if testing.Short() && !simMode { t.SkipNow() } @@ -250,13 +280,6 @@ func testIntegrationWithISS(tt *testing.T) { } runIntegrationWithISSConfig(t, test.Config) - - if t.Failed() { - t.Logf("Test #%03d (%s) failed", i, test.Desc) - if simMode { - t.Logf("Reproduce with RANDOM_SEED=%d", test.Config.RandomSeed) - } - } }) } } @@ -390,15 +413,15 @@ func createDeploymentDir(tb testing.TB, conf *TestConfig) { if conf.Directory != "" { conf.Directory = filepath.Join(os.TempDir(), conf.Directory) - tb.Logf("Using deployment dir: %s\n", conf.Directory) - err := os.MkdirAll(conf.Directory, 0777) - require.NoError(tb, err) - tb.Cleanup(func() { os.RemoveAll(conf.Directory) }) } else { // If no directory is configured, create a temporary directory in the OS-default location. - conf.Directory = tb.TempDir() - tb.Logf("Created temp dir: %s\n", conf.Directory) + conf.Directory = filepath.Join(tb.TempDir(), tb.Name()) } + + tb.Logf("Using deployment dir: %s\n", conf.Directory) + err := os.MkdirAll(conf.Directory, 0777) + require.NoError(tb, err) + tb.Cleanup(func() { os.RemoveAll(conf.Directory) }) } func newDeployment(conf *TestConfig) (*deploytest.Deployment, error) { @@ -406,7 +429,7 @@ func newDeployment(conf *TestConfig) (*deploytest.Deployment, error) { logger := deploytest.NewLogger(conf.Logger) var simulation *deploytest.Simulation - if conf.Transport == "sim" { + if conf.Transport == simTransportName { r := rand.New(rand.NewSource(conf.RandomSeed)) // nolint: gosec eventDelayFn := func(e stdtypes.Event) time.Duration { // TODO: Make min and max event processing delay configurable