materialize-bigquery: increase job timeouts due to concurrent updates…

… or rate limits

Increases the number of times the materialization will retry failed jobs due to
concurrent updates on the table or rate limits before failing the connector.

Most often this happens when two materializations share the same checkpoints
table, and long-running transactions can "lock" the checkpoints table for
extended periods of time.

materialize-bigquery/query.go

@@ -39,7 +39,7 @@ func (c client) newQuery(queryString string, parameters ...interface{}) *bigquer
 }
 
 const (
-	maxAttempts            = 10
+	maxAttempts            = 30
 	initialBackoff float64 = 200 // Milliseconds
 	maxBackoff             = time.Duration(60 * time.Second)
 )
@@ -99,12 +99,18 @@ func (c client) runQuery(ctx context.Context, query *bigquery.Query) (*bigquery.
 						delay = maxBackoff
 					}
 
-					log.WithFields(log.Fields{
+					ll := log.WithFields(log.Fields{
 						"attempt":   attempt,
 						"jobStatus": status,
 						"error":     err,
 						"delay":     delay.String(),
-					}).Warn("job failed (will retry)")
+					})
+
+					if attempt > 10 {
+						ll.Info("job failed (will retry)")
+					} else {
+						ll.Debug("job failed (will retry)")
+					}
 
 					select {
 					case <-ctx.Done():