Don't fail the workflow when failed to emit metrics for one domain (c…

…adence-workflow#6640)

* Don't fail the workflow when failed to emit metrics for one domain in ES analyzer

* Return error when all domains fail to emit metrics

service/worker/esanalyzer/domainWorkflowTypeCountWorkflow.go

@@ -148,6 +148,7 @@ func (w *Workflow) emitWorkflowTypeCountMetrics(ctx context.Context) error {
 		if err != nil {
 			return err
 		}
+		var failedDomains []string
 		for _, domainName := range workflowMetricDomainNames {
 			switch w.analyzer.readMode {
 			case ES:
@@ -158,9 +159,13 @@ func (w *Workflow) emitWorkflowTypeCountMetrics(ctx context.Context) error {
 				err = w.emitWorkflowTypeCountMetricsES(ctx, domainName, logger)
 			}
 			if err != nil {
-				return err
+				logger.Error(fmt.Sprintf("failed to emit workflow type metrics for domain %s", domainName), zap.Error(err))
+				failedDomains = append(failedDomains, domainName)
 			}
 		}
+		if len(failedDomains) == len(workflowMetricDomainNames) {
+			return fmt.Errorf("failed to emit workflow type metrics for all domains")
+		}
 	}
 	return nil
 }

service/worker/esanalyzer/workflow.go

@@ -197,6 +197,7 @@ func (w *Workflow) emitWorkflowVersionMetrics(ctx context.Context) error {
 		if err != nil {
 			return err
 		}
+		var failedDomains []string
 		for _, domainName := range workflowMetricDomainNames {
 			switch w.analyzer.readMode {
 			case ES:
@@ -207,9 +208,13 @@ func (w *Workflow) emitWorkflowVersionMetrics(ctx context.Context) error {
 				err = w.emitWorkflowVersionMetricsES(ctx, domainName, logger)
 			}
 			if err != nil {
-				return err
+				logger.Error(fmt.Sprintf("failed to emit workflow version metrics for domain %s", domainName), zap.Error(err))
+				failedDomains = append(failedDomains, domainName)
 			}
 		}
+		if len(failedDomains) == len(workflowMetricDomainNames) {
+			return fmt.Errorf("failed to emit workflow version metrics for all domains")
+		}
 	}
 	return nil
 }