diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager.go b/npm/pkg/dataplane/ipsets/ipsetmanager.go index dc80dfae56..b57bf67e91 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager.go @@ -51,6 +51,9 @@ type IPSetManager struct { setMap map[string]*IPSet dirtyCache dirtyCacheInterface ioShim *common.IOShim + // consecutiveApplyFailures is used in Linux to count the number of consecutive failures to apply ipsets + // if this count exceeds a threshold, we will panic + consecutiveApplyFailures int sync.RWMutex } @@ -71,6 +74,8 @@ func NewIPSetManager(iMgrCfg *IPSetManagerCfg, ioShim *common.IOShim) *IPSetMana setMap: make(map[string]*IPSet), dirtyCache: newDirtyCache(), ioShim: ioShim, + // set to 0 to avoid lint error for windows + consecutiveApplyFailures: 0, } } diff --git a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go index 8a7a8adbf0..3a29d14e37 100644 --- a/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go +++ b/npm/pkg/dataplane/ipsets/ipsetmanager_linux.go @@ -54,6 +54,8 @@ const ( destroySectionPrefix = "delete" addOrUpdateSectionPrefix = "add/update" ipsetRestoreLineFailurePattern = "Error in line (\\d+):" + + maxConsecutiveFailures = 100 ) var ( @@ -408,8 +410,19 @@ func (iMgr *IPSetManager) applyIPSets() error { creator := iMgr.fileCreatorForApply(maxTryCount) restoreError := creator.RunCommandWithFile(ipsetCommand, ipsetRestoreFlag) if restoreError != nil { + iMgr.consecutiveApplyFailures++ + if iMgr.consecutiveApplyFailures >= maxConsecutiveFailures { + msg := fmt.Sprintf("exceeded max consecutive failures (%d) when applying ipsets. final error: %s", maxConsecutiveFailures, restoreError.Error()) + klog.Error(msg) + metrics.SendErrorLogAndMetric(util.IpsmID, msg) + panic(msg) + } + return npmerrors.SimpleErrorWrapper("ipset restore failed when applying ipsets", restoreError) } + + iMgr.consecutiveApplyFailures = 0 + return nil }