From 7d33065019f78d9150a8c89ddb4593f81e6ff9b3 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Tue, 31 Dec 2024 11:15:42 +0800 Subject: [PATCH 01/33] grafana: Add `gRPC Received commands rate` panel (#8921) close tikv/pd#8920 Signed-off-by: okJiang <819421878@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- metrics/grafana/pd.json | 220 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 220 insertions(+) diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 0f4e91afd50..62b2e7234ef 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -8937,6 +8937,226 @@ "align": false, "alignLevel": null } + }, + { + "aliasColors": {}, + "dashLength": 10, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The rate of received each kind of gRPC commands", + "editable": true, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 127 + }, + "id": 904, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "paceLength": 10, + "pointradius": 5, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "targets": [ + { + "expr": "sum(rate(grpc_server_msg_received_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\"}[1m])) by (instance, grpc_method)", + "legendFormat": "{{instance}}-{{grpc_method}}", + "interval": "", + "exemplar": true, + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "gRPC Received commands rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "options": { + "alertThreshold": true + }, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "pluginVersion": "7.5.17", + "bars": false, + "dashes": false, + "decimals": null, + "error": false, + "percentage": false, + "points": false, + "stack": false, + "steppedLine": false, + "timeFrom": null, + "timeShift": null, + "fillGradient": 0, + "hiddenSeries": false + }, + { + "aliasColors": {}, + "dashLength": 10, + "datasource": "${DS_TEST-CLUSTER}", + "description": "The error rate of handled gRPC commands.Note: It can't catch the error hide in the header, like this https://github.com/tikv/pd/blob/2d970a619a8917c35d306f401326141481c133e0/server/grpc_service.go#L2071", + "editable": true, + "fieldConfig": { + "defaults": {}, + "overrides": [] + }, + "fill": 1, + "grid": {}, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 135 + }, + "id": 905, + "legend": { + "alignAsTable": true, + "avg": false, + "current": true, + "hideEmpty": true, + "hideZero": true, + "max": true, + "min": false, + "rightSide": true, + "show": true, + "sideWidth": 300, + "total": false, + "values": true + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "options": { + "alertThreshold": true + }, + "paceLength": 10, + "pluginVersion": "7.5.17", + "pointradius": 5, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "targets": [ + { + "expr": "sum(rate(grpc_server_handled_total{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\", instance=~\"$instance\", grpc_type=\"unary\", grpc_code!=\"OK\"}[1m])) by (grpc_method)", + "legendFormat": "{{grpc_method}}", + "interval": "", + "exemplar": true, + "intervalFactor": 2, + "refId": "A", + "step": 4 + } + ], + "thresholds": [], + "timeRegions": [], + "title": "gRPC Error rate", + "tooltip": { + "msResolution": false, + "shared": true, + "sort": 0, + "value_type": "cumulative" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "ops", + "label": null, + "logBase": 10, + "max": null, + "min": null, + "show": true, + "$$hashKey": "object:132" + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true, + "$$hashKey": "object:133" + } + ], + "yaxis": { + "align": false, + "alignLevel": null + }, + "bars": false, + "dashes": false, + "decimals": null, + "error": false, + "fillGradient": 0, + "hiddenSeries": false, + "percentage": false, + "points": false, + "stack": false, + "steppedLine": false, + "timeFrom": null, + "timeShift": null } ], "repeat": null, From 5ad4301b7a095a348ba9d5a5ffd2df8d021dcf44 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Tue, 31 Dec 2024 13:54:33 +0800 Subject: [PATCH 02/33] client/pkg: introduce the deadline watcher (#8955) ref tikv/pd#8690 Introduce the deadline watcher. Signed-off-by: JmPotato Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/clients/tso/dispatcher.go | 73 +++--------------- client/errs/errno.go | 1 - client/pkg/deadline/watcher.go | 111 ++++++++++++++++++++++++++++ client/pkg/deadline/watcher_test.go | 58 +++++++++++++++ errors.toml | 5 -- pkg/errs/errno.go | 1 - 6 files changed, 181 insertions(+), 68 deletions(-) create mode 100644 client/pkg/deadline/watcher.go create mode 100644 client/pkg/deadline/watcher_test.go diff --git a/client/clients/tso/dispatcher.go b/client/clients/tso/dispatcher.go index c05ab27d755..1cc2b2aa940 100644 --- a/client/clients/tso/dispatcher.go +++ b/client/clients/tso/dispatcher.go @@ -36,33 +36,12 @@ import ( "github.com/tikv/pd/client/opt" "github.com/tikv/pd/client/pkg/batch" cctx "github.com/tikv/pd/client/pkg/connectionctx" + "github.com/tikv/pd/client/pkg/deadline" "github.com/tikv/pd/client/pkg/retry" - "github.com/tikv/pd/client/pkg/utils/timerutil" "github.com/tikv/pd/client/pkg/utils/tsoutil" sd "github.com/tikv/pd/client/servicediscovery" ) -// deadline is used to control the TS request timeout manually, -// it will be sent to the `tsDeadlineCh` to be handled by the `watchTSDeadline` goroutine. -type deadline struct { - timer *time.Timer - done chan struct{} - cancel context.CancelFunc -} - -func newTSDeadline( - timeout time.Duration, - done chan struct{}, - cancel context.CancelFunc, -) *deadline { - timer := timerutil.GlobalTimerPool.Get(timeout) - return &deadline{ - timer: timer, - done: done, - cancel: cancel, - } -} - type tsoInfo struct { tsoServer string reqKeyspaceGroupID uint32 @@ -86,10 +65,10 @@ type tsoDispatcher struct { ctx context.Context cancel context.CancelFunc - provider tsoServiceProvider - tsoRequestCh chan *Request - tsDeadlineCh chan *deadline - latestTSOInfo atomic.Pointer[tsoInfo] + provider tsoServiceProvider + tsoRequestCh chan *Request + deadlineWatcher *deadline.Watcher + latestTSOInfo atomic.Pointer[tsoInfo] // For reusing `*batchController` objects batchBufferPool *sync.Pool @@ -119,11 +98,11 @@ func newTSODispatcher( tokenCh := make(chan struct{}, tokenChCapacity) td := &tsoDispatcher{ - ctx: dispatcherCtx, - cancel: dispatcherCancel, - provider: provider, - tsoRequestCh: tsoRequestCh, - tsDeadlineCh: make(chan *deadline, tokenChCapacity), + ctx: dispatcherCtx, + cancel: dispatcherCancel, + provider: provider, + tsoRequestCh: tsoRequestCh, + deadlineWatcher: deadline.NewWatcher(dispatcherCtx, tokenChCapacity, "tso"), batchBufferPool: &sync.Pool{ New: func() any { return batch.NewController[*Request]( @@ -135,34 +114,9 @@ func newTSODispatcher( }, tokenCh: tokenCh, } - go td.watchTSDeadline() return td } -func (td *tsoDispatcher) watchTSDeadline() { - log.Info("[tso] start tso deadline watcher") - defer log.Info("[tso] exit tso deadline watcher") - for { - select { - case d := <-td.tsDeadlineCh: - select { - case <-d.timer.C: - log.Error("[tso] tso request is canceled due to timeout", - errs.ZapError(errs.ErrClientGetTSOTimeout)) - d.cancel() - timerutil.GlobalTimerPool.Put(d.timer) - case <-d.done: - timerutil.GlobalTimerPool.Put(d.timer) - case <-td.ctx.Done(): - timerutil.GlobalTimerPool.Put(d.timer) - return - } - case <-td.ctx.Done(): - return - } - } -} - func (td *tsoDispatcher) revokePendingRequests(err error) { for range len(td.tsoRequestCh) { req := <-td.tsoRequestCh @@ -378,14 +332,11 @@ tsoBatchLoop: } } - done := make(chan struct{}) - dl := newTSDeadline(option.Timeout, done, cancel) - select { - case <-ctx.Done(): + done := td.deadlineWatcher.Start(ctx, option.Timeout, cancel) + if done == nil { // Finish the collected requests if the context is canceled. td.cancelCollectedRequests(tsoBatchController, invalidStreamID, errors.WithStack(ctx.Err())) return - case td.tsDeadlineCh <- dl: } // processRequests guarantees that the collected requests could be finished properly. err = td.processRequests(stream, tsoBatchController, done) diff --git a/client/errs/errno.go b/client/errs/errno.go index 25665f01017..99a426d0776 100644 --- a/client/errs/errno.go +++ b/client/errs/errno.go @@ -56,7 +56,6 @@ var ( ErrClientGetMetaStorageClient = errors.Normalize("failed to get meta storage client", errors.RFCCodeText("PD:client:ErrClientGetMetaStorageClient")) ErrClientCreateTSOStream = errors.Normalize("create TSO stream failed, %s", errors.RFCCodeText("PD:client:ErrClientCreateTSOStream")) ErrClientTSOStreamClosed = errors.Normalize("encountered TSO stream being closed unexpectedly", errors.RFCCodeText("PD:client:ErrClientTSOStreamClosed")) - ErrClientGetTSOTimeout = errors.Normalize("get TSO timeout", errors.RFCCodeText("PD:client:ErrClientGetTSOTimeout")) ErrClientGetTSO = errors.Normalize("get TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetTSO")) ErrClientGetMinTSO = errors.Normalize("get min TSO failed, %v", errors.RFCCodeText("PD:client:ErrClientGetMinTSO")) ErrClientGetLeader = errors.Normalize("get leader failed, %v", errors.RFCCodeText("PD:client:ErrClientGetLeader")) diff --git a/client/pkg/deadline/watcher.go b/client/pkg/deadline/watcher.go new file mode 100644 index 00000000000..b40857edbfd --- /dev/null +++ b/client/pkg/deadline/watcher.go @@ -0,0 +1,111 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deadline + +import ( + "context" + "time" + + "go.uber.org/zap" + + "github.com/pingcap/log" + + "github.com/tikv/pd/client/pkg/utils/timerutil" +) + +// The `cancel` function will be invoked once the specified `timeout` elapses without receiving a `done` signal. +type deadline struct { + timer *time.Timer + done chan struct{} + cancel context.CancelFunc +} + +// Watcher is used to watch and manage the deadlines. +type Watcher struct { + ctx context.Context + source string + Ch chan *deadline +} + +// NewWatcher is used to create a new deadline watcher. +func NewWatcher(ctx context.Context, capacity int, source string) *Watcher { + watcher := &Watcher{ + ctx: ctx, + source: source, + Ch: make(chan *deadline, capacity), + } + go watcher.Watch() + return watcher +} + +// Watch is used to watch the deadlines and invoke the `cancel` function when the deadline is reached. +// The `err` will be returned if the deadline is reached. +func (w *Watcher) Watch() { + log.Info("[pd] start the deadline watcher", zap.String("source", w.source)) + defer log.Info("[pd] exit the deadline watcher", zap.String("source", w.source)) + for { + select { + case d := <-w.Ch: + select { + case <-d.timer.C: + log.Error("[pd] the deadline is reached", zap.String("source", w.source)) + d.cancel() + timerutil.GlobalTimerPool.Put(d.timer) + case <-d.done: + timerutil.GlobalTimerPool.Put(d.timer) + case <-w.ctx.Done(): + timerutil.GlobalTimerPool.Put(d.timer) + return + } + case <-w.ctx.Done(): + return + } + } +} + +// Start is used to start a deadline. It returns a channel which will be closed when the deadline is reached. +// Returns nil if the deadline is not started. +func (w *Watcher) Start( + ctx context.Context, + timeout time.Duration, + cancel context.CancelFunc, +) chan struct{} { + // Check if the watcher is already canceled. + select { + case <-w.ctx.Done(): + return nil + case <-ctx.Done(): + return nil + default: + } + // Initialize the deadline. + timer := timerutil.GlobalTimerPool.Get(timeout) + d := &deadline{ + timer: timer, + done: make(chan struct{}), + cancel: cancel, + } + // Send the deadline to the watcher. + select { + case <-w.ctx.Done(): + timerutil.GlobalTimerPool.Put(timer) + return nil + case <-ctx.Done(): + timerutil.GlobalTimerPool.Put(timer) + return nil + case w.Ch <- d: + return d.done + } +} diff --git a/client/pkg/deadline/watcher_test.go b/client/pkg/deadline/watcher_test.go new file mode 100644 index 00000000000..b93987b8874 --- /dev/null +++ b/client/pkg/deadline/watcher_test.go @@ -0,0 +1,58 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package deadline + +import ( + "context" + "sync/atomic" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestWatcher(t *testing.T) { + re := require.New(t) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + watcher := NewWatcher(ctx, 10, "test") + var deadlineReached atomic.Bool + done := watcher.Start(ctx, time.Millisecond, func() { + deadlineReached.Store(true) + }) + re.NotNil(done) + time.Sleep(5 * time.Millisecond) + re.True(deadlineReached.Load()) + + deadlineReached.Store(false) + done = watcher.Start(ctx, 500*time.Millisecond, func() { + deadlineReached.Store(true) + }) + re.NotNil(done) + done <- struct{}{} + time.Sleep(time.Second) + re.False(deadlineReached.Load()) + + deadCtx, deadCancel := context.WithCancel(ctx) + deadCancel() + deadlineReached.Store(false) + done = watcher.Start(deadCtx, time.Millisecond, func() { + deadlineReached.Store(true) + }) + re.Nil(done) + time.Sleep(5 * time.Millisecond) + re.False(deadlineReached.Load()) +} diff --git a/errors.toml b/errors.toml index 2ab3b014f5a..9980a98ab14 100644 --- a/errors.toml +++ b/errors.toml @@ -131,11 +131,6 @@ error = ''' get TSO failed ''' -["PD:client:ErrClientGetTSOTimeout"] -error = ''' -get TSO timeout -''' - ["PD:cluster:ErrInvalidStoreID"] error = ''' invalid store id %d, not found diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go index ee24b4d0673..834bf4f824e 100644 --- a/pkg/errs/errno.go +++ b/pkg/errs/errno.go @@ -144,7 +144,6 @@ var ( // client errors var ( ErrClientCreateTSOStream = errors.Normalize("create TSO stream failed, %s", errors.RFCCodeText("PD:client:ErrClientCreateTSOStream")) - ErrClientGetTSOTimeout = errors.Normalize("get TSO timeout", errors.RFCCodeText("PD:client:ErrClientGetTSOTimeout")) ErrClientGetTSO = errors.Normalize("get TSO failed", errors.RFCCodeText("PD:client:ErrClientGetTSO")) ErrClientGetLeader = errors.Normalize("get leader failed, %v", errors.RFCCodeText("PD:client:ErrClientGetLeader")) ErrClientGetMember = errors.Normalize("get member failed", errors.RFCCodeText("PD:client:ErrClientGetMember")) From c2d48542d8c4c1d1a42ebc0b2449993080d57d29 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 2 Jan 2025 11:20:24 +0800 Subject: [PATCH 03/33] *: format log fields using kebab-case style (#8956) ref tikv/pd#4322 Signed-off-by: Ryan Leung --- pkg/core/region.go | 4 ++-- pkg/encryption/key_manager.go | 2 +- pkg/keyspace/keyspace.go | 2 +- .../resourcemanager/server/token_buckets.go | 2 +- pkg/memory/meminfo.go | 4 ++-- pkg/schedule/config/store_config.go | 2 +- pkg/schedule/schedulers/grant_hot_region.go | 2 +- pkg/tso/global_allocator.go | 2 +- pkg/tso/keyspace_group_manager.go | 22 +++++++++---------- server/grpc_service.go | 2 +- server/server.go | 2 +- server/util.go | 2 +- 12 files changed, 24 insertions(+), 24 deletions(-) diff --git a/pkg/core/region.go b/pkg/core/region.go index 5f5a4a5f2e0..706e6bbd712 100644 --- a/pkg/core/region.go +++ b/pkg/core/region.go @@ -1885,7 +1885,7 @@ func scanRegion(regionTree *regionTree, keyRange *KeyRange, limit int, outputMus keyRange.StartKey, keyRange.EndKey, lastRegion.GetStartKey(), lastRegion.GetEndKey(), region.GetStartKey(), region.GetEndKey()) - log.Warn("scan regions failed", zap.Bool("outputMustContainAllKeyRange", + log.Warn("scan regions failed", zap.Bool("contain-all-key-range", outputMustContainAllKeyRange), zap.Error(err)) if outputMustContainAllKeyRange { return false @@ -1907,7 +1907,7 @@ func scanRegion(regionTree *regionTree, keyRange *KeyRange, limit int, outputMus keyRange.StartKey, keyRange.EndKey, lastRegion.GetStartKey(), lastRegion.GetEndKey(), lastRegion.GetEndKey(), keyRange.EndKey) - log.Warn("scan regions failed", zap.Bool("outputMustContainAllKeyRange", + log.Warn("scan regions failed", zap.Bool("contain-all-key-range", outputMustContainAllKeyRange), zap.Error(err)) if outputMustContainAllKeyRange { return nil, err diff --git a/pkg/encryption/key_manager.go b/pkg/encryption/key_manager.go index 54e5fa01b35..5fc6788c549 100644 --- a/pkg/encryption/key_manager.go +++ b/pkg/encryption/key_manager.go @@ -413,7 +413,7 @@ func (m *Manager) rotateKeyIfNeeded(forceUpdate bool) error { keys.Keys[keyID] = key keys.CurrentKeyId = keyID rotated = true - log.Info("ready to create or rotate data encryption key", zap.Uint64("keyID", keyID)) + log.Info("ready to create or rotate data encryption key", zap.Uint64("key-id", keyID)) break } // Duplicated key id. retry. diff --git a/pkg/keyspace/keyspace.go b/pkg/keyspace/keyspace.go index c9e390df47a..93312ae4ff1 100644 --- a/pkg/keyspace/keyspace.go +++ b/pkg/keyspace/keyspace.go @@ -593,7 +593,7 @@ func (manager *Manager) UpdateKeyspaceState(name string, newState keyspacepb.Key return nil, err } log.Info("[keyspace] keyspace state updated", - zap.Uint32("ID", meta.GetId()), + zap.Uint32("id", meta.GetId()), zap.String("keyspace-id", meta.GetName()), zap.String("new-state", newState.String()), ) diff --git a/pkg/mcs/resourcemanager/server/token_buckets.go b/pkg/mcs/resourcemanager/server/token_buckets.go index 50dc78c9d68..be32884e1ea 100644 --- a/pkg/mcs/resourcemanager/server/token_buckets.go +++ b/pkg/mcs/resourcemanager/server/token_buckets.go @@ -174,7 +174,7 @@ func (gts *GroupTokenBucketState) balanceSlotTokens( if time.Since(slot.lastReqTime) >= slotExpireTimeout { delete(gts.tokenSlots, clientUniqueID) log.Info("delete resource group slot because expire", zap.Time("last-req-time", slot.lastReqTime), - zap.Any("expire timeout", slotExpireTimeout), zap.Any("del client id", clientUniqueID), zap.Any("len", len(gts.tokenSlots))) + zap.Duration("expire-timeout", slotExpireTimeout), zap.Uint64("del-client-id", clientUniqueID), zap.Int("len", len(gts.tokenSlots))) } } } diff --git a/pkg/memory/meminfo.go b/pkg/memory/meminfo.go index 7ed1afb579b..9ce9ff9b886 100644 --- a/pkg/memory/meminfo.go +++ b/pkg/memory/meminfo.go @@ -210,9 +210,9 @@ func InitMemoryHook() { MemTotal = MemTotalCGroup MemUsed = MemUsedCGroup sysutil.RegisterGetMemoryCapacity(MemTotalCGroup) - log.Info("use cgroup memory hook", zap.Int64("cgroupMemorySize", int64(cgroupValue)), zap.Int64("physicalMemorySize", int64(physicalValue))) + log.Info("use cgroup memory hook", zap.Int64("cgroup-memory-size", int64(cgroupValue)), zap.Int64("physical-memory-size", int64(physicalValue))) } else { - log.Info("use physical memory hook", zap.Int64("cgroupMemorySize", int64(cgroupValue)), zap.Int64("physicalMemorySize", int64(physicalValue))) + log.Info("use physical memory hook", zap.Int64("cgroup-memory-size", int64(cgroupValue)), zap.Int64("physical-memory-size", int64(physicalValue))) } _, err = MemTotal() mustNil(err) diff --git a/pkg/schedule/config/store_config.go b/pkg/schedule/config/store_config.go index 5575f0d9d56..cbf085d93fb 100644 --- a/pkg/schedule/config/store_config.go +++ b/pkg/schedule/config/store_config.go @@ -190,7 +190,7 @@ func (c *StoreConfig) CheckRegionKeys(keys, mergeKeys uint64) error { } if smallKeys := keys % c.GetRegionSplitKeys(); smallKeys <= mergeKeys && smallKeys > 0 { - log.Debug("region keys is too small", zap.Uint64("keys", keys), zap.Uint64("merge-keys", mergeKeys), zap.Uint64("smallSize", smallKeys)) + log.Debug("region keys is too small", zap.Uint64("keys", keys), zap.Uint64("merge-keys", mergeKeys), zap.Uint64("small-keys", smallKeys)) return errs.ErrCheckerMergeAgain.FastGenByArgs("the smallest region of the split regions is less than max-merge-region-keys") } return nil diff --git a/pkg/schedule/schedulers/grant_hot_region.go b/pkg/schedule/schedulers/grant_hot_region.go index 79be126f1d4..005e6b4182a 100644 --- a/pkg/schedule/schedulers/grant_hot_region.go +++ b/pkg/schedule/schedulers/grant_hot_region.go @@ -262,7 +262,7 @@ func (s *grantHotRegionScheduler) randomSchedule(cluster sche.SchedulerCluster, op, err := s.transfer(cluster, peer.RegionID, srcStoreID, isLeader) if err != nil { log.Debug("fail to create grant hot region operator", zap.Uint64("region-id", peer.RegionID), - zap.Uint64("src store id", srcStoreID), errs.ZapError(err)) + zap.Uint64("src-store-id", srcStoreID), errs.ZapError(err)) continue } return []*operator.Operator{op} diff --git a/pkg/tso/global_allocator.go b/pkg/tso/global_allocator.go index d6bf27878a7..553e0b0effd 100644 --- a/pkg/tso/global_allocator.go +++ b/pkg/tso/global_allocator.go @@ -212,7 +212,7 @@ func (gta *GlobalTSOAllocator) primaryElectionLoop() { zap.String("server-name", gta.member.Name()), zap.String("expected-primary-id", expectedPrimary), zap.Uint64("member-id", gta.member.ID()), - zap.String("cur-memberValue", gta.member.MemberValue())) + zap.String("cur-member-value", gta.member.MemberValue())) time.Sleep(200 * time.Millisecond) continue } diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index d22d284e1be..149c68029be 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -1297,7 +1297,7 @@ func (kgm *KeyspaceGroupManager) mergingChecker(ctx context.Context, mergeTarget log.Info("start to merge the keyspace group", zap.String("member", kgm.tsoServiceID.ServiceAddr), zap.Uint32("merge-target-id", mergeTargetID), - zap.Any("merge-list", mergeList)) + zap.Uint32s("merge-list", mergeList)) defer logutil.LogPanic() defer kgm.wg.Done() @@ -1316,7 +1316,7 @@ mergeLoop: log.Info("merging checker is closed", zap.String("member", kgm.tsoServiceID.ServiceAddr), zap.Uint32("merge-target-id", mergeTargetID), - zap.Any("merge-list", mergeList)) + zap.Uint32s("merge-list", mergeList)) return case <-checkTicker.C: } @@ -1326,7 +1326,7 @@ mergeLoop: log.Warn("unable to get the merge target allocator manager", zap.String("member", kgm.tsoServiceID.ServiceAddr), zap.Uint32("keyspace-group-id", mergeTargetID), - zap.Any("merge-list", mergeList), + zap.Uint32s("merge-list", mergeList), zap.Error(err)) continue } @@ -1336,7 +1336,7 @@ mergeLoop: log.Debug("current tso node is not the merge target primary", zap.String("member", kgm.tsoServiceID.ServiceAddr), zap.Uint32("merge-target-id", mergeTargetID), - zap.Any("merge-list", mergeList)) + zap.Uint32s("merge-list", mergeList)) continue } // Check if the keyspace group primaries in the merge map are all gone. @@ -1351,7 +1351,7 @@ mergeLoop: log.Error("failed to check if the keyspace group primary in the merge list has gone", zap.String("member", kgm.tsoServiceID.ServiceAddr), zap.Uint32("merge-target-id", mergeTargetID), - zap.Any("merge-list", mergeList), + zap.Uint32s("merge-list", mergeList), zap.Uint32("merge-id", id), zap.Any("remaining", mergeMap), zap.Error(err)) @@ -1370,7 +1370,7 @@ mergeLoop: "start to calculate the newly merged TSO", zap.String("member", kgm.tsoServiceID.ServiceAddr), zap.Uint32("merge-target-id", mergeTargetID), - zap.Any("merge-list", mergeList)) + zap.Uint32s("merge-list", mergeList)) // All the keyspace group primaries in the merge list are gone, // calculate the newly merged TSO to make sure it is greater than the original ones. var mergedTS time.Time @@ -1380,7 +1380,7 @@ mergeLoop: log.Error("failed to load the keyspace group TSO", zap.String("member", kgm.tsoServiceID.ServiceAddr), zap.Uint32("merge-target-id", mergeTargetID), - zap.Any("merge-list", mergeList), + zap.Uint32s("merge-list", mergeList), zap.Uint32("merge-id", id), zap.Time("ts", ts), zap.Error(err)) @@ -1396,7 +1396,7 @@ mergeLoop: log.Info("start to set the newly merged TSO", zap.String("member", kgm.tsoServiceID.ServiceAddr), zap.Uint32("merge-target-id", mergeTargetID), - zap.Any("merge-list", mergeList), + zap.Uint32s("merge-list", mergeList), zap.Time("merged-ts", mergedTS)) err = am.GetAllocator().SetTSO( tsoutil.GenerateTS(tsoutil.GenerateTimestamp(mergedTS, 1)), @@ -1405,7 +1405,7 @@ mergeLoop: log.Error("failed to update the newly merged TSO", zap.String("member", kgm.tsoServiceID.ServiceAddr), zap.Uint32("merge-target-id", mergeTargetID), - zap.Any("merge-list", mergeList), + zap.Uint32s("merge-list", mergeList), zap.Time("merged-ts", mergedTS), zap.Error(err)) continue @@ -1417,7 +1417,7 @@ mergeLoop: log.Error("failed to finish the merge", zap.String("member", kgm.tsoServiceID.ServiceAddr), zap.Uint32("merge-target-id", mergeTargetID), - zap.Any("merge-list", mergeList), + zap.Uint32s("merge-list", mergeList), zap.Error(err)) continue } @@ -1425,7 +1425,7 @@ mergeLoop: log.Info("finished merging keyspace group", zap.String("member", kgm.tsoServiceID.ServiceAddr), zap.Uint32("merge-target-id", mergeTargetID), - zap.Any("merge-list", mergeList), + zap.Uint32s("merge-list", mergeList), zap.Time("merged-ts", mergedTS)) return } diff --git a/server/grpc_service.go b/server/grpc_service.go index 398325cd30a..8db79d3b8f5 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -2803,7 +2803,7 @@ func (s *GrpcServer) ReportMinResolvedTS(ctx context.Context, request *pdpb.Repo } log.Debug("updated min resolved-ts", zap.Uint64("store", storeID), - zap.Uint64("min resolved-ts", minResolvedTS)) + zap.Uint64("min-resolved-ts", minResolvedTS)) return &pdpb.ReportMinResolvedTsResponse{ Header: wrapHeader(), }, nil diff --git a/server/server.go b/server/server.go index 3f397da4d0b..4cae86b6587 100644 --- a/server/server.go +++ b/server/server.go @@ -2042,7 +2042,7 @@ func (s *Server) GetExternalTS() uint64 { func (s *Server) SetExternalTS(externalTS, globalTS uint64) error { if tsoutil.CompareTimestampUint64(externalTS, globalTS) == 1 { desc := "the external timestamp should not be larger than global ts" - log.Error(desc, zap.Uint64("request timestamp", externalTS), zap.Uint64("global ts", globalTS)) + log.Error(desc, zap.Uint64("request-timestamp", externalTS), zap.Uint64("global-ts", globalTS)) return errors.New(desc) } c := s.GetRaftCluster() diff --git a/server/util.go b/server/util.go index 1764e4e9850..2f05c06b8f5 100644 --- a/server/util.go +++ b/server/util.go @@ -54,7 +54,7 @@ func CheckPDVersionWithClusterVersion(opt *config.PersistOptions) { if pdVersion.LessThan(clusterVersion) { log.Warn( "PD version less than cluster version, please upgrade PD", - zap.String("PD-version", pdVersion.String()), + zap.String("pd-version", pdVersion.String()), zap.String("cluster-version", clusterVersion.String())) } } From 0bfa31f9697dd7ce5a495f235eeb9eedaca67921 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 2 Jan 2025 15:23:21 +0800 Subject: [PATCH 04/33] cluster: fix panic when minResolvedTS is not initialized (#8965) close tikv/pd#8964 Signed-off-by: Ryan Leung --- server/cluster/cluster.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 699b43e7901..d2f3855d14e 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -2283,7 +2283,8 @@ func (c *RaftCluster) CheckAndUpdateMinResolvedTS() (uint64, bool) { newMinResolvedTS = s.GetMinResolvedTS() } } - oldMinResolvedTS := c.minResolvedTS.Load().(uint64) + // Avoid panic when minResolvedTS is not initialized. + oldMinResolvedTS, _ := c.minResolvedTS.Load().(uint64) if newMinResolvedTS == math.MaxUint64 || newMinResolvedTS <= oldMinResolvedTS { return oldMinResolvedTS, false } From 7a30ebc972ffa2f0f76e6f99f9c55a68f975669e Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Thu, 2 Jan 2025 15:35:42 +0800 Subject: [PATCH 05/33] server: advance ServerStart check (#8951) close tikv/pd#8950 Signed-off-by: okJiang <819421878@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/clients/tso/client.go | 3 +++ client/go.mod | 2 +- client/go.sum | 8 ++------ go.mod | 2 +- go.sum | 5 ++--- server/grpc_service.go | 15 ++++++++++---- server/server.go | 1 + tests/integrations/client/client_test.go | 26 ++++++++++++++++++++++++ tests/integrations/go.mod | 2 +- tests/integrations/go.sum | 7 ++----- tools/go.mod | 2 +- tools/go.sum | 7 ++----- 12 files changed, 53 insertions(+), 27 deletions(-) diff --git a/client/clients/tso/client.go b/client/clients/tso/client.go index c6caa8b985f..d24dba52394 100644 --- a/client/clients/tso/client.go +++ b/client/clients/tso/client.go @@ -281,6 +281,9 @@ func (c *Cli) connectionCtxsUpdater() { // Because the TSO Follower Proxy is enabled, // the periodic check needs to be performed. setNewUpdateTicker(sd.MemberUpdateInterval) + failpoint.Inject("speedUpTsoDispatcherUpdateInterval", func() { + setNewUpdateTicker(10 * time.Millisecond) + }) } else if !enableTSOFollowerProxy && updateTicker.C != nil { // Because the TSO Follower Proxy is disabled, // the periodic check needs to be turned off. diff --git a/client/go.mod b/client/go.mod index b26abcbea55..78aef084ff7 100644 --- a/client/go.mod +++ b/client/go.mod @@ -9,7 +9,7 @@ require ( github.com/gogo/protobuf v1.3.2 github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c - github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 + github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 diff --git a/client/go.sum b/client/go.sum index 36c58efb823..4cca5ba3ad5 100644 --- a/client/go.sum +++ b/client/go.sum @@ -45,11 +45,10 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= -github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= +github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 h1:xYNSJjYNur4Dr5bV+9BXK9n5E0T1zlcAN25XX68+mOg= github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= @@ -69,7 +68,6 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= @@ -158,7 +156,6 @@ google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6h google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= @@ -167,7 +164,6 @@ gopkg.in/natefinch/lumberjack.v2 v2.2.1/go.mod h1:YD8tP3GAjkrDg1eZH7EGmyESg/lsYs gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/go.mod b/go.mod index 9c8a7bc90a5..c1107b6ffc5 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c - github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 + github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 diff --git a/go.sum b/go.sum index a2f070397d1..e6156e9ecf7 100644 --- a/go.sum +++ b/go.sum @@ -386,12 +386,11 @@ github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12/go.mod h1:PYMCGwN0JH github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg= github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= -github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= +github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 h1:xYNSJjYNur4Dr5bV+9BXK9n5E0T1zlcAN25XX68+mOg= github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= diff --git a/server/grpc_service.go b/server/grpc_service.go index 8db79d3b8f5..d3fc5c58d7f 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -325,6 +325,9 @@ func (s *GrpcServer) GetMinTS( // GetMinTSFromTSOService queries all tso servers and gets the minimum timestamp across // all keyspace groups. func (s *GrpcServer) GetMinTSFromTSOService() (*pdpb.Timestamp, error) { + if s.IsClosed() { + return nil, errs.ErrNotStarted + } addrs := s.keyspaceGroupManager.GetTSOServiceAddrs() if len(addrs) == 0 { return &pdpb.Timestamp{}, errs.ErrGetMinTS.FastGenByArgs("no tso servers/pods discovered") @@ -536,6 +539,11 @@ func (s *GrpcServer) Tso(stream pdpb.PD_TsoServer) error { return errors.WithStack(err) } + // TSO uses leader lease to determine validity. No need to check leader here. + if s.IsClosed() { + return errs.ErrNotStarted + } + forwardedHost := grpcutil.GetForwardedHost(stream.Context()) if !s.isLocalRequest(forwardedHost) { clientConn, err := s.getDelegateClient(s.ctx, forwardedHost) @@ -570,10 +578,6 @@ func (s *GrpcServer) Tso(stream pdpb.PD_TsoServer) error { } start := time.Now() - // TSO uses leader lease to determine validity. No need to check leader here. - if s.IsClosed() { - return errs.ErrNotStarted - } if clusterID := keypath.ClusterID(); request.GetHeader().GetClusterId() != clusterID { return errs.ErrMismatchClusterID(clusterID, request.GetHeader().GetClusterId()) } @@ -710,6 +714,9 @@ func (s *GrpcServer) IsSnapshotRecovering(ctx context.Context, _ *pdpb.IsSnapsho return nil, errs.ErrGRPCRateLimitExceeded(err) } } + if s.IsClosed() { + return nil, errs.ErrNotStarted + } // recovering mark is stored in etcd directly, there's no need to forward. marked, err := s.Server.IsSnapshotRecovering(ctx) if err != nil { diff --git a/server/server.go b/server/server.go index 4cae86b6587..94250128fe3 100644 --- a/server/server.go +++ b/server/server.go @@ -506,6 +506,7 @@ func (s *Server) startServer(ctx context.Context) error { s.grpcServiceRateLimiter.Update(service, ratelimit.InitLimiter()) } + failpoint.InjectCall("delayStartServer") // Server has started. atomic.StoreInt64(&s.isRunning, 1) bs.ServerMaxProcsGauge.Set(float64(runtime.GOMAXPROCS(0))) diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index fadfb952e4c..2018860130e 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -270,6 +270,7 @@ func TestTSOFollowerProxy(t *testing.T) { defer cli1.Close() cli2 := setupCli(ctx, re, endpoints) defer cli2.Close() + re.NoError(failpoint.Enable("github.com/tikv/pd/client/clients/tso/speedUpTsoDispatcherUpdateInterval", "return(true)")) err = cli2.UpdateOption(opt.EnableTSOFollowerProxy, true) re.NoError(err) @@ -296,6 +297,31 @@ func TestTSOFollowerProxy(t *testing.T) { } wg.Wait() + followerServer := cluster.GetServer(cluster.GetFollower()) + re.NoError(followerServer.Stop()) + ch := make(chan struct{}) + re.NoError(failpoint.EnableCall("github.com/tikv/pd/server/delayStartServer", func() { + // Server is not in `Running` state, so the follower proxy should return + // error while create stream. + ch <- struct{}{} + })) + wg.Add(1) + go func() { + defer wg.Done() + re.NoError(followerServer.Run()) + }() + re.Eventually(func() bool { + _, _, err := cli2.GetTS(context.Background()) + if err == nil { + return false + } + return strings.Contains(err.Error(), "server not started") + }, 3*time.Second, 10*time.Millisecond) + <-ch + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServer")) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/clients/tso/speedUpTsoDispatcherUpdateInterval")) + wg.Wait() + // Disable the follower proxy and check if the stream is updated. err = cli2.UpdateOption(opt.EnableTSOFollowerProxy, false) re.NoError(err) diff --git a/tests/integrations/go.mod b/tests/integrations/go.mod index c5fcb617014..ec1d74923d6 100644 --- a/tests/integrations/go.mod +++ b/tests/integrations/go.mod @@ -13,7 +13,7 @@ require ( github.com/docker/go-units v0.5.0 github.com/go-sql-driver/mysql v1.7.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c - github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c + github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 diff --git a/tests/integrations/go.sum b/tests/integrations/go.sum index 5a48549ef65..5f78324c3c2 100644 --- a/tests/integrations/go.sum +++ b/tests/integrations/go.sum @@ -379,12 +379,11 @@ github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12/go.mod h1:PYMCGwN0JH github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg= github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c h1:CgbKAHto5CQgWM9fSBIvaxsJHuGP0uM74HXtv3MyyGQ= -github.com/pingcap/failpoint v0.0.0-20220801062533-2eaa32854a6c/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= +github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 h1:xYNSJjYNur4Dr5bV+9BXK9n5E0T1zlcAN25XX68+mOg= github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= @@ -429,7 +428,6 @@ github.com/samber/lo v1.37.0 h1:XjVcB8g6tgUp8rsPsJ2CvhClfImrpL04YpQHXeHPhRw= github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpopA= github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb2NsU= github.com/sasha-s/go-deadlock v0.3.5/go.mod h1:bugP6EGbdGYObIlx7pUZtWqlvo8k9H6vCBBsiChJQ5U= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA= github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE= github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU= @@ -748,7 +746,6 @@ google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6h google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= diff --git a/tools/go.mod b/tools/go.mod index ded2e2e82c8..31309986d92 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -21,7 +21,7 @@ require ( github.com/influxdata/tdigest v0.0.1 github.com/mattn/go-shellwords v1.0.12 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c - github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 + github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 diff --git a/tools/go.sum b/tools/go.sum index bd68a4f0ca1..b9c49d466ef 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -380,12 +380,11 @@ github.com/pingcap/check v0.0.0-20191216031241-8a5a85928f12/go.mod h1:PYMCGwN0JH github.com/pingcap/errcode v0.3.0 h1:IF6LC/4+b1KNwrMlr2rBTUrojFPMexXBcDWZSpNwxjg= github.com/pingcap/errcode v0.3.0/go.mod h1:4b2X8xSqxIroj/IZ9MX/VGZhAwc11wB9wRIzHvz6SeM= github.com/pingcap/errors v0.11.0/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= -github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20190809092503-95897b64e011/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTmyFqUwr+jcCvpVkK7sumiz+ko5H9eq4= github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= -github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00 h1:C3N3itkduZXDZFh4N3vQ5HEtld3S+Y+StULhWVvumU0= -github.com/pingcap/failpoint v0.0.0-20210918120811-547c13e3eb00/go.mod h1:4qGtCB0QK0wBzKtFEGDhxXnSnbQApw1gc9siScUl8ew= +github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= +github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 h1:xYNSJjYNur4Dr5bV+9BXK9n5E0T1zlcAN25XX68+mOg= github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= @@ -432,7 +431,6 @@ github.com/samber/lo v1.37.0 h1:XjVcB8g6tgUp8rsPsJ2CvhClfImrpL04YpQHXeHPhRw= github.com/samber/lo v1.37.0/go.mod h1:9vaz2O4o8oOnK23pd2TrXufcbdbJIa3b6cstBWKpopA= github.com/sasha-s/go-deadlock v0.3.5 h1:tNCOEEDG6tBqrNDOX35j/7hL5FcFViG6awUGROb2NsU= github.com/sasha-s/go-deadlock v0.3.5/go.mod h1:bugP6EGbdGYObIlx7pUZtWqlvo8k9H6vCBBsiChJQ5U= -github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shirou/gopsutil/v3 v3.21.12/go.mod h1:BToYZVTlSVlfazpDDYFnsVZLaoRG+g8ufT6fPQLdJzA= github.com/shirou/gopsutil/v3 v3.23.3 h1:Syt5vVZXUDXPEXpIBt5ziWsJ4LdSAAxF4l/xZeQgSEE= github.com/shirou/gopsutil/v3 v3.23.3/go.mod h1:lSBNN6t3+D6W5e5nXTxc8KIMMVxAcS+6IJlffjRRlMU= @@ -759,7 +757,6 @@ google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6h google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= From 41919ad57acca17c6e2c24a19a1c0185c355e469 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Thu, 2 Jan 2025 16:46:51 +0800 Subject: [PATCH 06/33] test: make TestPreparingProgress stable (#8966) close tikv/pd#8693 Signed-off-by: okJiang <819421878@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/server/api/api_test.go | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index faa22ce08f4..44a0ae69a46 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -1092,13 +1092,24 @@ func TestPreparingProgress(t *testing.T) { re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) } -func sendRequest(re *require.Assertions, url string, method string, statusCode int) []byte { +func sendRequest(re *require.Assertions, url string, method string, statusCode int) (output []byte) { req, _ := http.NewRequest(method, url, http.NoBody) - resp, err := tests.TestDialClient.Do(req) - re.NoError(err) - re.Equal(statusCode, resp.StatusCode) - output, err := io.ReadAll(resp.Body) - re.NoError(err) - resp.Body.Close() + + testutil.Eventually(re, func() bool { + resp, err := tests.TestDialClient.Do(req) + re.NoError(err) + defer resp.Body.Close() + + // Due to service unavailability caused by environmental issues, + // we will retry it. + if resp.StatusCode == http.StatusServiceUnavailable { + return false + } + re.Equal(statusCode, resp.StatusCode) + output, err = io.ReadAll(resp.Body) + re.NoError(err) + return true + }) + return output } From 6a0ed869ce3d233a02c3c6ecff642546dca5d324 Mon Sep 17 00:00:00 2001 From: okJiang <819421878@qq.com> Date: Mon, 6 Jan 2025 13:53:22 +0800 Subject: [PATCH 07/33] client: retry checkServiceModeChanged to make tso client work normal (#8963) close tikv/pd#8962 Signed-off-by: okJiang <819421878@qq.com> Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/servicediscovery/pd_service_discovery.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/client/servicediscovery/pd_service_discovery.go b/client/servicediscovery/pd_service_discovery.go index 5530f3cfa9b..931f950c6d1 100644 --- a/client/servicediscovery/pd_service_discovery.go +++ b/client/servicediscovery/pd_service_discovery.go @@ -509,8 +509,9 @@ func (c *pdServiceDiscovery) Init() error { } } - if err := c.checkServiceModeChanged(); err != nil { - log.Warn("[pd] failed to check service mode and will check later", zap.Error(err)) + if err := c.initRetry(c.checkServiceModeChanged); err != nil { + c.cancel() + return err } c.wg.Add(3) From 5c4ab57d68dea7c82b511c06f8ccfa1e075889b2 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 7 Jan 2025 11:26:58 +0800 Subject: [PATCH 08/33] client: use interceptor for circuit breaker (#8936) ref tikv/pd#8678 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 28 ++------- client/inner_client.go | 22 ++----- client/opt/option.go | 27 ++------ client/pkg/circuitbreaker/circuit_breaker.go | 60 ++++++++++++------ .../circuitbreaker/circuit_breaker_test.go | 63 +++++++++---------- client/pkg/utils/grpcutil/grpcutil.go | 40 +++++++++++- tests/integrations/client/client_test.go | 63 ++++++++++--------- 7 files changed, 156 insertions(+), 147 deletions(-) diff --git a/client/client.go b/client/client.go index 2fc9bd3ef0d..7aa28cbc0cd 100644 --- a/client/client.go +++ b/client/client.go @@ -25,8 +25,6 @@ import ( "github.com/opentracing/opentracing-go" "github.com/prometheus/client_golang/prometheus" "go.uber.org/zap" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "github.com/pingcap/errors" "github.com/pingcap/failpoint" @@ -42,7 +40,6 @@ import ( "github.com/tikv/pd/client/metrics" "github.com/tikv/pd/client/opt" "github.com/tikv/pd/client/pkg/caller" - cb "github.com/tikv/pd/client/pkg/circuitbreaker" "github.com/tikv/pd/client/pkg/utils/tlsutil" sd "github.com/tikv/pd/client/servicediscovery" ) @@ -460,12 +457,6 @@ func (c *client) UpdateOption(option opt.DynamicOption, value any) error { return errors.New("[pd] invalid value type for TSOClientRPCConcurrency option, it should be int") } c.inner.option.SetTSOClientRPCConcurrency(value) - case opt.RegionMetadataCircuitBreakerSettings: - applySettingsChange, ok := value.(func(config *cb.Settings)) - if !ok { - return errors.New("[pd] invalid value type for RegionMetadataCircuitBreakerSettings option, it should be pd.Settings") - } - c.inner.regionMetaCircuitBreaker.ChangeSettings(applySettingsChange) default: return errors.New("[pd] unsupported client option") } @@ -660,13 +651,7 @@ func (c *client) GetRegion(ctx context.Context, key []byte, opts ...opt.GetRegio if serviceClient == nil { return nil, errs.ErrClientGetProtoClient } - resp, err := c.inner.regionMetaCircuitBreaker.Execute(func() (*pdpb.GetRegionResponse, cb.Overloading, error) { - region, err := pdpb.NewPDClient(serviceClient.GetClientConn()).GetRegion(cctx, req) - failpoint.Inject("triggerCircuitBreaker", func() { - err = status.Error(codes.ResourceExhausted, "resource exhausted") - }) - return region, isOverloaded(err), err - }) + resp, err := pdpb.NewPDClient(serviceClient.GetClientConn()).GetRegion(cctx, req) if serviceClient.NeedRetry(resp.GetHeader().GetError(), err) { protoClient, cctx := c.getClientAndContext(ctx) if protoClient == nil { @@ -706,10 +691,7 @@ func (c *client) GetPrevRegion(ctx context.Context, key []byte, opts ...opt.GetR if serviceClient == nil { return nil, errs.ErrClientGetProtoClient } - resp, err := c.inner.regionMetaCircuitBreaker.Execute(func() (*pdpb.GetRegionResponse, cb.Overloading, error) { - resp, err := pdpb.NewPDClient(serviceClient.GetClientConn()).GetPrevRegion(cctx, req) - return resp, isOverloaded(err), err - }) + resp, err := pdpb.NewPDClient(serviceClient.GetClientConn()).GetPrevRegion(cctx, req) if serviceClient.NeedRetry(resp.GetHeader().GetError(), err) { protoClient, cctx := c.getClientAndContext(ctx) if protoClient == nil { @@ -749,10 +731,8 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64, opts ...opt if serviceClient == nil { return nil, errs.ErrClientGetProtoClient } - resp, err := c.inner.regionMetaCircuitBreaker.Execute(func() (*pdpb.GetRegionResponse, cb.Overloading, error) { - resp, err := pdpb.NewPDClient(serviceClient.GetClientConn()).GetRegionByID(cctx, req) - return resp, isOverloaded(err), err - }) + + resp, err := pdpb.NewPDClient(serviceClient.GetClientConn()).GetRegionByID(cctx, req) if serviceClient.NeedRetry(resp.GetHeader().GetError(), err) { protoClient, cctx := c.getClientAndContext(ctx) if protoClient == nil { diff --git a/client/inner_client.go b/client/inner_client.go index 91f999dd3b5..404cbcf0b80 100644 --- a/client/inner_client.go +++ b/client/inner_client.go @@ -8,8 +8,6 @@ import ( "go.uber.org/zap" "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/status" "github.com/pingcap/errors" "github.com/pingcap/kvproto/pkg/pdpb" @@ -19,7 +17,6 @@ import ( "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/metrics" "github.com/tikv/pd/client/opt" - cb "github.com/tikv/pd/client/pkg/circuitbreaker" sd "github.com/tikv/pd/client/servicediscovery" ) @@ -29,11 +26,10 @@ const ( ) type innerClient struct { - keyspaceID uint32 - svrUrls []string - pdSvcDiscovery sd.ServiceDiscovery - tokenDispatcher *tokenDispatcher - regionMetaCircuitBreaker *cb.CircuitBreaker[*pdpb.GetRegionResponse] + keyspaceID uint32 + svrUrls []string + pdSvcDiscovery sd.ServiceDiscovery + tokenDispatcher *tokenDispatcher // For service mode switching. serviceModeKeeper @@ -59,7 +55,6 @@ func (c *innerClient) init(updateKeyspaceIDCb sd.UpdateKeyspaceIDFunc) error { } return err } - c.regionMetaCircuitBreaker = cb.NewCircuitBreaker[*pdpb.GetRegionResponse]("region_meta", c.option.RegionMetaCircuitBreakerSettings) return nil } @@ -252,12 +247,3 @@ func (c *innerClient) dispatchTSORequestWithRetry(ctx context.Context) tso.TSFut } return req } - -func isOverloaded(err error) cb.Overloading { - switch status.Code(errors.Cause(err)) { - case codes.DeadlineExceeded, codes.Unavailable, codes.ResourceExhausted: - return cb.Yes - default: - return cb.No - } -} diff --git a/client/opt/option.go b/client/opt/option.go index af95a225fab..2aa9be8ae7f 100644 --- a/client/opt/option.go +++ b/client/opt/option.go @@ -23,7 +23,6 @@ import ( "github.com/pingcap/errors" - cb "github.com/tikv/pd/client/pkg/circuitbreaker" "github.com/tikv/pd/client/pkg/retry" ) @@ -50,8 +49,6 @@ const ( EnableFollowerHandle // TSOClientRPCConcurrency controls the amount of ongoing TSO RPC requests at the same time in a single TSO client. TSOClientRPCConcurrency - // RegionMetadataCircuitBreakerSettings controls settings for circuit breaker for region metadata requests. - RegionMetadataCircuitBreakerSettings dynamicOptionCount ) @@ -72,18 +69,16 @@ type Option struct { // Dynamic options. dynamicOptions [dynamicOptionCount]atomic.Value - EnableTSOFollowerProxyCh chan struct{} - RegionMetaCircuitBreakerSettings cb.Settings + EnableTSOFollowerProxyCh chan struct{} } // NewOption creates a new PD client option with the default values set. func NewOption() *Option { co := &Option{ - Timeout: defaultPDTimeout, - MaxRetryTimes: maxInitClusterRetries, - EnableTSOFollowerProxyCh: make(chan struct{}, 1), - InitMetrics: true, - RegionMetaCircuitBreakerSettings: cb.AlwaysClosedSettings, + Timeout: defaultPDTimeout, + MaxRetryTimes: maxInitClusterRetries, + EnableTSOFollowerProxyCh: make(chan struct{}, 1), + InitMetrics: true, } co.dynamicOptions[MaxTSOBatchWaitInterval].Store(defaultMaxTSOBatchWaitInterval) @@ -154,11 +149,6 @@ func (o *Option) GetTSOClientRPCConcurrency() int { return o.dynamicOptions[TSOClientRPCConcurrency].Load().(int) } -// GetRegionMetadataCircuitBreakerSettings gets circuit breaker settings for PD region metadata calls. -func (o *Option) GetRegionMetadataCircuitBreakerSettings() cb.Settings { - return o.dynamicOptions[RegionMetadataCircuitBreakerSettings].Load().(cb.Settings) -} - // ClientOption configures client. type ClientOption func(*Option) @@ -213,13 +203,6 @@ func WithInitMetricsOption(initMetrics bool) ClientOption { } } -// WithRegionMetaCircuitBreaker configures the client with circuit breaker for region meta calls -func WithRegionMetaCircuitBreaker(config cb.Settings) ClientOption { - return func(op *Option) { - op.RegionMetaCircuitBreakerSettings = config - } -} - // WithBackoffer configures the client with backoffer. func WithBackoffer(bo *retry.Backoffer) ClientOption { return func(op *Option) { diff --git a/client/pkg/circuitbreaker/circuit_breaker.go b/client/pkg/circuitbreaker/circuit_breaker.go index 2c65f4f1965..0acee5d5c8d 100644 --- a/client/pkg/circuitbreaker/circuit_breaker.go +++ b/client/pkg/circuitbreaker/circuit_breaker.go @@ -14,6 +14,7 @@ package circuitbreaker import ( + "context" "fmt" "strings" "sync" @@ -62,12 +63,12 @@ var AlwaysClosedSettings = Settings{ } // CircuitBreaker is a state machine to prevent sending requests that are likely to fail. -type CircuitBreaker[T any] struct { +type CircuitBreaker struct { config *Settings name string mutex sync.Mutex - state *State[T] + state *State successCounter prometheus.Counter errorCounter prometheus.Counter @@ -102,8 +103,8 @@ func (s StateType) String() string { var replacer = strings.NewReplacer(" ", "_", "-", "_") // NewCircuitBreaker returns a new CircuitBreaker configured with the given Settings. -func NewCircuitBreaker[T any](name string, st Settings) *CircuitBreaker[T] { - cb := new(CircuitBreaker[T]) +func NewCircuitBreaker(name string, st Settings) *CircuitBreaker { + cb := new(CircuitBreaker) cb.name = name cb.config = &st cb.state = cb.newState(time.Now(), StateClosed) @@ -118,7 +119,7 @@ func NewCircuitBreaker[T any](name string, st Settings) *CircuitBreaker[T] { // ChangeSettings changes the CircuitBreaker settings. // The changes will be reflected only in the next evaluation window. -func (cb *CircuitBreaker[T]) ChangeSettings(apply func(config *Settings)) { +func (cb *CircuitBreaker) ChangeSettings(apply func(config *Settings)) { cb.mutex.Lock() defer cb.mutex.Unlock() @@ -129,12 +130,11 @@ func (cb *CircuitBreaker[T]) ChangeSettings(apply func(config *Settings)) { // Execute calls the given function if the CircuitBreaker is closed and returns the result of execution. // Execute returns an error instantly if the CircuitBreaker is open. // https://github.com/tikv/rfcs/blob/master/text/0115-circuit-breaker.md -func (cb *CircuitBreaker[T]) Execute(call func() (T, Overloading, error)) (T, error) { +func (cb *CircuitBreaker) Execute(call func() (Overloading, error)) error { state, err := cb.onRequest() if err != nil { cb.fastFailCounter.Inc() - var defaultValue T - return defaultValue, err + return err } defer func() { @@ -146,13 +146,13 @@ func (cb *CircuitBreaker[T]) Execute(call func() (T, Overloading, error)) (T, er } }() - result, overloaded, err := call() + overloaded, err := call() cb.emitMetric(overloaded, err) cb.onResult(state, overloaded) - return result, err + return err } -func (cb *CircuitBreaker[T]) onRequest() (*State[T], error) { +func (cb *CircuitBreaker) onRequest() (*State, error) { cb.mutex.Lock() defer cb.mutex.Unlock() @@ -161,7 +161,7 @@ func (cb *CircuitBreaker[T]) onRequest() (*State[T], error) { return state, err } -func (cb *CircuitBreaker[T]) onResult(state *State[T], overloaded Overloading) { +func (cb *CircuitBreaker) onResult(state *State, overloaded Overloading) { cb.mutex.Lock() defer cb.mutex.Unlock() @@ -170,7 +170,7 @@ func (cb *CircuitBreaker[T]) onResult(state *State[T], overloaded Overloading) { state.onResult(overloaded) } -func (cb *CircuitBreaker[T]) emitMetric(overloaded Overloading, err error) { +func (cb *CircuitBreaker) emitMetric(overloaded Overloading, err error) { switch overloaded { case No: cb.successCounter.Inc() @@ -185,9 +185,9 @@ func (cb *CircuitBreaker[T]) emitMetric(overloaded Overloading, err error) { } // State represents the state of CircuitBreaker. -type State[T any] struct { +type State struct { stateType StateType - cb *CircuitBreaker[T] + cb *CircuitBreaker end time.Time pendingCount uint32 @@ -196,7 +196,7 @@ type State[T any] struct { } // newState creates a new State with the given configuration and reset all success/failure counters. -func (cb *CircuitBreaker[T]) newState(now time.Time, stateType StateType) *State[T] { +func (cb *CircuitBreaker) newState(now time.Time, stateType StateType) *State { var end time.Time var pendingCount uint32 switch stateType { @@ -211,7 +211,7 @@ func (cb *CircuitBreaker[T]) newState(now time.Time, stateType StateType) *State default: panic("unknown state") } - return &State[T]{ + return &State{ cb: cb, stateType: stateType, pendingCount: pendingCount, @@ -227,7 +227,7 @@ func (cb *CircuitBreaker[T]) newState(now time.Time, stateType StateType) *State // Open state fails all request, it has a fixed duration of `Settings.CoolDownInterval` and always moves to HalfOpen state at the end of the interval. // HalfOpen state does not have a fixed duration and lasts till `Settings.HalfOpenSuccessCount` are evaluated. // If any of `Settings.HalfOpenSuccessCount` fails then it moves back to Open state, otherwise it moves to Closed state. -func (s *State[T]) onRequest(cb *CircuitBreaker[T]) (*State[T], error) { +func (s *State) onRequest(cb *CircuitBreaker) (*State, error) { var now = time.Now() switch s.stateType { case StateClosed: @@ -299,7 +299,7 @@ func (s *State[T]) onRequest(cb *CircuitBreaker[T]) (*State[T], error) { } } -func (s *State[T]) onResult(overloaded Overloading) { +func (s *State) onResult(overloaded Overloading) { switch overloaded { case No: s.successCount++ @@ -309,3 +309,25 @@ func (s *State[T]) onResult(overloaded Overloading) { panic("unknown state") } } + +// Define context key type +type cbCtxKey struct{} + +// Key used to store circuit breaker +var CircuitBreakerKey = cbCtxKey{} + +// FromContext retrieves the circuit breaker from the context +func FromContext(ctx context.Context) *CircuitBreaker { + if ctx == nil { + return nil + } + if cb, ok := ctx.Value(CircuitBreakerKey).(*CircuitBreaker); ok { + return cb + } + return nil +} + +// WithCircuitBreaker stores the circuit breaker into a new context +func WithCircuitBreaker(ctx context.Context, cb *CircuitBreaker) context.Context { + return context.WithValue(ctx, CircuitBreakerKey, cb) +} diff --git a/client/pkg/circuitbreaker/circuit_breaker_test.go b/client/pkg/circuitbreaker/circuit_breaker_test.go index 07a3c06f86e..e62e55c1ab8 100644 --- a/client/pkg/circuitbreaker/circuit_breaker_test.go +++ b/client/pkg/circuitbreaker/circuit_breaker_test.go @@ -24,7 +24,7 @@ import ( ) // advance emulate the state machine clock moves forward by the given duration -func (cb *CircuitBreaker[T]) advance(duration time.Duration) { +func (cb *CircuitBreaker) advance(duration time.Duration) { cb.state.end = cb.state.end.Add(-duration - 1) } @@ -40,26 +40,24 @@ var minCountToOpen = int(settings.MinQPSForOpen * uint32(settings.ErrorRateWindo func TestCircuitBreakerExecuteWrapperReturnValues(t *testing.T) { re := require.New(t) - cb := NewCircuitBreaker[int]("test_cb", settings) + cb := NewCircuitBreaker("test_cb", settings) originalError := errors.New("circuit breaker is open") - result, err := cb.Execute(func() (int, Overloading, error) { - return 42, No, originalError + err := cb.Execute(func() (Overloading, error) { + return No, originalError }) re.Equal(err, originalError) - re.Equal(42, result) // same by interpret the result as overloading error - result, err = cb.Execute(func() (int, Overloading, error) { - return 42, Yes, originalError + err = cb.Execute(func() (Overloading, error) { + return Yes, originalError }) re.Equal(err, originalError) - re.Equal(42, result) } func TestCircuitBreakerOpenState(t *testing.T) { re := require.New(t) - cb := NewCircuitBreaker[int]("test_cb", settings) + cb := NewCircuitBreaker("test_cb", settings) driveQPS(cb, minCountToOpen, Yes, re) re.Equal(StateClosed, cb.state.stateType) assertSucceeds(cb, re) // no error till ErrorRateWindow is finished @@ -70,7 +68,7 @@ func TestCircuitBreakerOpenState(t *testing.T) { func TestCircuitBreakerCloseStateNotEnoughQPS(t *testing.T) { re := require.New(t) - cb := NewCircuitBreaker[int]("test_cb", settings) + cb := NewCircuitBreaker("test_cb", settings) re.Equal(StateClosed, cb.state.stateType) driveQPS(cb, minCountToOpen/2, Yes, re) cb.advance(settings.ErrorRateWindow) @@ -80,7 +78,7 @@ func TestCircuitBreakerCloseStateNotEnoughQPS(t *testing.T) { func TestCircuitBreakerCloseStateNotEnoughErrorRate(t *testing.T) { re := require.New(t) - cb := NewCircuitBreaker[int]("test_cb", settings) + cb := NewCircuitBreaker("test_cb", settings) re.Equal(StateClosed, cb.state.stateType) driveQPS(cb, minCountToOpen/4, Yes, re) driveQPS(cb, minCountToOpen, No, re) @@ -91,7 +89,7 @@ func TestCircuitBreakerCloseStateNotEnoughErrorRate(t *testing.T) { func TestCircuitBreakerHalfOpenToClosed(t *testing.T) { re := require.New(t) - cb := NewCircuitBreaker[int]("test_cb", settings) + cb := NewCircuitBreaker("test_cb", settings) re.Equal(StateClosed, cb.state.stateType) driveQPS(cb, minCountToOpen, Yes, re) cb.advance(settings.ErrorRateWindow) @@ -109,7 +107,7 @@ func TestCircuitBreakerHalfOpenToClosed(t *testing.T) { func TestCircuitBreakerHalfOpenToOpen(t *testing.T) { re := require.New(t) - cb := NewCircuitBreaker[int]("test_cb", settings) + cb := NewCircuitBreaker("test_cb", settings) re.Equal(StateClosed, cb.state.stateType) driveQPS(cb, minCountToOpen, Yes, re) cb.advance(settings.ErrorRateWindow) @@ -118,8 +116,8 @@ func TestCircuitBreakerHalfOpenToOpen(t *testing.T) { cb.advance(settings.CoolDownInterval) assertSucceeds(cb, re) re.Equal(StateHalfOpen, cb.state.stateType) - _, err := cb.Execute(func() (int, Overloading, error) { - return 42, Yes, nil // this trip circuit breaker again + err := cb.Execute(func() (Overloading, error) { + return Yes, nil // this trip circuit breaker again }) re.NoError(err) re.Equal(StateHalfOpen, cb.state.stateType) @@ -149,10 +147,10 @@ func TestCircuitBreakerHalfOpenFailOverPendingCount(t *testing.T) { defer func() { end <- true }() - _, err := cb.Execute(func() (int, Overloading, error) { + err := cb.Execute(func() (Overloading, error) { start <- true <-wait - return 42, No, nil + return No, nil }) re.NoError(err) }() @@ -178,7 +176,7 @@ func TestCircuitBreakerHalfOpenFailOverPendingCount(t *testing.T) { func TestCircuitBreakerCountOnlyRequestsInSameWindow(t *testing.T) { re := require.New(t) - cb := NewCircuitBreaker[int]("test_cb", settings) + cb := NewCircuitBreaker("test_cb", settings) re.Equal(StateClosed, cb.state.stateType) start := make(chan bool) @@ -188,10 +186,10 @@ func TestCircuitBreakerCountOnlyRequestsInSameWindow(t *testing.T) { defer func() { end <- true }() - _, err := cb.Execute(func() (int, Overloading, error) { + err := cb.Execute(func() (Overloading, error) { start <- true <-wait - return 42, No, nil + return No, nil }) re.NoError(err) }() @@ -214,7 +212,7 @@ func TestCircuitBreakerCountOnlyRequestsInSameWindow(t *testing.T) { func TestCircuitBreakerChangeSettings(t *testing.T) { re := require.New(t) - cb := NewCircuitBreaker[int]("test_cb", AlwaysClosedSettings) + cb := NewCircuitBreaker("test_cb", AlwaysClosedSettings) driveQPS(cb, int(AlwaysClosedSettings.MinQPSForOpen*uint32(AlwaysClosedSettings.ErrorRateWindow.Seconds())), Yes, re) cb.advance(AlwaysClosedSettings.ErrorRateWindow) assertSucceeds(cb, re) @@ -231,8 +229,8 @@ func TestCircuitBreakerChangeSettings(t *testing.T) { re.Equal(StateOpen, cb.state.stateType) } -func newCircuitBreakerMovedToHalfOpenState(re *require.Assertions) *CircuitBreaker[int] { - cb := NewCircuitBreaker[int]("test_cb", settings) +func newCircuitBreakerMovedToHalfOpenState(re *require.Assertions) *CircuitBreaker { + cb := NewCircuitBreaker("test_cb", settings) re.Equal(StateClosed, cb.state.stateType) driveQPS(cb, minCountToOpen, Yes, re) cb.advance(settings.ErrorRateWindow) @@ -242,29 +240,28 @@ func newCircuitBreakerMovedToHalfOpenState(re *require.Assertions) *CircuitBreak return cb } -func driveQPS(cb *CircuitBreaker[int], count int, overload Overloading, re *require.Assertions) { +func driveQPS(cb *CircuitBreaker, count int, overload Overloading, re *require.Assertions) { for range count { - _, err := cb.Execute(func() (int, Overloading, error) { - return 42, overload, nil + err := cb.Execute(func() (Overloading, error) { + return overload, nil }) re.NoError(err) } } -func assertFastFail(cb *CircuitBreaker[int], re *require.Assertions) { +func assertFastFail(cb *CircuitBreaker, re *require.Assertions) { var executed = false - _, err := cb.Execute(func() (int, Overloading, error) { + err := cb.Execute(func() (Overloading, error) { executed = true - return 42, No, nil + return No, nil }) re.Equal(err, errs.ErrCircuitBreakerOpen) re.False(executed) } -func assertSucceeds(cb *CircuitBreaker[int], re *require.Assertions) { - result, err := cb.Execute(func() (int, Overloading, error) { - return 42, No, nil +func assertSucceeds(cb *CircuitBreaker, re *require.Assertions) { + err := cb.Execute(func() (Overloading, error) { + return No, nil }) re.NoError(err) - re.Equal(42, result) } diff --git a/client/pkg/utils/grpcutil/grpcutil.go b/client/pkg/utils/grpcutil/grpcutil.go index b73d117fe84..235e1088747 100644 --- a/client/pkg/utils/grpcutil/grpcutil.go +++ b/client/pkg/utils/grpcutil/grpcutil.go @@ -24,15 +24,18 @@ import ( "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/backoff" + "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" "google.golang.org/grpc/credentials/insecure" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" "github.com/pingcap/errors" "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/client/errs" + "github.com/tikv/pd/client/pkg/circuitbreaker" "github.com/tikv/pd/client/pkg/retry" ) @@ -71,6 +74,36 @@ func UnaryBackofferInterceptor() grpc.UnaryClientInterceptor { } } +// UnaryCircuitBreakerInterceptor is a gRPC interceptor that adds a circuit breaker to the call. +func UnaryCircuitBreakerInterceptor() grpc.UnaryClientInterceptor { + return func(ctx context.Context, method string, req, reply any, cc *grpc.ClientConn, invoker grpc.UnaryInvoker, opts ...grpc.CallOption) error { + cb := circuitbreaker.FromContext(ctx) + if cb == nil { + return invoker(ctx, method, req, reply, cc, opts...) + } + err := cb.Execute(func() (circuitbreaker.Overloading, error) { + err := invoker(ctx, method, req, reply, cc, opts...) + failpoint.Inject("triggerCircuitBreaker", func() { + err = status.Error(codes.ResourceExhausted, "resource exhausted") + }) + return isOverloaded(err), err + }) + if err != nil { + return err + } + return nil + } +} + +func isOverloaded(err error) circuitbreaker.Overloading { + switch status.Code(errors.Cause(err)) { + case codes.DeadlineExceeded, codes.Unavailable, codes.ResourceExhausted: + return circuitbreaker.Yes + default: + return circuitbreaker.No + } +} + // GetClientConn returns a gRPC client connection. // creates a client connection to the given target. By default, it's // a non-blocking dial (the function won't wait for connections to be @@ -96,7 +129,10 @@ func GetClientConn(ctx context.Context, addr string, tlsCfg *tls.Config, do ...g } // Add backoffer interceptor - retryOpt := grpc.WithUnaryInterceptor(UnaryBackofferInterceptor()) + retryOpt := grpc.WithChainUnaryInterceptor(UnaryBackofferInterceptor()) + + // Add circuit breaker interceptor + cbOpt := grpc.WithChainUnaryInterceptor(UnaryCircuitBreakerInterceptor()) // Add retry related connection parameters backoffOpts := grpc.WithConnectParams(grpc.ConnectParams{ @@ -108,7 +144,7 @@ func GetClientConn(ctx context.Context, addr string, tlsCfg *tls.Config, do ...g }, }) - do = append(do, opt, retryOpt, backoffOpts) + do = append(do, opt, retryOpt, cbOpt, backoffOpts) cc, err := grpc.DialContext(ctx, u.Host, do...) if err != nil { return nil, errs.ErrGRPCDial.Wrap(err).GenWithStackByCause() diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 2018860130e..397e1079af3 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -2096,28 +2096,30 @@ func TestCircuitBreaker(t *testing.T) { } endpoints := runServer(re, cluster) - cli := setupCli(ctx, re, endpoints, opt.WithRegionMetaCircuitBreaker(circuitBreakerSettings)) + cli := setupCli(ctx, re, endpoints) defer cli.Close() + circuitBreaker := cb.NewCircuitBreaker("region_meta", circuitBreakerSettings) + ctx = cb.WithCircuitBreaker(ctx, circuitBreaker) for range 10 { - region, err := cli.GetRegion(context.TODO(), []byte("a")) + region, err := cli.GetRegion(ctx, []byte("a")) re.NoError(err) re.NotNil(region) } - re.NoError(failpoint.Enable("github.com/tikv/pd/client/triggerCircuitBreaker", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/pkg/utils/grpcutil/triggerCircuitBreaker", "return(true)")) for range 100 { - _, err := cli.GetRegion(context.TODO(), []byte("a")) + _, err := cli.GetRegion(ctx, []byte("a")) re.Error(err) } - _, err = cli.GetRegion(context.TODO(), []byte("a")) + _, err = cli.GetRegion(ctx, []byte("a")) re.Error(err) re.Contains(err.Error(), "circuit breaker is open") - re.NoError(failpoint.Disable("github.com/tikv/pd/client/triggerCircuitBreaker")) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/pkg/utils/grpcutil/triggerCircuitBreaker")) - _, err = cli.GetRegion(context.TODO(), []byte("a")) + _, err = cli.GetRegion(ctx, []byte("a")) re.Error(err) re.Contains(err.Error(), "circuit breaker is open") @@ -2125,7 +2127,7 @@ func TestCircuitBreaker(t *testing.T) { time.Sleep(time.Second) for range 10 { - region, err := cli.GetRegion(context.TODO(), []byte("a")) + region, err := cli.GetRegion(ctx, []byte("a")) re.NoError(err) re.NotNil(region) } @@ -2149,34 +2151,35 @@ func TestCircuitBreakerOpenAndChangeSettings(t *testing.T) { } endpoints := runServer(re, cluster) - cli := setupCli(ctx, re, endpoints, opt.WithRegionMetaCircuitBreaker(circuitBreakerSettings)) + cli := setupCli(ctx, re, endpoints) defer cli.Close() + circuitBreaker := cb.NewCircuitBreaker("region_meta", circuitBreakerSettings) + ctx = cb.WithCircuitBreaker(ctx, circuitBreaker) for range 10 { - region, err := cli.GetRegion(context.TODO(), []byte("a")) + region, err := cli.GetRegion(ctx, []byte("a")) re.NoError(err) re.NotNil(region) } - re.NoError(failpoint.Enable("github.com/tikv/pd/client/triggerCircuitBreaker", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/pkg/utils/grpcutil/triggerCircuitBreaker", "return(true)")) for range 100 { - _, err := cli.GetRegion(context.TODO(), []byte("a")) + _, err := cli.GetRegion(ctx, []byte("a")) re.Error(err) } - _, err = cli.GetRegion(context.TODO(), []byte("a")) + _, err = cli.GetRegion(ctx, []byte("a")) re.Error(err) re.Contains(err.Error(), "circuit breaker is open") - cli.UpdateOption(opt.RegionMetadataCircuitBreakerSettings, func(config *cb.Settings) { + circuitBreaker.ChangeSettings(func(config *cb.Settings) { *config = cb.AlwaysClosedSettings }) - - _, err = cli.GetRegion(context.TODO(), []byte("a")) + _, err = cli.GetRegion(ctx, []byte("a")) re.Error(err) re.Contains(err.Error(), "ResourceExhausted") - re.NoError(failpoint.Disable("github.com/tikv/pd/client/triggerCircuitBreaker")) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/pkg/utils/grpcutil/triggerCircuitBreaker")) } func TestCircuitBreakerHalfOpenAndChangeSettings(t *testing.T) { @@ -2197,23 +2200,26 @@ func TestCircuitBreakerHalfOpenAndChangeSettings(t *testing.T) { } endpoints := runServer(re, cluster) - cli := setupCli(ctx, re, endpoints, opt.WithRegionMetaCircuitBreaker(circuitBreakerSettings)) + + cli := setupCli(ctx, re, endpoints) defer cli.Close() + circuitBreaker := cb.NewCircuitBreaker("region_meta", circuitBreakerSettings) + ctx = cb.WithCircuitBreaker(ctx, circuitBreaker) for range 10 { - region, err := cli.GetRegion(context.TODO(), []byte("a")) + region, err := cli.GetRegion(ctx, []byte("a")) re.NoError(err) re.NotNil(region) } - re.NoError(failpoint.Enable("github.com/tikv/pd/client/triggerCircuitBreaker", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/pkg/utils/grpcutil/triggerCircuitBreaker", "return(true)")) for range 100 { - _, err := cli.GetRegion(context.TODO(), []byte("a")) + _, err := cli.GetRegion(ctx, []byte("a")) re.Error(err) } - _, err = cli.GetRegion(context.TODO(), []byte("a")) + _, err = cli.GetRegion(ctx, []byte("a")) re.Error(err) re.Contains(err.Error(), "circuit breaker is open") @@ -2221,9 +2227,9 @@ func TestCircuitBreakerHalfOpenAndChangeSettings(t *testing.T) { defer os.RemoveAll(fname) // wait for cooldown time.Sleep(time.Second) - re.NoError(failpoint.Disable("github.com/tikv/pd/client/triggerCircuitBreaker")) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/pkg/utils/grpcutil/triggerCircuitBreaker")) // trigger circuit breaker state to be half open - _, err = cli.GetRegion(context.TODO(), []byte("a")) + _, err = cli.GetRegion(ctx, []byte("a")) re.NoError(err) testutil.Eventually(re, func() bool { b, _ := os.ReadFile(fname) @@ -2233,17 +2239,16 @@ func TestCircuitBreakerHalfOpenAndChangeSettings(t *testing.T) { }) // The state is half open - re.NoError(failpoint.Enable("github.com/tikv/pd/client/triggerCircuitBreaker", "return(true)")) + re.NoError(failpoint.Enable("github.com/tikv/pd/client/pkg/utils/grpcutil/triggerCircuitBreaker", "return(true)")) // change settings to always closed - cli.UpdateOption(opt.RegionMetadataCircuitBreakerSettings, func(config *cb.Settings) { + circuitBreaker.ChangeSettings(func(config *cb.Settings) { *config = cb.AlwaysClosedSettings }) - // It won't be changed to open state. for range 100 { - _, err := cli.GetRegion(context.TODO(), []byte("a")) + _, err := cli.GetRegion(ctx, []byte("a")) re.Error(err) re.NotContains(err.Error(), "circuit breaker is open") } - re.NoError(failpoint.Disable("github.com/tikv/pd/client/triggerCircuitBreaker")) + re.NoError(failpoint.Disable("github.com/tikv/pd/client/pkg/utils/grpcutil/triggerCircuitBreaker")) } From 973234dd090b342b04dde39c8b43c5160e67d5a4 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 7 Jan 2025 16:12:28 +0800 Subject: [PATCH 09/33] *: remove api mode concept (#8952) ref tikv/pd#8477 Signed-off-by: Ryan Leung --- client/client.go | 26 +-- client/clients/tso/client.go | 2 +- client/clients/tso/dispatcher_test.go | 2 +- client/http/client.go | 10 +- client/inner_client.go | 36 ++-- client/keyspace_client.go | 8 +- client/meta_storage_client.go | 8 +- client/resource_manager_client.go | 4 +- ...discovery.go => mock_service_discovery.go} | 50 ++--- ...vice_discovery.go => service_discovery.go} | 204 +++++++++--------- ...very_test.go => service_discovery_test.go} | 20 +- .../servicediscovery/tso_service_discovery.go | 26 +-- cmd/pd-server/main.go | 14 +- pkg/mcs/registry/registry.go | 8 +- pkg/mcs/scheduling/server/cluster.go | 18 +- pkg/mcs/scheduling/server/config/config.go | 6 +- pkg/mcs/scheduling/server/config/watcher.go | 4 +- pkg/mcs/scheduling/server/grpc_service.go | 2 +- pkg/mcs/scheduling/server/meta/watcher.go | 4 +- pkg/mcs/scheduling/server/rule/watcher.go | 4 +- pkg/mcs/scheduling/server/server.go | 8 +- pkg/mcs/utils/constant/constant.go | 4 +- pkg/member/election_leader.go | 2 +- pkg/schedule/coordinator.go | 2 +- .../schedulers/scheduler_controller.go | 2 +- pkg/tso/keyspace_group_manager.go | 2 +- pkg/utils/apiutil/serverapi/middleware.go | 4 +- server/api/admin.go | 2 +- server/api/server.go | 4 +- server/apiv2/handlers/micro_service.go | 4 +- server/cluster/cluster.go | 22 +- server/config/config.go | 2 +- server/grpc_service.go | 4 +- server/server.go | 22 +- server/server_test.go | 4 +- tests/cluster.go | 44 ++-- tests/integrations/client/client_test.go | 2 +- .../mcs/discovery/register_test.go | 8 +- .../mcs/keyspace/tso_keyspace_group_test.go | 2 +- tests/integrations/mcs/members/member_test.go | 2 +- tests/integrations/mcs/scheduling/api_test.go | 46 ++-- .../mcs/scheduling/config_test.go | 14 +- .../integrations/mcs/scheduling/meta_test.go | 2 +- .../integrations/mcs/scheduling/rule_test.go | 4 +- .../mcs/scheduling/server_test.go | 38 ++-- tests/integrations/mcs/tso/api_test.go | 8 +- .../mcs/tso/keyspace_group_manager_test.go | 14 +- tests/integrations/mcs/tso/proxy_test.go | 2 +- tests/integrations/mcs/tso/server_test.go | 52 ++--- tests/integrations/tso/client_test.go | 6 +- tests/integrations/tso/consistency_test.go | 2 +- tests/integrations/tso/server_test.go | 2 +- tests/server/api/scheduler_test.go | 2 +- .../apiv2/handlers/tso_keyspace_group_test.go | 2 +- tests/server/server_test.go | 2 +- tests/testutil.go | 28 +-- tools/pd-ctl/pdctl/command/config_command.go | 24 +-- tools/pd-ctl/tests/config/config_test.go | 54 ++--- .../tests/keyspace/keyspace_group_test.go | 14 +- tools/pd-ctl/tests/keyspace/keyspace_test.go | 4 +- tools/pd-simulator/simulator/drive.go | 2 +- 61 files changed, 455 insertions(+), 469 deletions(-) rename client/servicediscovery/{mock_pd_service_discovery.go => mock_service_discovery.go} (57%) rename client/servicediscovery/{pd_service_discovery.go => service_discovery.go} (81%) rename client/servicediscovery/{pd_service_discovery_test.go => service_discovery_test.go} (96%) diff --git a/client/client.go b/client/client.go index 7aa28cbc0cd..e3d3f4e5b14 100644 --- a/client/client.go +++ b/client/client.go @@ -360,8 +360,8 @@ func newClientWithKeyspaceName( c := &client{ callerComponent: adjustCallerComponent(callerComponent), inner: &innerClient{ - // Create a PD service discovery with null keyspace id, then query the real id with the keyspace name, - // finally update the keyspace id to the PD service discovery for the following interactions. + // Create a service discovery with null keyspace id, then query the real id with the keyspace name, + // finally update the keyspace id to the service discovery for the following interactions. keyspaceID: constants.NullKeyspaceID, updateTokenConnectionCh: make(chan struct{}, 1), ctx: clientCtx, @@ -384,7 +384,7 @@ func newClientWithKeyspaceName( } c.inner.keyspaceID = keyspaceMeta.GetId() // c.keyspaceID is the source of truth for keyspace id. - c.inner.pdSvcDiscovery.SetKeyspaceID(c.inner.keyspaceID) + c.inner.serviceDiscovery.SetKeyspaceID(c.inner.keyspaceID) return nil } @@ -412,17 +412,17 @@ func (c *client) ResetTSOClient() { // GetClusterID returns the ClusterID. func (c *client) GetClusterID(context.Context) uint64 { - return c.inner.pdSvcDiscovery.GetClusterID() + return c.inner.serviceDiscovery.GetClusterID() } // GetLeaderURL returns the leader URL. func (c *client) GetLeaderURL() string { - return c.inner.pdSvcDiscovery.GetServingURL() + return c.inner.serviceDiscovery.GetServingURL() } // GetServiceDiscovery returns the client-side service discovery object func (c *client) GetServiceDiscovery() sd.ServiceDiscovery { - return c.inner.pdSvcDiscovery + return c.inner.serviceDiscovery } // UpdateOption updates the client option. @@ -438,7 +438,7 @@ func (c *client) UpdateOption(option opt.DynamicOption, value any) error { } case opt.EnableTSOFollowerProxy: if c.inner.getServiceMode() != pdpb.ServiceMode_PD_SVC_MODE { - return errors.New("[pd] tso follower proxy is only supported in PD service mode") + return errors.New("[pd] tso follower proxy is only supported in PD mode") } enable, ok := value.(bool) if !ok { @@ -485,7 +485,7 @@ func (c *client) GetAllMembers(ctx context.Context) ([]*pdpb.Member, error) { // getClientAndContext returns the leader pd client and the original context. If leader is unhealthy, it returns // follower pd client and the context which holds forward information. func (c *client) getClientAndContext(ctx context.Context) (pdpb.PDClient, context.Context) { - serviceClient := c.inner.pdSvcDiscovery.GetServiceClient() + serviceClient := c.inner.serviceDiscovery.GetServiceClient() if serviceClient == nil || serviceClient.GetClientConn() == nil { return nil, ctx } @@ -526,7 +526,7 @@ func (c *client) GetLocalTS(ctx context.Context, _ string) (physical int64, logi // GetMinTS implements the TSOClient interface. func (c *client) GetMinTS(ctx context.Context) (physical int64, logical int64, err error) { - // Handle compatibility issue in case of PD/API server doesn't support GetMinTS API. + // Handle compatibility issue in case of PD/PD service doesn't support GetMinTS API. serviceMode := c.inner.getServiceMode() switch serviceMode { case pdpb.ServiceMode_UNKNOWN_SVC_MODE: @@ -598,7 +598,7 @@ func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs var resp *pdpb.GetRegionResponse for _, url := range memberURLs { - conn, err := c.inner.pdSvcDiscovery.GetOrCreateGRPCConn(url) + conn, err := c.inner.serviceDiscovery.GetOrCreateGRPCConn(url) if err != nil { log.Error("[pd] can't get grpc connection", zap.String("member-URL", url), errs.ZapError(err)) continue @@ -619,7 +619,7 @@ func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs if resp == nil { metrics.CmdFailedDurationGetRegion.Observe(time.Since(start).Seconds()) - c.inner.pdSvcDiscovery.ScheduleCheckMemberChanged() + c.inner.serviceDiscovery.ScheduleCheckMemberChanged() errorMsg := fmt.Sprintf("[pd] can't get region info from member URLs: %+v", memberURLs) return nil, errors.WithStack(errors.New(errorMsg)) } @@ -1150,7 +1150,7 @@ func (c *client) SplitRegions(ctx context.Context, splitKeys [][]byte, opts ...o func (c *client) requestHeader() *pdpb.RequestHeader { return &pdpb.RequestHeader{ - ClusterId: c.inner.pdSvcDiscovery.GetClusterID(), + ClusterId: c.inner.serviceDiscovery.GetClusterID(), CallerId: string(caller.GetCallerID()), CallerComponent: string(c.callerComponent), } @@ -1334,7 +1334,7 @@ func (c *client) respForErr(observer prometheus.Observer, start time.Time, err e if err != nil || header.GetError() != nil { observer.Observe(time.Since(start).Seconds()) if err != nil { - c.inner.pdSvcDiscovery.ScheduleCheckMemberChanged() + c.inner.serviceDiscovery.ScheduleCheckMemberChanged() return errors.WithStack(err) } return errors.WithStack(errors.New(header.GetError().String())) diff --git a/client/clients/tso/client.go b/client/clients/tso/client.go index d24dba52394..7bc768ee21b 100644 --- a/client/clients/tso/client.go +++ b/client/clients/tso/client.go @@ -563,7 +563,7 @@ func (c *Cli) DispatchRequest(request *Request) (bool, error) { // Client is closed, no need to retry. return false, request.clientCtx.Err() case <-c.ctx.Done(): - // tsoClient is closed due to the PD service mode switch, which is retryable. + // tsoClient is closed due to the service mode switch, which is retryable. return true, c.ctx.Err() default: // This failpoint will increase the possibility that the request is sent to a closed dispatcher. diff --git a/client/clients/tso/dispatcher_test.go b/client/clients/tso/dispatcher_test.go index 7e5554c7c7b..6d8f94791e6 100644 --- a/client/clients/tso/dispatcher_test.go +++ b/client/clients/tso/dispatcher_test.go @@ -53,7 +53,7 @@ func (m *mockTSOServiceProvider) getOption() *opt.Option { } func (*mockTSOServiceProvider) getServiceDiscovery() sd.ServiceDiscovery { - return sd.NewMockPDServiceDiscovery([]string{mockStreamURL}, nil) + return sd.NewMockServiceDiscovery([]string{mockStreamURL}, nil) } func (m *mockTSOServiceProvider) getConnectionCtxMgr() *cctx.Manager[*tsoStream] { diff --git a/client/http/client.go b/client/http/client.go index fa9801cf764..b7109166a30 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -245,7 +245,7 @@ func (ci *clientInner) doRequest( if readErr != nil { logFields = append(logFields, zap.NamedError("read-body-error", err)) } else { - // API server will return a JSON body containing the detailed error message + // PD service will return a JSON body containing the detailed error message // when the status code is not `http.StatusOK` 200. bs = bytes.TrimSpace(bs) logFields = append(logFields, zap.ByteString("body", bs)) @@ -304,7 +304,7 @@ func WithMetrics( } } -// NewClientWithServiceDiscovery creates a PD HTTP client with the given PD service discovery. +// NewClientWithServiceDiscovery creates a PD HTTP client with the given service discovery. func NewClientWithServiceDiscovery( source string, sd sd.ServiceDiscovery, @@ -332,7 +332,7 @@ func NewClient( for _, opt := range opts { opt(c) } - sd := sd.NewDefaultPDServiceDiscovery(ctx, cancel, pdAddrs, c.inner.tlsConf) + sd := sd.NewDefaultServiceDiscovery(ctx, cancel, pdAddrs, c.inner.tlsConf) if err := sd.Init(); err != nil { log.Error("[pd] init service discovery failed", zap.String("source", source), zap.Strings("pd-addrs", pdAddrs), zap.Error(err)) @@ -420,7 +420,7 @@ func NewHTTPClientWithRequestChecker(checker requestChecker) *http.Client { } } -// newClientWithMockServiceDiscovery creates a new PD HTTP client with a mock PD service discovery. +// newClientWithMockServiceDiscovery creates a new PD HTTP client with a mock service discovery. func newClientWithMockServiceDiscovery( source string, pdAddrs []string, @@ -432,7 +432,7 @@ func newClientWithMockServiceDiscovery( for _, opt := range opts { opt(c) } - sd := sd.NewMockPDServiceDiscovery(pdAddrs, c.inner.tlsConf) + sd := sd.NewMockServiceDiscovery(pdAddrs, c.inner.tlsConf) if err := sd.Init(); err != nil { log.Error("[pd] init mock service discovery failed", zap.String("source", source), zap.Strings("pd-addrs", pdAddrs), zap.Error(err)) diff --git a/client/inner_client.go b/client/inner_client.go index 404cbcf0b80..8379b6a51a9 100644 --- a/client/inner_client.go +++ b/client/inner_client.go @@ -26,10 +26,10 @@ const ( ) type innerClient struct { - keyspaceID uint32 - svrUrls []string - pdSvcDiscovery sd.ServiceDiscovery - tokenDispatcher *tokenDispatcher + keyspaceID uint32 + svrUrls []string + serviceDiscovery sd.ServiceDiscovery + tokenDispatcher *tokenDispatcher // For service mode switching. serviceModeKeeper @@ -45,13 +45,13 @@ type innerClient struct { } func (c *innerClient) init(updateKeyspaceIDCb sd.UpdateKeyspaceIDFunc) error { - c.pdSvcDiscovery = sd.NewPDServiceDiscovery( + c.serviceDiscovery = sd.NewServiceDiscovery( c.ctx, c.cancel, &c.wg, c.setServiceMode, updateKeyspaceIDCb, c.keyspaceID, c.svrUrls, c.tlsCfg, c.option) if err := c.setup(); err != nil { c.cancel() - if c.pdSvcDiscovery != nil { - c.pdSvcDiscovery.Close() + if c.serviceDiscovery != nil { + c.serviceDiscovery.Close() } return err } @@ -92,10 +92,10 @@ func (c *innerClient) resetTSOClientLocked(mode pdpb.ServiceMode) { switch mode { case pdpb.ServiceMode_PD_SVC_MODE: newTSOCli = tso.NewClient(c.ctx, c.option, - c.pdSvcDiscovery, &tso.PDStreamBuilderFactory{}) + c.serviceDiscovery, &tso.PDStreamBuilderFactory{}) case pdpb.ServiceMode_API_SVC_MODE: newTSOSvcDiscovery = sd.NewTSOServiceDiscovery( - c.ctx, c, c.pdSvcDiscovery, + c.ctx, c, c.serviceDiscovery, c.keyspaceID, c.tlsCfg, c.option) // At this point, the keyspace group isn't known yet. Starts from the default keyspace group, // and will be updated later. @@ -119,12 +119,12 @@ func (c *innerClient) resetTSOClientLocked(mode pdpb.ServiceMode) { oldTSOClient.Close() // Replace the old TSO service discovery if needed. oldTSOSvcDiscovery := c.tsoSvcDiscovery - // If newTSOSvcDiscovery is nil, that's expected, as it means we are switching to PD service mode and + // If newTSOSvcDiscovery is nil, that's expected, as it means we are switching to PD mode and // no tso microservice discovery is needed. c.tsoSvcDiscovery = newTSOSvcDiscovery // Close the old TSO service discovery safely after both the old client and service discovery are replaced. if oldTSOSvcDiscovery != nil { - // We are switching from API service mode to PD service mode, so delete the old tso microservice discovery. + // We are switching from PD service mode to PD mode, so delete the old tso microservice discovery. oldTSOSvcDiscovery.Close() } } @@ -153,7 +153,7 @@ func (c *innerClient) close() { c.wg.Wait() c.serviceModeKeeper.close() - c.pdSvcDiscovery.Close() + c.serviceDiscovery.Close() if c.tokenDispatcher != nil { tokenErr := errors.WithStack(errs.ErrClosing) @@ -169,12 +169,12 @@ func (c *innerClient) setup() error { } // Init the client base. - if err := c.pdSvcDiscovery.Init(); err != nil { + if err := c.serviceDiscovery.Init(); err != nil { return err } // Register callbacks - c.pdSvcDiscovery.AddServingURLSwitchedCallback(c.scheduleUpdateTokenConnection) + c.serviceDiscovery.AddServingURLSwitchedCallback(c.scheduleUpdateTokenConnection) // Create dispatchers c.createTokenDispatcher() @@ -186,12 +186,12 @@ func (c *innerClient) setup() error { func (c *innerClient) getRegionAPIClientAndContext(ctx context.Context, allowFollower bool) (sd.ServiceClient, context.Context) { var serviceClient sd.ServiceClient if allowFollower { - serviceClient = c.pdSvcDiscovery.GetServiceClientByKind(sd.UniversalAPIKind) + serviceClient = c.serviceDiscovery.GetServiceClientByKind(sd.UniversalAPIKind) if serviceClient != nil { return serviceClient, serviceClient.BuildGRPCTargetContext(ctx, !allowFollower) } } - serviceClient = c.pdSvcDiscovery.GetServiceClient() + serviceClient = c.serviceDiscovery.GetServiceClient() if serviceClient == nil || serviceClient.GetClientConn() == nil { return nil, ctx } @@ -201,12 +201,12 @@ func (c *innerClient) getRegionAPIClientAndContext(ctx context.Context, allowFol // gRPCErrorHandler is used to handle the gRPC error returned by the resource manager service. func (c *innerClient) gRPCErrorHandler(err error) { if errs.IsLeaderChange(err) { - c.pdSvcDiscovery.ScheduleCheckMemberChanged() + c.serviceDiscovery.ScheduleCheckMemberChanged() } } func (c *innerClient) getOrCreateGRPCConn() (*grpc.ClientConn, error) { - cc, err := c.pdSvcDiscovery.GetOrCreateGRPCConn(c.pdSvcDiscovery.GetServingURL()) + cc, err := c.serviceDiscovery.GetOrCreateGRPCConn(c.serviceDiscovery.GetServingURL()) if err != nil { return nil, err } diff --git a/client/keyspace_client.go b/client/keyspace_client.go index 84bc29054eb..507279e906c 100644 --- a/client/keyspace_client.go +++ b/client/keyspace_client.go @@ -41,7 +41,7 @@ type KeyspaceClient interface { // keyspaceClient returns the KeyspaceClient from current PD leader. func (c *client) keyspaceClient() keyspacepb.KeyspaceClient { - if client := c.inner.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { + if client := c.inner.serviceDiscovery.GetServingEndpointClientConn(); client != nil { return keyspacepb.NewKeyspaceClient(client) } return nil @@ -70,7 +70,7 @@ func (c *client) LoadKeyspace(ctx context.Context, name string) (*keyspacepb.Key if err != nil { metrics.CmdFailedDurationLoadKeyspace.Observe(time.Since(start).Seconds()) - c.inner.pdSvcDiscovery.ScheduleCheckMemberChanged() + c.inner.serviceDiscovery.ScheduleCheckMemberChanged() return nil, err } @@ -115,7 +115,7 @@ func (c *client) UpdateKeyspaceState(ctx context.Context, id uint32, state keysp if err != nil { metrics.CmdFailedDurationUpdateKeyspaceState.Observe(time.Since(start).Seconds()) - c.inner.pdSvcDiscovery.ScheduleCheckMemberChanged() + c.inner.serviceDiscovery.ScheduleCheckMemberChanged() return nil, err } @@ -159,7 +159,7 @@ func (c *client) GetAllKeyspaces(ctx context.Context, startID uint32, limit uint if err != nil { metrics.CmdDurationGetAllKeyspaces.Observe(time.Since(start).Seconds()) - c.inner.pdSvcDiscovery.ScheduleCheckMemberChanged() + c.inner.serviceDiscovery.ScheduleCheckMemberChanged() return nil, err } diff --git a/client/meta_storage_client.go b/client/meta_storage_client.go index fbabd60debd..7652884720d 100644 --- a/client/meta_storage_client.go +++ b/client/meta_storage_client.go @@ -33,7 +33,7 @@ import ( // metaStorageClient gets the meta storage client from current PD leader. func (c *innerClient) metaStorageClient() meta_storagepb.MetaStorageClient { - if client := c.pdSvcDiscovery.GetServingEndpointClientConn(); client != nil { + if client := c.serviceDiscovery.GetServingEndpointClientConn(); client != nil { return meta_storagepb.NewMetaStorageClient(client) } return nil @@ -74,7 +74,7 @@ func (c *innerClient) Put(ctx context.Context, key, value []byte, opts ...opt.Me Lease: options.Lease, PrevKv: options.PrevKv, } - ctx = grpcutil.BuildForwardContext(ctx, c.pdSvcDiscovery.GetServingURL()) + ctx = grpcutil.BuildForwardContext(ctx, c.serviceDiscovery.GetServingURL()) cli := c.metaStorageClient() if cli == nil { cancel() @@ -113,7 +113,7 @@ func (c *innerClient) Get(ctx context.Context, key []byte, opts ...opt.MetaStora Limit: options.Limit, Revision: options.Revision, } - ctx = grpcutil.BuildForwardContext(ctx, c.pdSvcDiscovery.GetServingURL()) + ctx = grpcutil.BuildForwardContext(ctx, c.serviceDiscovery.GetServingURL()) cli := c.metaStorageClient() if cli == nil { cancel() @@ -179,7 +179,7 @@ func (c *innerClient) respForMetaStorageErr(observer prometheus.Observer, start if err != nil || header.GetError() != nil { observer.Observe(time.Since(start).Seconds()) if err != nil { - c.pdSvcDiscovery.ScheduleCheckMemberChanged() + c.serviceDiscovery.ScheduleCheckMemberChanged() return errors.WithStack(err) } return errors.WithStack(errors.New(header.GetError().String())) diff --git a/client/resource_manager_client.go b/client/resource_manager_client.go index 0c481631b93..3e4cd1a3cc8 100644 --- a/client/resource_manager_client.go +++ b/client/resource_manager_client.go @@ -331,7 +331,7 @@ func (c *innerClient) handleResourceTokenDispatcher(dispatcherCtx context.Contex // If the stream is still nil, return an error. if stream == nil { firstRequest.done <- errors.Errorf("failed to get the stream connection") - c.pdSvcDiscovery.ScheduleCheckMemberChanged() + c.serviceDiscovery.ScheduleCheckMemberChanged() connection.reset() continue } @@ -343,7 +343,7 @@ func (c *innerClient) handleResourceTokenDispatcher(dispatcherCtx context.Contex default: } if err = c.processTokenRequests(stream, firstRequest); err != nil { - c.pdSvcDiscovery.ScheduleCheckMemberChanged() + c.serviceDiscovery.ScheduleCheckMemberChanged() connection.reset() log.Info("[resource_manager] token request error", zap.Error(err)) } diff --git a/client/servicediscovery/mock_pd_service_discovery.go b/client/servicediscovery/mock_service_discovery.go similarity index 57% rename from client/servicediscovery/mock_pd_service_discovery.go rename to client/servicediscovery/mock_service_discovery.go index 87b74ae2136..6ca649f4575 100644 --- a/client/servicediscovery/mock_pd_service_discovery.go +++ b/client/servicediscovery/mock_service_discovery.go @@ -21,24 +21,24 @@ import ( "google.golang.org/grpc" ) -var _ ServiceDiscovery = (*mockPDServiceDiscovery)(nil) +var _ ServiceDiscovery = (*mockServiceDiscovery)(nil) -type mockPDServiceDiscovery struct { +type mockServiceDiscovery struct { urls []string tlsCfg *tls.Config clients []ServiceClient } -// NewMockPDServiceDiscovery creates a mock PD service discovery. -func NewMockPDServiceDiscovery(urls []string, tlsCfg *tls.Config) *mockPDServiceDiscovery { - return &mockPDServiceDiscovery{ +// NewMockServiceDiscovery creates a mock service discovery. +func NewMockServiceDiscovery(urls []string, tlsCfg *tls.Config) *mockServiceDiscovery { + return &mockServiceDiscovery{ urls: urls, tlsCfg: tlsCfg, } } // Init directly creates the service clients with the given URLs. -func (m *mockPDServiceDiscovery) Init() error { +func (m *mockServiceDiscovery) Init() error { m.clients = make([]ServiceClient, 0, len(m.urls)) for _, url := range m.urls { m.clients = append(m.clients, newPDServiceClient(url, m.urls[0], nil, false)) @@ -47,61 +47,61 @@ func (m *mockPDServiceDiscovery) Init() error { } // Close clears the service clients. -func (m *mockPDServiceDiscovery) Close() { +func (m *mockServiceDiscovery) Close() { clear(m.clients) } -// GetAllServiceClients returns all service clients init in the mock PD service discovery. -func (m *mockPDServiceDiscovery) GetAllServiceClients() []ServiceClient { +// GetAllServiceClients returns all service clients init in the mock service discovery. +func (m *mockServiceDiscovery) GetAllServiceClients() []ServiceClient { return m.clients } // GetClusterID implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) GetClusterID() uint64 { return 0 } +func (*mockServiceDiscovery) GetClusterID() uint64 { return 0 } // GetKeyspaceID implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) GetKeyspaceID() uint32 { return 0 } +func (*mockServiceDiscovery) GetKeyspaceID() uint32 { return 0 } // SetKeyspaceID implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) SetKeyspaceID(uint32) {} +func (*mockServiceDiscovery) SetKeyspaceID(uint32) {} // GetKeyspaceGroupID implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) GetKeyspaceGroupID() uint32 { return 0 } +func (*mockServiceDiscovery) GetKeyspaceGroupID() uint32 { return 0 } // GetServiceURLs implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) GetServiceURLs() []string { return nil } +func (*mockServiceDiscovery) GetServiceURLs() []string { return nil } // GetServingEndpointClientConn implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) GetServingEndpointClientConn() *grpc.ClientConn { return nil } +func (*mockServiceDiscovery) GetServingEndpointClientConn() *grpc.ClientConn { return nil } // GetClientConns implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) GetClientConns() *sync.Map { return nil } +func (*mockServiceDiscovery) GetClientConns() *sync.Map { return nil } // GetServingURL implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) GetServingURL() string { return "" } +func (*mockServiceDiscovery) GetServingURL() string { return "" } // GetBackupURLs implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) GetBackupURLs() []string { return nil } +func (*mockServiceDiscovery) GetBackupURLs() []string { return nil } // GetServiceClient implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) GetServiceClient() ServiceClient { return nil } +func (*mockServiceDiscovery) GetServiceClient() ServiceClient { return nil } // GetServiceClientByKind implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) GetServiceClientByKind(APIKind) ServiceClient { return nil } +func (*mockServiceDiscovery) GetServiceClientByKind(APIKind) ServiceClient { return nil } // GetOrCreateGRPCConn implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) GetOrCreateGRPCConn(string) (*grpc.ClientConn, error) { +func (*mockServiceDiscovery) GetOrCreateGRPCConn(string) (*grpc.ClientConn, error) { return nil, nil } // ScheduleCheckMemberChanged implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) ScheduleCheckMemberChanged() {} +func (*mockServiceDiscovery) ScheduleCheckMemberChanged() {} // CheckMemberChanged implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) CheckMemberChanged() error { return nil } +func (*mockServiceDiscovery) CheckMemberChanged() error { return nil } // AddServingURLSwitchedCallback implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) AddServingURLSwitchedCallback(...func()) {} +func (*mockServiceDiscovery) AddServingURLSwitchedCallback(...func()) {} // AddServiceURLsSwitchedCallback implements the ServiceDiscovery interface. -func (*mockPDServiceDiscovery) AddServiceURLsSwitchedCallback(...func()) {} +func (*mockServiceDiscovery) AddServiceURLsSwitchedCallback(...func()) {} diff --git a/client/servicediscovery/pd_service_discovery.go b/client/servicediscovery/service_discovery.go similarity index 81% rename from client/servicediscovery/pd_service_discovery.go rename to client/servicediscovery/service_discovery.go index 931f950c6d1..bef80e28a37 100644 --- a/client/servicediscovery/pd_service_discovery.go +++ b/client/servicediscovery/service_discovery.go @@ -157,11 +157,11 @@ type ServiceClient interface { } var ( - _ ServiceClient = (*pdServiceClient)(nil) - _ ServiceClient = (*pdServiceAPIClient)(nil) + _ ServiceClient = (*serviceClient)(nil) + _ ServiceClient = (*serviceAPIClient)(nil) ) -type pdServiceClient struct { +type serviceClient struct { url string conn *grpc.ClientConn isLeader bool @@ -171,10 +171,10 @@ type pdServiceClient struct { } // NOTE: In the current implementation, the URL passed in is bound to have a scheme, -// because it is processed in `newPDServiceDiscovery`, and the url returned by etcd member owns the scheme. +// because it is processed in `newServiceDiscovery`, and the url returned by etcd member owns the scheme. // When testing, the URL is also bound to have a scheme. func newPDServiceClient(url, leaderURL string, conn *grpc.ClientConn, isLeader bool) ServiceClient { - cli := &pdServiceClient{ + cli := &serviceClient{ url: url, conn: conn, isLeader: isLeader, @@ -187,7 +187,7 @@ func newPDServiceClient(url, leaderURL string, conn *grpc.ClientConn, isLeader b } // GetURL implements ServiceClient. -func (c *pdServiceClient) GetURL() string { +func (c *serviceClient) GetURL() string { if c == nil { return "" } @@ -195,7 +195,7 @@ func (c *pdServiceClient) GetURL() string { } // BuildGRPCTargetContext implements ServiceClient. -func (c *pdServiceClient) BuildGRPCTargetContext(ctx context.Context, toLeader bool) context.Context { +func (c *serviceClient) BuildGRPCTargetContext(ctx context.Context, toLeader bool) context.Context { if c == nil || c.isLeader { return ctx } @@ -206,7 +206,7 @@ func (c *pdServiceClient) BuildGRPCTargetContext(ctx context.Context, toLeader b } // IsConnectedToLeader implements ServiceClient. -func (c *pdServiceClient) IsConnectedToLeader() bool { +func (c *serviceClient) IsConnectedToLeader() bool { if c == nil { return false } @@ -214,14 +214,14 @@ func (c *pdServiceClient) IsConnectedToLeader() bool { } // Available implements ServiceClient. -func (c *pdServiceClient) Available() bool { +func (c *serviceClient) Available() bool { if c == nil { return false } return !c.networkFailure.Load() } -func (c *pdServiceClient) checkNetworkAvailable(ctx context.Context) { +func (c *serviceClient) checkNetworkAvailable(ctx context.Context) { if c == nil || c.conn == nil { return } @@ -242,7 +242,7 @@ func (c *pdServiceClient) checkNetworkAvailable(ctx context.Context) { } // GetClientConn implements ServiceClient. -func (c *pdServiceClient) GetClientConn() *grpc.ClientConn { +func (c *serviceClient) GetClientConn() *grpc.ClientConn { if c == nil { return nil } @@ -250,7 +250,7 @@ func (c *pdServiceClient) GetClientConn() *grpc.ClientConn { } // NeedRetry implements ServiceClient. -func (c *pdServiceClient) NeedRetry(pdErr *pdpb.Error, err error) bool { +func (c *serviceClient) NeedRetry(pdErr *pdpb.Error, err error) bool { if c.IsConnectedToLeader() { return false } @@ -267,9 +267,9 @@ func regionAPIErrorFn(pdErr *pdpb.Error) bool { return pdErr.GetType() == pdpb.ErrorType_REGION_NOT_FOUND } -// pdServiceAPIClient is a specific API client for PD service. -// It extends the pdServiceClient and adds additional fields for managing availability -type pdServiceAPIClient struct { +// serviceAPIClient is a specific API client for service. +// It extends the serviceClient and adds additional fields for managing availability +type serviceAPIClient struct { ServiceClient fn errFn @@ -278,19 +278,19 @@ type pdServiceAPIClient struct { } func newPDServiceAPIClient(client ServiceClient, f errFn) ServiceClient { - return &pdServiceAPIClient{ + return &serviceAPIClient{ ServiceClient: client, fn: f, } } // Available implements ServiceClient. -func (c *pdServiceAPIClient) Available() bool { +func (c *serviceAPIClient) Available() bool { return c.ServiceClient.Available() && !c.unavailable.Load() } // markAsAvailable is used to try to mark the client as available if unavailable status is expired. -func (c *pdServiceAPIClient) markAsAvailable() { +func (c *serviceAPIClient) markAsAvailable() { if !c.unavailable.Load() { return } @@ -301,7 +301,7 @@ func (c *pdServiceAPIClient) markAsAvailable() { } // NeedRetry implements ServiceClient. -func (c *pdServiceAPIClient) NeedRetry(pdErr *pdpb.Error, err error) bool { +func (c *serviceAPIClient) NeedRetry(pdErr *pdpb.Error, err error) bool { if c.IsConnectedToLeader() { return false } @@ -317,43 +317,43 @@ func (c *pdServiceAPIClient) NeedRetry(pdErr *pdpb.Error, err error) bool { return true } -// pdServiceBalancerNode is a balancer node for PD service. -// It extends the pdServiceClient and adds additional fields for the next polling client in the chain. -type pdServiceBalancerNode struct { - *pdServiceAPIClient - next *pdServiceBalancerNode +// serviceBalancerNode is a balancer node for PD. +// It extends the serviceClient and adds additional fields for the next polling client in the chain. +type serviceBalancerNode struct { + *serviceAPIClient + next *serviceBalancerNode } -// pdServiceBalancer is a load balancer for PD service clients. -// It is used to balance the request to all servers and manage the connections to multiple PD service nodes. -type pdServiceBalancer struct { +// serviceBalancer is a load balancer for clients. +// It is used to balance the request to all servers and manage the connections to multiple nodes. +type serviceBalancer struct { mu sync.Mutex - now *pdServiceBalancerNode + now *serviceBalancerNode totalNode int errFn errFn } -func newPDServiceBalancer(fn errFn) *pdServiceBalancer { - return &pdServiceBalancer{ +func newServiceBalancer(fn errFn) *serviceBalancer { + return &serviceBalancer{ errFn: fn, } } -func (c *pdServiceBalancer) set(clients []ServiceClient) { +func (c *serviceBalancer) set(clients []ServiceClient) { c.mu.Lock() defer c.mu.Unlock() if len(clients) == 0 { return } c.totalNode = len(clients) - head := &pdServiceBalancerNode{ - pdServiceAPIClient: newPDServiceAPIClient(clients[c.totalNode-1], c.errFn).(*pdServiceAPIClient), + head := &serviceBalancerNode{ + serviceAPIClient: newPDServiceAPIClient(clients[c.totalNode-1], c.errFn).(*serviceAPIClient), } head.next = head last := head for i := c.totalNode - 2; i >= 0; i-- { - next := &pdServiceBalancerNode{ - pdServiceAPIClient: newPDServiceAPIClient(clients[i], c.errFn).(*pdServiceAPIClient), - next: head, + next := &serviceBalancerNode{ + serviceAPIClient: newPDServiceAPIClient(clients[i], c.errFn).(*serviceAPIClient), + next: head, } head = next last.next = head @@ -361,7 +361,7 @@ func (c *pdServiceBalancer) set(clients []ServiceClient) { c.now = head } -func (c *pdServiceBalancer) check() { +func (c *serviceBalancer) check() { c.mu.Lock() defer c.mu.Unlock() for range c.totalNode { @@ -370,11 +370,11 @@ func (c *pdServiceBalancer) check() { } } -func (c *pdServiceBalancer) next() { +func (c *serviceBalancer) next() { c.now = c.now.next } -func (c *pdServiceBalancer) get() (ret ServiceClient) { +func (c *serviceBalancer) get() (ret ServiceClient) { c.mu.Lock() defer c.mu.Unlock() i := 0 @@ -403,22 +403,22 @@ type TSOEventSource interface { } var ( - _ ServiceDiscovery = (*pdServiceDiscovery)(nil) - _ TSOEventSource = (*pdServiceDiscovery)(nil) + _ ServiceDiscovery = (*serviceDiscovery)(nil) + _ TSOEventSource = (*serviceDiscovery)(nil) ) -// pdServiceDiscovery is the service discovery client of PD/API service which is quorum based -type pdServiceDiscovery struct { +// serviceDiscovery is the service discovery client of PD/PD service which is quorum based +type serviceDiscovery struct { isInitialized bool urls atomic.Value // Store as []string // PD leader - leader atomic.Value // Store as pdServiceClient + leader atomic.Value // Store as serviceClient // PD follower - followers sync.Map // Store as map[string]pdServiceClient + followers sync.Map // Store as map[string]serviceClient // PD leader and PD followers - all atomic.Value // Store as []pdServiceClient - apiCandidateNodes [apiKindCount]*pdServiceBalancer + all atomic.Value // Store as []serviceClient + apiCandidateNodes [apiKindCount]*serviceBalancer // PD follower URLs. Only for tso. followerURLs atomic.Value // Store as []string @@ -450,17 +450,17 @@ type pdServiceDiscovery struct { option *opt.Option } -// NewDefaultPDServiceDiscovery returns a new default PD service discovery-based client. -func NewDefaultPDServiceDiscovery( +// NewDefaultServiceDiscovery returns a new default service discovery-based client. +func NewDefaultServiceDiscovery( ctx context.Context, cancel context.CancelFunc, urls []string, tlsCfg *tls.Config, ) ServiceDiscovery { var wg sync.WaitGroup - return NewPDServiceDiscovery(ctx, cancel, &wg, nil, nil, constants.DefaultKeyspaceID, urls, tlsCfg, opt.NewOption()) + return NewServiceDiscovery(ctx, cancel, &wg, nil, nil, constants.DefaultKeyspaceID, urls, tlsCfg, opt.NewOption()) } -// NewPDServiceDiscovery returns a new PD service discovery-based client. -func NewPDServiceDiscovery( +// NewServiceDiscovery returns a new service discovery-based client. +func NewServiceDiscovery( ctx context.Context, cancel context.CancelFunc, wg *sync.WaitGroup, serviceModeUpdateCb func(pdpb.ServiceMode), @@ -468,12 +468,12 @@ func NewPDServiceDiscovery( keyspaceID uint32, urls []string, tlsCfg *tls.Config, option *opt.Option, ) ServiceDiscovery { - pdsd := &pdServiceDiscovery{ + pdsd := &serviceDiscovery{ checkMembershipCh: make(chan struct{}, 1), ctx: ctx, cancel: cancel, wg: wg, - apiCandidateNodes: [apiKindCount]*pdServiceBalancer{newPDServiceBalancer(emptyErrorFn), newPDServiceBalancer(regionAPIErrorFn)}, + apiCandidateNodes: [apiKindCount]*serviceBalancer{newServiceBalancer(emptyErrorFn), newServiceBalancer(regionAPIErrorFn)}, serviceModeUpdateCb: serviceModeUpdateCb, updateKeyspaceIDFunc: updateKeyspaceIDFunc, keyspaceID: keyspaceID, @@ -485,8 +485,8 @@ func NewPDServiceDiscovery( return pdsd } -// Init initializes the PD service discovery. -func (c *pdServiceDiscovery) Init() error { +// Init initializes the service discovery. +func (c *serviceDiscovery) Init() error { if c.isInitialized { return nil } @@ -523,7 +523,7 @@ func (c *pdServiceDiscovery) Init() error { return nil } -func (c *pdServiceDiscovery) initRetry(f func() error) error { +func (c *serviceDiscovery) initRetry(f func() error) error { var err error ticker := time.NewTicker(time.Second) defer ticker.Stop() @@ -540,7 +540,7 @@ func (c *pdServiceDiscovery) initRetry(f func() error) error { return errors.WithStack(err) } -func (c *pdServiceDiscovery) updateMemberLoop() { +func (c *serviceDiscovery) updateMemberLoop() { defer c.wg.Done() ctx, cancel := context.WithCancel(c.ctx) @@ -564,7 +564,7 @@ func (c *pdServiceDiscovery) updateMemberLoop() { } } -func (c *pdServiceDiscovery) updateServiceModeLoop() { +func (c *serviceDiscovery) updateServiceModeLoop() { defer c.wg.Done() failpoint.Inject("skipUpdateServiceMode", func() { failpoint.Return() @@ -596,7 +596,7 @@ func (c *pdServiceDiscovery) updateServiceModeLoop() { } } -func (c *pdServiceDiscovery) memberHealthCheckLoop() { +func (c *serviceDiscovery) memberHealthCheckLoop() { defer c.wg.Done() memberCheckLoopCtx, memberCheckLoopCancel := context.WithCancel(c.ctx) @@ -616,19 +616,19 @@ func (c *pdServiceDiscovery) memberHealthCheckLoop() { } } -func (c *pdServiceDiscovery) checkLeaderHealth(ctx context.Context) { +func (c *serviceDiscovery) checkLeaderHealth(ctx context.Context) { ctx, cancel := context.WithTimeout(ctx, c.option.Timeout) defer cancel() leader := c.getLeaderServiceClient() leader.checkNetworkAvailable(ctx) } -func (c *pdServiceDiscovery) checkFollowerHealth(ctx context.Context) { +func (c *serviceDiscovery) checkFollowerHealth(ctx context.Context) { c.followers.Range(func(_, value any) bool { // To ensure that the leader's healthy check is not delayed, shorten the duration. ctx, cancel := context.WithTimeout(ctx, MemberHealthCheckInterval/3) defer cancel() - serviceClient := value.(*pdServiceClient) + serviceClient := value.(*serviceClient) serviceClient.checkNetworkAvailable(ctx) return true }) @@ -638,12 +638,12 @@ func (c *pdServiceDiscovery) checkFollowerHealth(ctx context.Context) { } // Close releases all resources. -func (c *pdServiceDiscovery) Close() { +func (c *serviceDiscovery) Close() { if c == nil { return } c.closeOnce.Do(func() { - log.Info("[pd] close pd service discovery client") + log.Info("[pd] close service discovery client") c.clientConns.Range(func(key, cc any) bool { if err := cc.(*grpc.ClientConn).Close(); err != nil { log.Error("[pd] failed to close grpc clientConn", errs.ZapError(errs.ErrCloseGRPCConn, err)) @@ -655,28 +655,28 @@ func (c *pdServiceDiscovery) Close() { } // GetClusterID returns the ClusterID. -func (c *pdServiceDiscovery) GetClusterID() uint64 { +func (c *serviceDiscovery) GetClusterID() uint64 { return c.clusterID } // GetKeyspaceID returns the ID of the keyspace -func (c *pdServiceDiscovery) GetKeyspaceID() uint32 { +func (c *serviceDiscovery) GetKeyspaceID() uint32 { return c.keyspaceID } // SetKeyspaceID sets the ID of the keyspace -func (c *pdServiceDiscovery) SetKeyspaceID(keyspaceID uint32) { +func (c *serviceDiscovery) SetKeyspaceID(keyspaceID uint32) { c.keyspaceID = keyspaceID } // GetKeyspaceGroupID returns the ID of the keyspace group -func (*pdServiceDiscovery) GetKeyspaceGroupID() uint32 { - // PD/API service only supports the default keyspace group +func (*serviceDiscovery) GetKeyspaceGroupID() uint32 { + // PD only supports the default keyspace group return constants.DefaultKeyspaceGroupID } // DiscoverMicroservice discovers the microservice with the specified type and returns the server urls. -func (c *pdServiceDiscovery) discoverMicroservice(svcType serviceType) (urls []string, err error) { +func (c *serviceDiscovery) discoverMicroservice(svcType serviceType) (urls []string, err error) { switch svcType { case apiService: urls = c.GetServiceURLs() @@ -703,14 +703,14 @@ func (c *pdServiceDiscovery) discoverMicroservice(svcType serviceType) (urls []s // GetServiceURLs returns the URLs of the servers. // For testing use. It should only be called when the client is closed. -func (c *pdServiceDiscovery) GetServiceURLs() []string { +func (c *serviceDiscovery) GetServiceURLs() []string { return c.urls.Load().([]string) } // GetServingEndpointClientConn returns the grpc client connection of the serving endpoint // which is the leader in a quorum-based cluster or the primary in a primary/secondary // configured cluster. -func (c *pdServiceDiscovery) GetServingEndpointClientConn() *grpc.ClientConn { +func (c *serviceDiscovery) GetServingEndpointClientConn() *grpc.ClientConn { if cc, ok := c.clientConns.Load(c.getLeaderURL()); ok { return cc.(*grpc.ClientConn) } @@ -718,32 +718,32 @@ func (c *pdServiceDiscovery) GetServingEndpointClientConn() *grpc.ClientConn { } // GetClientConns returns the mapping {URL -> a gRPC connection} -func (c *pdServiceDiscovery) GetClientConns() *sync.Map { +func (c *serviceDiscovery) GetClientConns() *sync.Map { return &c.clientConns } // GetServingURL returns the leader url -func (c *pdServiceDiscovery) GetServingURL() string { +func (c *serviceDiscovery) GetServingURL() string { return c.getLeaderURL() } // GetBackupURLs gets the URLs of the current reachable followers // in a quorum-based cluster. Used for tso currently. -func (c *pdServiceDiscovery) GetBackupURLs() []string { +func (c *serviceDiscovery) GetBackupURLs() []string { return c.getFollowerURLs() } // getLeaderServiceClient returns the leader ServiceClient. -func (c *pdServiceDiscovery) getLeaderServiceClient() *pdServiceClient { +func (c *serviceDiscovery) getLeaderServiceClient() *serviceClient { leader := c.leader.Load() if leader == nil { return nil } - return leader.(*pdServiceClient) + return leader.(*serviceClient) } // GetServiceClientByKind returns ServiceClient of the specific kind. -func (c *pdServiceDiscovery) GetServiceClientByKind(kind APIKind) ServiceClient { +func (c *serviceDiscovery) GetServiceClientByKind(kind APIKind) ServiceClient { client := c.apiCandidateNodes[kind].get() if client == nil { return nil @@ -752,7 +752,7 @@ func (c *pdServiceDiscovery) GetServiceClientByKind(kind APIKind) ServiceClient } // GetServiceClient returns the leader/primary ServiceClient if it is healthy. -func (c *pdServiceDiscovery) GetServiceClient() ServiceClient { +func (c *serviceDiscovery) GetServiceClient() ServiceClient { leaderClient := c.getLeaderServiceClient() if c.option.EnableForwarding && !leaderClient.Available() { if followerClient := c.GetServiceClientByKind(ForwardAPIKind); followerClient != nil { @@ -767,7 +767,7 @@ func (c *pdServiceDiscovery) GetServiceClient() ServiceClient { } // GetAllServiceClients implements ServiceDiscovery -func (c *pdServiceDiscovery) GetAllServiceClients() []ServiceClient { +func (c *serviceDiscovery) GetAllServiceClients() []ServiceClient { all := c.all.Load() if all == nil { return nil @@ -778,7 +778,7 @@ func (c *pdServiceDiscovery) GetAllServiceClients() []ServiceClient { // ScheduleCheckMemberChanged is used to check if there is any membership // change among the leader and the followers. -func (c *pdServiceDiscovery) ScheduleCheckMemberChanged() { +func (c *serviceDiscovery) ScheduleCheckMemberChanged() { select { case c.checkMembershipCh <- struct{}{}: default: @@ -787,24 +787,24 @@ func (c *pdServiceDiscovery) ScheduleCheckMemberChanged() { // CheckMemberChanged Immediately check if there is any membership change among the leader/followers in a // quorum-based cluster or among the primary/secondaries in a primary/secondary configured cluster. -func (c *pdServiceDiscovery) CheckMemberChanged() error { +func (c *serviceDiscovery) CheckMemberChanged() error { return c.updateMember() } // AddServingURLSwitchedCallback adds callbacks which will be called // when the leader is switched. -func (c *pdServiceDiscovery) AddServingURLSwitchedCallback(callbacks ...func()) { +func (c *serviceDiscovery) AddServingURLSwitchedCallback(callbacks ...func()) { c.leaderSwitchedCbs = append(c.leaderSwitchedCbs, callbacks...) } // AddServiceURLsSwitchedCallback adds callbacks which will be called when // any leader/follower is changed. -func (c *pdServiceDiscovery) AddServiceURLsSwitchedCallback(callbacks ...func()) { +func (c *serviceDiscovery) AddServiceURLsSwitchedCallback(callbacks ...func()) { c.membersChangedCbs = append(c.membersChangedCbs, callbacks...) } // SetTSOLeaderURLUpdatedCallback adds a callback which will be called when the TSO leader is updated. -func (c *pdServiceDiscovery) SetTSOLeaderURLUpdatedCallback(callback tsoLeaderURLUpdatedFunc) { +func (c *serviceDiscovery) SetTSOLeaderURLUpdatedCallback(callback tsoLeaderURLUpdatedFunc) { url := c.getLeaderURL() if len(url) > 0 { if err := callback(url); err != nil { @@ -815,12 +815,12 @@ func (c *pdServiceDiscovery) SetTSOLeaderURLUpdatedCallback(callback tsoLeaderUR } // getLeaderURL returns the leader URL. -func (c *pdServiceDiscovery) getLeaderURL() string { +func (c *serviceDiscovery) getLeaderURL() string { return c.getLeaderServiceClient().GetURL() } // getFollowerURLs returns the follower URLs. -func (c *pdServiceDiscovery) getFollowerURLs() []string { +func (c *serviceDiscovery) getFollowerURLs() []string { followerURLs := c.followerURLs.Load() if followerURLs == nil { return []string{} @@ -828,7 +828,7 @@ func (c *pdServiceDiscovery) getFollowerURLs() []string { return followerURLs.([]string) } -func (c *pdServiceDiscovery) initClusterID() error { +func (c *serviceDiscovery) initClusterID() error { ctx, cancel := context.WithCancel(c.ctx) defer cancel() clusterID := uint64(0) @@ -855,7 +855,7 @@ func (c *pdServiceDiscovery) initClusterID() error { return nil } -func (c *pdServiceDiscovery) checkServiceModeChanged() error { +func (c *serviceDiscovery) checkServiceModeChanged() error { leaderURL := c.getLeaderURL() if len(leaderURL) == 0 { return errors.New("no leader found") @@ -883,7 +883,7 @@ func (c *pdServiceDiscovery) checkServiceModeChanged() error { return nil } -func (c *pdServiceDiscovery) updateMember() error { +func (c *serviceDiscovery) updateMember() error { for _, url := range c.GetServiceURLs() { members, err := c.getMembers(c.ctx, url, UpdateMemberTimeout) // Check the cluster ID. @@ -916,7 +916,7 @@ func (c *pdServiceDiscovery) updateMember() error { return errs.ErrClientGetMember.FastGenByArgs() } -func (c *pdServiceDiscovery) getClusterInfo(ctx context.Context, url string, timeout time.Duration) (*pdpb.GetClusterInfoResponse, error) { +func (c *serviceDiscovery) getClusterInfo(ctx context.Context, url string, timeout time.Duration) (*pdpb.GetClusterInfoResponse, error) { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() cc, err := c.GetOrCreateGRPCConn(url) @@ -935,7 +935,7 @@ func (c *pdServiceDiscovery) getClusterInfo(ctx context.Context, url string, tim return clusterInfo, nil } -func (c *pdServiceDiscovery) getMembers(ctx context.Context, url string, timeout time.Duration) (*pdpb.GetMembersResponse, error) { +func (c *serviceDiscovery) getMembers(ctx context.Context, url string, timeout time.Duration) (*pdpb.GetMembersResponse, error) { ctx, cancel := context.WithTimeout(ctx, timeout) defer cancel() cc, err := c.GetOrCreateGRPCConn(url) @@ -954,7 +954,7 @@ func (c *pdServiceDiscovery) getMembers(ctx context.Context, url string, timeout return members, nil } -func (c *pdServiceDiscovery) updateURLs(members []*pdpb.Member) { +func (c *serviceDiscovery) updateURLs(members []*pdpb.Member) { urls := make([]string, 0, len(members)) for _, m := range members { urls = append(urls, m.GetClientUrls()...) @@ -974,7 +974,7 @@ func (c *pdServiceDiscovery) updateURLs(members []*pdpb.Member) { log.Info("[pd] update member urls", zap.Strings("old-urls", oldURLs), zap.Strings("new-urls", urls)) } -func (c *pdServiceDiscovery) switchLeader(url string) (bool, error) { +func (c *serviceDiscovery) switchLeader(url string) (bool, error) { oldLeader := c.getLeaderServiceClient() if url == oldLeader.GetURL() && oldLeader.GetClientConn() != nil { return false, nil @@ -999,10 +999,10 @@ func (c *pdServiceDiscovery) switchLeader(url string) (bool, error) { return true, err } -func (c *pdServiceDiscovery) updateFollowers(members []*pdpb.Member, leaderID uint64, leaderURL string) (changed bool) { - followers := make(map[string]*pdServiceClient) +func (c *serviceDiscovery) updateFollowers(members []*pdpb.Member, leaderID uint64, leaderURL string) (changed bool) { + followers := make(map[string]*serviceClient) c.followers.Range(func(key, value any) bool { - followers[key.(string)] = value.(*pdServiceClient) + followers[key.(string)] = value.(*serviceClient) return true }) var followerURLs []string @@ -1015,7 +1015,7 @@ func (c *pdServiceDiscovery) updateFollowers(members []*pdpb.Member, leaderID ui // FIXME: How to safely compare urls(also for leader)? For now, only allows one client url. url := tlsutil.PickMatchedURL(member.GetClientUrls(), c.tlsCfg) if client, ok := c.followers.Load(url); ok { - if client.(*pdServiceClient).GetClientConn() == nil { + if client.(*serviceClient).GetClientConn() == nil { conn, err := c.GetOrCreateGRPCConn(url) if err != nil || conn == nil { log.Warn("[pd] failed to connect follower", zap.String("follower", url), errs.ZapError(err)) @@ -1048,7 +1048,7 @@ func (c *pdServiceDiscovery) updateFollowers(members []*pdpb.Member, leaderID ui return } -func (c *pdServiceDiscovery) updateServiceClient(members []*pdpb.Member, leader *pdpb.Member) error { +func (c *serviceDiscovery) updateServiceClient(members []*pdpb.Member, leader *pdpb.Member) error { // FIXME: How to safely compare leader urls? For now, only allows one client url. leaderURL := tlsutil.PickMatchedURL(leader.GetClientUrls(), c.tlsCfg) leaderChanged, err := c.switchLeader(leaderURL) @@ -1064,7 +1064,7 @@ func (c *pdServiceDiscovery) updateServiceClient(members []*pdpb.Member, leader clients = append(clients, leaderClient) } c.followers.Range(func(_, value any) bool { - clients = append(clients, value.(*pdServiceClient)) + clients = append(clients, value.(*serviceClient)) return true }) c.all.Store(clients) @@ -1076,6 +1076,6 @@ func (c *pdServiceDiscovery) updateServiceClient(members []*pdpb.Member, leader } // GetOrCreateGRPCConn returns the corresponding grpc client connection of the given URL. -func (c *pdServiceDiscovery) GetOrCreateGRPCConn(url string) (*grpc.ClientConn, error) { +func (c *serviceDiscovery) GetOrCreateGRPCConn(url string) (*grpc.ClientConn, error) { return grpcutil.GetOrCreateGRPCConn(c.ctx, &c.clientConns, url, c.tlsCfg, c.option.GRPCDialOptions...) } diff --git a/client/servicediscovery/pd_service_discovery_test.go b/client/servicediscovery/service_discovery_test.go similarity index 96% rename from client/servicediscovery/pd_service_discovery_test.go rename to client/servicediscovery/service_discovery_test.go index dc0a0bd4511..0a678718fdc 100644 --- a/client/servicediscovery/pd_service_discovery_test.go +++ b/client/servicediscovery/service_discovery_test.go @@ -193,14 +193,14 @@ func (suite *serviceClientTestSuite) TestServiceClient() { re.True(leader.IsConnectedToLeader()) re.NoError(failpoint.Enable("github.com/tikv/pd/client/servicediscovery/unreachableNetwork1", "return(true)")) - follower.(*pdServiceClient).checkNetworkAvailable(suite.ctx) - leader.(*pdServiceClient).checkNetworkAvailable(suite.ctx) + follower.(*serviceClient).checkNetworkAvailable(suite.ctx) + leader.(*serviceClient).checkNetworkAvailable(suite.ctx) re.False(follower.Available()) re.False(leader.Available()) re.NoError(failpoint.Disable("github.com/tikv/pd/client/servicediscovery/unreachableNetwork1")) - follower.(*pdServiceClient).checkNetworkAvailable(suite.ctx) - leader.(*pdServiceClient).checkNetworkAvailable(suite.ctx) + follower.(*serviceClient).checkNetworkAvailable(suite.ctx) + leader.(*serviceClient).checkNetworkAvailable(suite.ctx) re.True(follower.Available()) re.True(leader.Available()) @@ -259,11 +259,11 @@ func (suite *serviceClientTestSuite) TestServiceClient() { re.False(leaderAPIClient.NeedRetry(pdErr2, nil)) re.False(followerAPIClient.Available()) re.True(leaderAPIClient.Available()) - followerAPIClient.(*pdServiceAPIClient).markAsAvailable() - leaderAPIClient.(*pdServiceAPIClient).markAsAvailable() + followerAPIClient.(*serviceAPIClient).markAsAvailable() + leaderAPIClient.(*serviceAPIClient).markAsAvailable() re.False(followerAPIClient.Available()) time.Sleep(time.Millisecond * 100) - followerAPIClient.(*pdServiceAPIClient).markAsAvailable() + followerAPIClient.(*serviceAPIClient).markAsAvailable() re.True(followerAPIClient.Available()) re.True(followerAPIClient.NeedRetry(nil, err)) @@ -278,7 +278,7 @@ func (suite *serviceClientTestSuite) TestServiceClientBalancer() { re := suite.Require() follower := suite.followerClient leader := suite.leaderClient - b := &pdServiceBalancer{} + b := &serviceBalancer{} b.set([]ServiceClient{leader, follower}) re.Equal(2, b.totalNode) @@ -400,7 +400,7 @@ func TestUpdateURLs(t *testing.T) { } return } - cli := &pdServiceDiscovery{option: opt.NewOption()} + cli := &serviceDiscovery{option: opt.NewOption()} cli.urls.Store([]string{}) cli.updateURLs(members[1:]) re.Equal(getURLs([]*pdpb.Member{members[1], members[3], members[2]}), cli.GetServiceURLs()) @@ -421,7 +421,7 @@ func TestGRPCDialOption(t *testing.T) { start := time.Now() ctx, cancel := context.WithTimeout(context.TODO(), 500*time.Millisecond) defer cancel() - cli := &pdServiceDiscovery{ + cli := &serviceDiscovery{ checkMembershipCh: make(chan struct{}, 1), ctx: ctx, cancel: cancel, diff --git a/client/servicediscovery/tso_service_discovery.go b/client/servicediscovery/tso_service_discovery.go index 1d2130db804..7734fd23107 100644 --- a/client/servicediscovery/tso_service_discovery.go +++ b/client/servicediscovery/tso_service_discovery.go @@ -126,10 +126,10 @@ func (t *tsoServerDiscovery) resetFailure() { // tsoServiceDiscovery is the service discovery client of the independent TSO service type tsoServiceDiscovery struct { - metacli metastorage.Client - apiSvcDiscovery ServiceDiscovery - clusterID uint64 - keyspaceID atomic.Uint32 + metacli metastorage.Client + serviceDiscovery ServiceDiscovery + clusterID uint64 + keyspaceID atomic.Uint32 // defaultDiscoveryKey is the etcd path used for discovering the serving endpoints of // the default keyspace group @@ -161,7 +161,7 @@ type tsoServiceDiscovery struct { // NewTSOServiceDiscovery returns a new client-side service discovery for the independent TSO service. func NewTSOServiceDiscovery( - ctx context.Context, metacli metastorage.Client, apiSvcDiscovery ServiceDiscovery, + ctx context.Context, metacli metastorage.Client, serviceDiscovery ServiceDiscovery, keyspaceID uint32, tlsCfg *tls.Config, option *opt.Option, ) ServiceDiscovery { ctx, cancel := context.WithCancel(ctx) @@ -169,8 +169,8 @@ func NewTSOServiceDiscovery( ctx: ctx, cancel: cancel, metacli: metacli, - apiSvcDiscovery: apiSvcDiscovery, - clusterID: apiSvcDiscovery.GetClusterID(), + serviceDiscovery: serviceDiscovery, + clusterID: serviceDiscovery.GetClusterID(), tlsCfg: tlsCfg, option: option, checkMembershipCh: make(chan struct{}, 1), @@ -351,7 +351,7 @@ func (c *tsoServiceDiscovery) ScheduleCheckMemberChanged() { // CheckMemberChanged Immediately check if there is any membership change among the primary/secondaries in // a primary/secondary configured cluster. func (c *tsoServiceDiscovery) CheckMemberChanged() error { - if err := c.apiSvcDiscovery.CheckMemberChanged(); err != nil { + if err := c.serviceDiscovery.CheckMemberChanged(); err != nil { log.Warn("[tso] failed to check member changed", errs.ZapError(err)) } if err := c.retry(tsoQueryRetryMaxTimes, tsoQueryRetryInterval, c.updateMember); err != nil { @@ -382,17 +382,17 @@ func (c *tsoServiceDiscovery) SetTSOLeaderURLUpdatedCallback(callback tsoLeaderU // GetServiceClient implements ServiceDiscovery func (c *tsoServiceDiscovery) GetServiceClient() ServiceClient { - return c.apiSvcDiscovery.GetServiceClient() + return c.serviceDiscovery.GetServiceClient() } // GetServiceClientByKind implements ServiceDiscovery func (c *tsoServiceDiscovery) GetServiceClientByKind(kind APIKind) ServiceClient { - return c.apiSvcDiscovery.GetServiceClientByKind(kind) + return c.serviceDiscovery.GetServiceClientByKind(kind) } // GetAllServiceClients implements ServiceDiscovery func (c *tsoServiceDiscovery) GetAllServiceClients() []ServiceClient { - return c.apiSvcDiscovery.GetAllServiceClients() + return c.serviceDiscovery.GetAllServiceClients() } // getPrimaryURL returns the primary URL. @@ -425,7 +425,7 @@ func (c *tsoServiceDiscovery) afterPrimarySwitched(oldPrimary, newPrimary string func (c *tsoServiceDiscovery) updateMember() error { // The keyspace membership or the primary serving URL of the keyspace group, to which this // keyspace belongs, might have been changed. We need to query tso servers to get the latest info. - tsoServerURL, err := c.getTSOServer(c.apiSvcDiscovery) + tsoServerURL, err := c.getTSOServer(c.serviceDiscovery) if err != nil { log.Error("[tso] failed to get tso server", errs.ZapError(err)) return err @@ -589,7 +589,7 @@ func (c *tsoServiceDiscovery) getTSOServer(sd ServiceDiscovery) (string, error) ) t := c.tsoServerDiscovery if len(t.urls) == 0 || t.failureCount == len(t.urls) { - urls, err = sd.(*pdServiceDiscovery).discoverMicroservice(tsoService) + urls, err = sd.(*serviceDiscovery).discoverMicroservice(tsoService) if err != nil { return "", err } diff --git a/cmd/pd-server/main.go b/cmd/pd-server/main.go index 165bcd2a12f..24ca46e7d5e 100644 --- a/cmd/pd-server/main.go +++ b/cmd/pd-server/main.go @@ -82,7 +82,7 @@ func NewServiceCommand() *cobra.Command { } cmd.AddCommand(NewTSOServiceCommand()) cmd.AddCommand(NewSchedulingServiceCommand()) - cmd.AddCommand(NewAPIServiceCommand()) + cmd.AddCommand(NewPDServiceCommand()) return cmd } @@ -128,12 +128,12 @@ func NewSchedulingServiceCommand() *cobra.Command { return cmd } -// NewAPIServiceCommand returns the API service command. -func NewAPIServiceCommand() *cobra.Command { +// NewPDServiceCommand returns the PD service command. +func NewPDServiceCommand() *cobra.Command { cmd := &cobra.Command{ Use: apiMode, - Short: "Run the API service", - Run: createAPIServerWrapper, + Short: "Run the PD service", + Run: createPDServiceWrapper, } addFlags(cmd) return cmd @@ -160,7 +160,7 @@ func addFlags(cmd *cobra.Command) { cmd.Flags().BoolP("force-new-cluster", "", false, "force to create a new one-member cluster") } -func createAPIServerWrapper(cmd *cobra.Command, args []string) { +func createPDServiceWrapper(cmd *cobra.Command, args []string) { start(cmd, args, cmd.CalledAs()) } @@ -219,7 +219,7 @@ func start(cmd *cobra.Command, args []string, services ...string) { defer log.Sync() memory.InitMemoryHook() if len(services) != 0 { - versioninfo.Log(server.APIServiceMode) + versioninfo.Log(server.PDServiceMode) } else { versioninfo.Log(server.PDMode) } diff --git a/pkg/mcs/registry/registry.go b/pkg/mcs/registry/registry.go index 6a01f091e52..2ffa04b1bf9 100644 --- a/pkg/mcs/registry/registry.go +++ b/pkg/mcs/registry/registry.go @@ -85,18 +85,18 @@ func (r *ServiceRegistry) InstallAllRESTHandler(srv bs.Server, h map[string]http serviceName := createServiceName(prefix, name) if l, ok := r.services[serviceName]; ok { if err := l.RegisterRESTHandler(h); err != nil { - log.Error("register restful API service failed", zap.String("prefix", prefix), zap.String("service-name", name), zap.Error(err)) + log.Error("register restful PD service failed", zap.String("prefix", prefix), zap.String("service-name", name), zap.Error(err)) } else { - log.Info("restful API service already registered", zap.String("prefix", prefix), zap.String("service-name", name)) + log.Info("restful PD service already registered", zap.String("prefix", prefix), zap.String("service-name", name)) } continue } l := builder(srv) r.services[serviceName] = l if err := l.RegisterRESTHandler(h); err != nil { - log.Error("register restful API service failed", zap.String("prefix", prefix), zap.String("service-name", name), zap.Error(err)) + log.Error("register restful PD service failed", zap.String("prefix", prefix), zap.String("service-name", name), zap.Error(err)) } else { - log.Info("restful API service registered successfully", zap.String("prefix", prefix), zap.String("service-name", name)) + log.Info("restful PD service registered successfully", zap.String("prefix", prefix), zap.String("service-name", name)) } } } diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 5c7166fba09..6f80572673c 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -55,7 +55,7 @@ type Cluster struct { storage storage.Storage coordinator *schedule.Coordinator checkMembershipCh chan struct{} - apiServerLeader atomic.Value + pdLeader atomic.Value running atomic.Bool // heartbeatRunner is used to process the subtree update task asynchronously. @@ -227,7 +227,7 @@ func (c *Cluster) GetStoreConfig() sc.StoreConfigProvider { return c.persistConf // AllocID allocates a new ID. func (c *Cluster) AllocID() (uint64, error) { - client, err := c.getAPIServerLeaderClient() + client, err := c.getPDLeaderClient() if err != nil { return 0, err } @@ -241,11 +241,11 @@ func (c *Cluster) AllocID() (uint64, error) { return resp.GetId(), nil } -func (c *Cluster) getAPIServerLeaderClient() (pdpb.PDClient, error) { - cli := c.apiServerLeader.Load() +func (c *Cluster) getPDLeaderClient() (pdpb.PDClient, error) { + cli := c.pdLeader.Load() if cli == nil { c.triggerMembershipCheck() - return nil, errors.New("API server leader is not found") + return nil, errors.New("PD leader is not found") } return cli.(pdpb.PDClient), nil } @@ -257,10 +257,10 @@ func (c *Cluster) triggerMembershipCheck() { } } -// SwitchAPIServerLeader switches the API server leader. -func (c *Cluster) SwitchAPIServerLeader(new pdpb.PDClient) bool { - old := c.apiServerLeader.Load() - return c.apiServerLeader.CompareAndSwap(old, new) +// SwitchPDServiceLeader switches the PD service leader. +func (c *Cluster) SwitchPDServiceLeader(new pdpb.PDClient) bool { + old := c.pdLeader.Load() + return c.pdLeader.CompareAndSwap(old, new) } func trySend(notifier chan struct{}) { diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index 784d1f45a82..413a6c601cc 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -243,7 +243,7 @@ func NewPersistConfig(cfg *Config, ttl *cache.TTLString) *PersistConfig { o.SetClusterVersion(&cfg.ClusterVersion) o.schedule.Store(&cfg.Schedule) o.replication.Store(&cfg.Replication) - // storeConfig will be fetched from TiKV by PD API server, + // storeConfig will be fetched from TiKV by PD service, // so we just set an empty value here first. o.storeConfig.Store(&sc.StoreConfig{}) o.ttl = ttl @@ -748,11 +748,11 @@ func (o *PersistConfig) IsRaftKV2() bool { // TODO: implement the following methods // AddSchedulerCfg adds the scheduler configurations. -// This method is a no-op since we only use configurations derived from one-way synchronization from API server now. +// This method is a no-op since we only use configurations derived from one-way synchronization from PD service now. func (*PersistConfig) AddSchedulerCfg(types.CheckerSchedulerType, []string) {} // RemoveSchedulerCfg removes the scheduler configurations. -// This method is a no-op since we only use configurations derived from one-way synchronization from API server now. +// This method is a no-op since we only use configurations derived from one-way synchronization from PD service now. func (*PersistConfig) RemoveSchedulerCfg(types.CheckerSchedulerType) {} // CheckLabelProperty checks if the label property is satisfied. diff --git a/pkg/mcs/scheduling/server/config/watcher.go b/pkg/mcs/scheduling/server/config/watcher.go index f499a0d7d50..9db2d47d0f4 100644 --- a/pkg/mcs/scheduling/server/config/watcher.go +++ b/pkg/mcs/scheduling/server/config/watcher.go @@ -36,7 +36,7 @@ import ( "github.com/tikv/pd/pkg/utils/keypath" ) -// Watcher is used to watch the PD API server for any configuration changes. +// Watcher is used to watch the PD service for any configuration changes. type Watcher struct { wg sync.WaitGroup ctx context.Context @@ -76,7 +76,7 @@ type persistedConfig struct { Store sc.StoreConfig `json:"store"` } -// NewWatcher creates a new watcher to watch the config meta change from PD API server. +// NewWatcher creates a new watcher to watch the config meta change from PD service. func NewWatcher( ctx context.Context, etcdClient *clientv3.Client, diff --git a/pkg/mcs/scheduling/server/grpc_service.go b/pkg/mcs/scheduling/server/grpc_service.go index 3d1183bf734..bd2cc40c21d 100644 --- a/pkg/mcs/scheduling/server/grpc_service.go +++ b/pkg/mcs/scheduling/server/grpc_service.go @@ -159,7 +159,7 @@ func (s *Service) RegionHeartbeat(stream schedulingpb.Scheduling_RegionHeartbeat region := core.RegionFromHeartbeat(request, 0) err = c.HandleRegionHeartbeat(region) if err != nil { - // TODO: if we need to send the error back to API server. + // TODO: if we need to send the error back to PD service. log.Error("failed handle region heartbeat", zap.Error(err)) continue } diff --git a/pkg/mcs/scheduling/server/meta/watcher.go b/pkg/mcs/scheduling/server/meta/watcher.go index 27fe6687f3d..c51f10027d7 100644 --- a/pkg/mcs/scheduling/server/meta/watcher.go +++ b/pkg/mcs/scheduling/server/meta/watcher.go @@ -33,7 +33,7 @@ import ( "github.com/tikv/pd/pkg/utils/keypath" ) -// Watcher is used to watch the PD API server for any meta changes. +// Watcher is used to watch the PD service for any meta changes. type Watcher struct { wg sync.WaitGroup ctx context.Context @@ -48,7 +48,7 @@ type Watcher struct { storeWatcher *etcdutil.LoopWatcher } -// NewWatcher creates a new watcher to watch the meta change from PD API server. +// NewWatcher creates a new watcher to watch the meta change from PD service. func NewWatcher( ctx context.Context, etcdClient *clientv3.Client, diff --git a/pkg/mcs/scheduling/server/rule/watcher.go b/pkg/mcs/scheduling/server/rule/watcher.go index cc6480a0cb4..014a3abc2be 100644 --- a/pkg/mcs/scheduling/server/rule/watcher.go +++ b/pkg/mcs/scheduling/server/rule/watcher.go @@ -34,7 +34,7 @@ import ( "github.com/tikv/pd/pkg/utils/keypath" ) -// Watcher is used to watch the PD API server for any Placement Rule changes. +// Watcher is used to watch the PD service for any Placement Rule changes. type Watcher struct { ctx context.Context cancel context.CancelFunc @@ -74,7 +74,7 @@ type Watcher struct { patch *placement.RuleConfigPatch } -// NewWatcher creates a new watcher to watch the Placement Rule change from PD API server. +// NewWatcher creates a new watcher to watch the Placement Rule change from PD service. func NewWatcher( ctx context.Context, etcdClient *clientv3.Client, diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index 8c9972d5eec..80156c1e26b 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -110,7 +110,7 @@ type Server struct { hbStreams *hbstream.HeartbeatStreams storage *endpoint.StorageEndpoint - // for watching the PD API server meta info updates that are related to the scheduling. + // for watching the PD service meta info updates that are related to the scheduling. configWatcher *config.Watcher ruleWatcher *rule.Watcher metaWatcher *meta.Watcher @@ -169,10 +169,10 @@ func (s *Server) startServerLoop() { s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(s.Context()) s.serverLoopWg.Add(2) go s.primaryElectionLoop() - go s.updateAPIServerMemberLoop() + go s.updatePDServiceMemberLoop() } -func (s *Server) updateAPIServerMemberLoop() { +func (s *Server) updatePDServiceMemberLoop() { defer logutil.LogPanic() defer s.serverLoopWg.Done() @@ -220,7 +220,7 @@ func (s *Server) updateAPIServerMemberLoop() { // double check break } - if s.cluster.SwitchAPIServerLeader(pdpb.NewPDClient(cc)) { + if s.cluster.SwitchPDServiceLeader(pdpb.NewPDClient(cc)) { if status.Leader != curLeader { log.Info("switch leader", zap.String("leader-id", fmt.Sprintf("%x", ep.ID)), zap.String("endpoint", ep.ClientURLs[0])) } diff --git a/pkg/mcs/utils/constant/constant.go b/pkg/mcs/utils/constant/constant.go index 87fcf29f678..6f684bdb977 100644 --- a/pkg/mcs/utils/constant/constant.go +++ b/pkg/mcs/utils/constant/constant.go @@ -57,8 +57,8 @@ const ( // MicroserviceRootPath is the root path of microservice in etcd. MicroserviceRootPath = "/ms" - // APIServiceName is the name of api server. - APIServiceName = "api" + // PDServiceName is the name of pd server. + PDServiceName = "pd" // TSOServiceName is the name of tso server. TSOServiceName = "tso" // SchedulingServiceName is the name of scheduling server. diff --git a/pkg/member/election_leader.go b/pkg/member/election_leader.go index 81afc5dbd0a..2e5769d7dc4 100644 --- a/pkg/member/election_leader.go +++ b/pkg/member/election_leader.go @@ -21,7 +21,7 @@ import ( ) // ElectionLeader defines the common interface of the leader, which is the pdpb.Member -// for in PD/API service or the tsopb.Participant in the micro services. +// for in PD/PD service or the tsopb.Participant in the micro services. type ElectionLeader interface { // GetListenUrls returns the listen urls GetListenUrls() []string diff --git a/pkg/schedule/coordinator.go b/pkg/schedule/coordinator.go index e792560cb37..80299bf1e25 100644 --- a/pkg/schedule/coordinator.go +++ b/pkg/schedule/coordinator.go @@ -389,7 +389,7 @@ func (c *Coordinator) LoadPlugin(pluginPath string, ch chan string) { return } log.Info("create scheduler", zap.String("scheduler-name", s.GetName())) - // TODO: handle the plugin in API service mode. + // TODO: handle the plugin in PD service mode. if err = c.schedulers.AddScheduler(s); err != nil { log.Error("can't add scheduler", zap.String("scheduler-name", s.GetName()), errs.ZapError(err)) return diff --git a/pkg/schedule/schedulers/scheduler_controller.go b/pkg/schedule/schedulers/scheduler_controller.go index 28973631570..5f461d326c5 100644 --- a/pkg/schedule/schedulers/scheduler_controller.go +++ b/pkg/schedule/schedulers/scheduler_controller.go @@ -55,7 +55,7 @@ type Controller struct { // and used in the PD leader service mode now. schedulers map[string]*ScheduleController // schedulerHandlers is used to manage the HTTP handlers of schedulers, - // which will only be initialized and used in the API service mode now. + // which will only be initialized and used in the PD service mode now. schedulerHandlers map[string]http.Handler opController *operator.Controller } diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index 149c68029be..9793939fa17 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -334,7 +334,7 @@ type KeyspaceGroupManager struct { // Value: discover.ServiceRegistryEntry tsoServiceKey string // legacySvcRootPath defines the legacy root path for all etcd paths which derives from - // the PD/API service. It's in the format of "/pd/{cluster_id}". + // the PD/PD service. It's in the format of "/pd/{cluster_id}". // The main paths for different usages include: // 1. The path, used by the default keyspace group, for LoadTimestamp/SaveTimestamp in the // storage endpoint. diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index 85b958a5554..823deed64ea 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -116,7 +116,7 @@ func MicroserviceRedirectRule(matchPath, targetPath, targetServiceName string, } func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, string) { - if !h.s.IsAPIServiceMode() { + if !h.s.IsPDServiceMode() { return false, "" } if len(h.microserviceRedirectRules) == 0 { @@ -223,7 +223,7 @@ func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http clientUrls = leader.GetClientUrls() r.Header.Set(apiutil.PDRedirectorHeader, h.s.Name()) } else { - // Prevent more than one redirection among PD/API servers. + // Prevent more than one redirection among PD/PD service. log.Error("redirect but server is not leader", zap.String("from", name), zap.String("server", h.s.Name()), errs.ZapError(errs.ErrRedirectToNotLeader)) http.Error(w, errs.ErrRedirectToNotLeader.FastGenByArgs().Error(), http.StatusInternalServerError) return diff --git a/server/api/admin.go b/server/api/admin.go index d2be53cf40e..561f4ec4bff 100644 --- a/server/api/admin.go +++ b/server/api/admin.go @@ -254,5 +254,5 @@ func (h *adminHandler) deleteRegionCacheInSchedulingServer(id ...uint64) error { } func buildMsg(err error) string { - return fmt.Sprintf("This operation was executed in API server but needs to be re-executed on scheduling server due to the following error: %s", err.Error()) + return fmt.Sprintf("This operation was executed in PD service but needs to be re-executed on scheduling server due to the following error: %s", err.Error()) } diff --git a/server/api/server.go b/server/api/server.go index 1a744635e2d..8e352b6a36e 100644 --- a/server/api/server.go +++ b/server/api/server.go @@ -63,7 +63,7 @@ func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.AP // Following requests are **not** redirected: // "/schedulers", http.MethodPost // "/schedulers/{name}", http.MethodDelete - // Because the writing of all the config of the scheduling service is in the API server, + // Because the writing of all the config of the scheduling service is in the PD service, // we should not post and delete the scheduler directly in the scheduling service. router.PathPrefix(apiPrefix).Handler(negroni.New( serverapi.NewRuntimeServiceValidator(svr, group), @@ -153,7 +153,7 @@ func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.AP scheapi.APIPathPrefix+"/config/placement-rule", constant.SchedulingServiceName, []string{http.MethodGet}), - // because the writing of all the meta information of the scheduling service is in the API server, + // because the writing of all the meta information of the scheduling service is in the PD service, // we should not post and delete the scheduler directly in the scheduling service. serverapi.MicroserviceRedirectRule( prefix+"/schedulers", diff --git a/server/apiv2/handlers/micro_service.go b/server/apiv2/handlers/micro_service.go index c7fa0dc94f2..b4d3d6bbe89 100644 --- a/server/apiv2/handlers/micro_service.go +++ b/server/apiv2/handlers/micro_service.go @@ -39,7 +39,7 @@ func RegisterMicroService(r *gin.RouterGroup) { // @Router /ms/members/{service} [get] func GetMembers(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) - if !svr.IsAPIServiceMode() { + if !svr.IsPDServiceMode() { c.AbortWithStatusJSON(http.StatusNotFound, "not support micro service") return } @@ -65,7 +65,7 @@ func GetMembers(c *gin.Context) { // @Router /ms/primary/{service} [get] func GetPrimary(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) - if !svr.IsAPIServiceMode() { + if !svr.IsPDServiceMode() { c.AbortWithStatusJSON(http.StatusNotFound, "not support micro service") return } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index d2f3855d14e..e482b7b8b68 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -131,7 +131,7 @@ type Server interface { GetMembers() ([]*pdpb.Member, error) ReplicateFileToMember(ctx context.Context, member *pdpb.Member, name string, data []byte) error GetKeyspaceGroupManager() *keyspace.GroupManager - IsAPIServiceMode() bool + IsPDServiceMode() bool GetSafePointV2Manager() *gc.SafePointV2Manager } @@ -156,12 +156,12 @@ type RaftCluster struct { etcdClient *clientv3.Client httpClient *http.Client - running bool - isAPIServiceMode bool - meta *metapb.Cluster - storage storage.Storage - minResolvedTS atomic.Value // Store as uint64 - externalTS atomic.Value // Store as uint64 + running bool + isPDServiceMode bool + meta *metapb.Cluster + storage storage.Storage + minResolvedTS atomic.Value // Store as uint64 + externalTS atomic.Value // Store as uint64 // Keep the previous store limit settings when removing a store. prevStoreLimit map[uint64]map[storelimit.Type]float64 @@ -325,7 +325,7 @@ func (c *RaftCluster) Start(s Server, bootstrap bool) (err error) { log.Warn("raft cluster has already been started") return nil } - c.isAPIServiceMode = s.IsAPIServiceMode() + c.isPDServiceMode = s.IsPDServiceMode() err = c.InitCluster(s.GetAllocator(), s.GetPersistOptions(), s.GetHBStreams(), s.GetKeyspaceGroupManager()) if err != nil { return err @@ -376,7 +376,7 @@ func (c *RaftCluster) Start(s Server, bootstrap bool) (err error) { c.loadExternalTS() c.loadMinResolvedTS() - if c.isAPIServiceMode { + if c.isPDServiceMode { // bootstrap keyspace group manager after starting other parts successfully. // This order avoids a stuck goroutine in keyspaceGroupManager when it fails to create raftcluster. err = c.keyspaceGroupManager.Bootstrap(c.ctx) @@ -404,7 +404,7 @@ func (c *RaftCluster) Start(s Server, bootstrap bool) (err error) { } func (c *RaftCluster) checkSchedulingService() { - if c.isAPIServiceMode { + if c.isPDServiceMode { servers, err := discovery.Discover(c.etcdClient, constant.SchedulingServiceName) if c.opt.GetMicroServiceConfig().IsSchedulingFallbackEnabled() && (err != nil || len(servers) == 0) { c.startSchedulingJobs(c, c.hbstreams) @@ -425,7 +425,7 @@ func (c *RaftCluster) checkSchedulingService() { // checkTSOService checks the TSO service. func (c *RaftCluster) checkTSOService() { - if c.isAPIServiceMode { + if c.isPDServiceMode { if c.opt.GetMicroServiceConfig().IsTSODynamicSwitchingEnabled() { servers, err := discovery.Discover(c.etcdClient, constant.TSOServiceName) if err != nil || len(servers) == 0 { diff --git a/server/config/config.go b/server/config/config.go index 282b5264fe9..69cd76409bc 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -860,7 +860,7 @@ func (c *MicroServiceConfig) Clone() *MicroServiceConfig { return &cfg } -// IsSchedulingFallbackEnabled returns whether to enable scheduling service fallback to api service. +// IsSchedulingFallbackEnabled returns whether to enable scheduling service fallback to PD service. func (c *MicroServiceConfig) IsSchedulingFallbackEnabled() bool { return c.EnableSchedulingFallback } diff --git a/server/grpc_service.go b/server/grpc_service.go index d3fc5c58d7f..118b3d84748 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -271,8 +271,8 @@ func (s *GrpcServer) GetClusterInfo(context.Context, *pdpb.GetClusterInfoRequest }, nil } -// GetMinTS implements gRPC PDServer. In PD service mode, it simply returns a timestamp. -// In API service mode, it queries all tso servers and gets the minimum timestamp across +// GetMinTS implements gRPC PDServer. In PD mode, it simply returns a timestamp. +// In PD service mode, it queries all tso servers and gets the minimum timestamp across // all keyspace groups. func (s *GrpcServer) GetMinTS( ctx context.Context, request *pdpb.GetMinTSRequest, diff --git a/server/server.go b/server/server.go index 94250128fe3..8de2a5c57f3 100644 --- a/server/server.go +++ b/server/server.go @@ -101,8 +101,8 @@ const ( // PDMode represents that server is in PD mode. PDMode = "PD" - // APIServiceMode represents that server is in API service mode. - APIServiceMode = "API Service" + // PDServiceMode represents that server is in PD service mode which is in microservice architecture. + PDServiceMode = "PD Service" // maxRetryTimesGetServicePrimary is the max retry times for getting primary addr. // Note: it need to be less than client.defaultPDTimeout @@ -243,7 +243,7 @@ type HandlerBuilder func(context.Context, *Server) (http.Handler, apiutil.APISer func CreateServer(ctx context.Context, cfg *config.Config, services []string, legacyServiceBuilders ...HandlerBuilder) (*Server, error) { var mode string if len(services) != 0 { - mode = APIServiceMode + mode = PDServiceMode } else { mode = PDMode } @@ -478,7 +478,7 @@ func (s *Server) startServer(ctx context.Context) error { Member: s.member.MemberValue(), Step: keyspace.AllocStep, }) - if s.IsAPIServiceMode() { + if s.IsPDServiceMode() { s.keyspaceGroupManager = keyspace.NewKeyspaceGroupManager(s.ctx, s.storage, s.client) } s.keyspaceManager = keyspace.NewKeyspaceManager(s.ctx, s.storage, s.cluster, keyspaceIDAllocator, &s.cfg.Keyspace, s.keyspaceGroupManager) @@ -530,7 +530,7 @@ func (s *Server) Close() { s.cgMonitor.StopMonitor() s.stopServerLoop() - if s.IsAPIServiceMode() { + if s.IsPDServiceMode() { s.keyspaceGroupManager.Close() } @@ -641,7 +641,7 @@ func (s *Server) startServerLoop(ctx context.Context) { go s.etcdLeaderLoop() go s.serverMetricsLoop() go s.encryptionKeyManagerLoop() - if s.IsAPIServiceMode() { + if s.IsPDServiceMode() { s.initTSOPrimaryWatcher() s.initSchedulingPrimaryWatcher() } @@ -788,9 +788,9 @@ func (s *Server) stopRaftCluster() { s.cluster.Stop() } -// IsAPIServiceMode return whether the server is in API service mode. -func (s *Server) IsAPIServiceMode() bool { - return s.mode == APIServiceMode +// IsPDServiceMode return whether the server is in PD service mode. +func (s *Server) IsPDServiceMode() bool { + return s.mode == PDServiceMode } // GetAddr returns the server urls for clients. @@ -1390,7 +1390,7 @@ func (s *Server) GetRaftCluster() *cluster.RaftCluster { // IsServiceIndependent returns whether the service is independent. func (s *Server) IsServiceIndependent(name string) bool { - if s.mode == APIServiceMode && !s.IsClosed() { + if s.mode == PDServiceMode && !s.IsClosed() { if name == constant.TSOServiceName && !s.GetMicroServiceConfig().IsTSODynamicSwitchingEnabled() { return true } @@ -1667,7 +1667,7 @@ func (s *Server) campaignLeader() { log.Info(fmt.Sprintf("start to campaign %s leader", s.mode), zap.String("campaign-leader-name", s.Name())) if err := s.member.CampaignLeader(s.ctx, s.cfg.LeaderLease); err != nil { if err.Error() == errs.ErrEtcdTxnConflict.Error() { - log.Info(fmt.Sprintf("campaign %s leader meets error due to txn conflict, another PD/API server may campaign successfully", s.mode), + log.Info(fmt.Sprintf("campaign %s leader meets error due to txn conflict, another PD/PD service may campaign successfully", s.mode), zap.String("campaign-leader-name", s.Name())) } else { log.Error(fmt.Sprintf("campaign %s leader meets error due to etcd error", s.mode), diff --git a/server/server_test.go b/server/server_test.go index 23da2078cb2..28839b89389 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -266,13 +266,13 @@ func TestAPIService(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() mockHandler := CreateMockHandler(re, "127.0.0.1") - svr, err := CreateServer(ctx, cfg, []string{constant.APIServiceName}, mockHandler) + svr, err := CreateServer(ctx, cfg, []string{constant.PDServiceName}, mockHandler) re.NoError(err) defer svr.Close() err = svr.Run() re.NoError(err) MustWaitLeader(re, []*Server{svr}) - re.True(svr.IsAPIServiceMode()) + re.True(svr.IsPDServiceMode()) } func TestIsPathInDirectory(t *testing.T) { diff --git a/tests/cluster.go b/tests/cluster.go index a4f445155e1..4189b43902a 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -79,16 +79,7 @@ type TestServer struct { var zapLogOnce sync.Once // NewTestServer creates a new TestServer. -func NewTestServer(ctx context.Context, cfg *config.Config) (*TestServer, error) { - return createTestServer(ctx, cfg, nil) -} - -// NewTestAPIServer creates a new TestServer. -func NewTestAPIServer(ctx context.Context, cfg *config.Config) (*TestServer, error) { - return createTestServer(ctx, cfg, []string{constant.APIServiceName}) -} - -func createTestServer(ctx context.Context, cfg *config.Config, services []string) (*TestServer, error) { +func NewTestServer(ctx context.Context, cfg *config.Config, services []string) (*TestServer, error) { // disable the heartbeat async runner in test cfg.Schedule.EnableHeartbeatConcurrentRunner = false err := logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) @@ -435,15 +426,15 @@ type ConfigOption func(conf *config.Config, serverName string) // NewTestCluster creates a new TestCluster. func NewTestCluster(ctx context.Context, initialServerCount int, opts ...ConfigOption) (*TestCluster, error) { - return createTestCluster(ctx, initialServerCount, false, opts...) + return createTestCluster(ctx, initialServerCount, nil, opts...) } -// NewTestAPICluster creates a new TestCluster with API service. -func NewTestAPICluster(ctx context.Context, initialServerCount int, opts ...ConfigOption) (*TestCluster, error) { - return createTestCluster(ctx, initialServerCount, true, opts...) +// NewTestPDServiceCluster creates a new TestCluster with PD service. +func NewTestPDServiceCluster(ctx context.Context, initialServerCount int, opts ...ConfigOption) (*TestCluster, error) { + return createTestCluster(ctx, initialServerCount, []string{constant.PDServiceName}, opts...) } -func createTestCluster(ctx context.Context, initialServerCount int, isAPIServiceMode bool, opts ...ConfigOption) (*TestCluster, error) { +func createTestCluster(ctx context.Context, initialServerCount int, services []string, opts ...ConfigOption) (*TestCluster, error) { schedulers.Register() config := newClusterConfig(initialServerCount) servers := make(map[string]*TestServer) @@ -452,12 +443,7 @@ func createTestCluster(ctx context.Context, initialServerCount int, isAPIService if err != nil { return nil, err } - var s *TestServer - if isAPIServiceMode { - s, err = NewTestAPIServer(ctx, serverConf) - } else { - s, err = NewTestServer(ctx, serverConf) - } + s, err := NewTestServer(ctx, serverConf, services) if err != nil { return nil, err } @@ -481,7 +467,7 @@ func RestartTestAPICluster(ctx context.Context, cluster *TestCluster) (*TestClus } func restartTestCluster( - ctx context.Context, cluster *TestCluster, isAPIServiceMode bool, + ctx context.Context, cluster *TestCluster, isPDServiceMode bool, ) (newTestCluster *TestCluster, err error) { schedulers.Register() newTestCluster = &TestCluster{ @@ -508,10 +494,10 @@ func restartTestCluster( newServer *TestServer serverErr error ) - if isAPIServiceMode { - newServer, serverErr = NewTestAPIServer(ctx, serverCfg) + if isPDServiceMode { + newServer, serverErr = NewTestServer(ctx, serverCfg, []string{constant.PDServiceName}) } else { - newServer, serverErr = NewTestServer(ctx, serverCfg) + newServer, serverErr = NewTestServer(ctx, serverCfg, nil) } serverMap.Store(serverName, newServer) errorMap.Store(serverName, serverErr) @@ -735,7 +721,7 @@ func (c *TestCluster) Join(ctx context.Context, opts ...ConfigOption) (*TestServ if err != nil { return nil, err } - s, err := NewTestServer(ctx, conf) + s, err := NewTestServer(ctx, conf, nil) if err != nil { return nil, err } @@ -743,13 +729,13 @@ func (c *TestCluster) Join(ctx context.Context, opts ...ConfigOption) (*TestServ return s, nil } -// JoinAPIServer is used to add a new TestAPIServer into the cluster. -func (c *TestCluster) JoinAPIServer(ctx context.Context, opts ...ConfigOption) (*TestServer, error) { +// JoinPDServer is used to add a new TestServer into the cluster. +func (c *TestCluster) JoinPDServer(ctx context.Context, opts ...ConfigOption) (*TestServer, error) { conf, err := c.config.join().Generate(opts...) if err != nil { return nil, err } - s, err := NewTestAPIServer(ctx, conf) + s, err := NewTestServer(ctx, conf, []string{constant.PDServiceName}) if err != nil { return nil, err } diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 397e1079af3..ab3874a33a7 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -361,7 +361,7 @@ func TestTSOFollowerProxyWithTSOService(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/client/servicediscovery/fastUpdateServiceMode", `return(true)`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestAPICluster(ctx, 1) + cluster, err := tests.NewTestPDServiceCluster(ctx, 1) re.NoError(err) defer cluster.Destroy() err = cluster.RunInitialServers() diff --git a/tests/integrations/mcs/discovery/register_test.go b/tests/integrations/mcs/discovery/register_test.go index da6fa158307..eb8933e10d8 100644 --- a/tests/integrations/mcs/discovery/register_test.go +++ b/tests/integrations/mcs/discovery/register_test.go @@ -54,7 +54,7 @@ func (suite *serverRegisterTestSuite) SetupSuite() { re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() @@ -84,7 +84,7 @@ func (suite *serverRegisterTestSuite) checkServerRegister(serviceName string) { addr := s.GetAddr() client := suite.pdLeader.GetEtcdClient() - // test API server discovery + // test PD service discovery endpoints, err := discovery.Discover(client, serviceName) re.NoError(err) @@ -98,7 +98,7 @@ func (suite *serverRegisterTestSuite) checkServerRegister(serviceName string) { re.True(exist) re.Equal(expectedPrimary, primary) - // test API server discovery after unregister + // test PD service discovery after unregister cleanup() endpoints, err = discovery.Discover(client, serviceName) re.NoError(err) @@ -140,7 +140,7 @@ func (suite *serverRegisterTestSuite) checkServerPrimaryChange(serviceName strin delete(serverMap, primary) expectedPrimary = tests.WaitForPrimaryServing(re, serverMap) - // test API server discovery + // test PD service discovery client := suite.pdLeader.GetEtcdClient() endpoints, err := discovery.Discover(client, serviceName) re.NoError(err) diff --git a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go index 44347b4757d..b31d919324d 100644 --- a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go +++ b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go @@ -60,7 +60,7 @@ func (suite *keyspaceGroupTestSuite) SetupTest() { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) ctx, cancel := context.WithCancel(context.Background()) suite.ctx = ctx - cluster, err := tests.NewTestAPICluster(suite.ctx, 1) + cluster, err := tests.NewTestPDServiceCluster(suite.ctx, 1) suite.cluster = cluster re.NoError(err) re.NoError(cluster.RunInitialServers()) diff --git a/tests/integrations/mcs/members/member_test.go b/tests/integrations/mcs/members/member_test.go index 28275849073..7e83ea570b9 100644 --- a/tests/integrations/mcs/members/member_test.go +++ b/tests/integrations/mcs/members/member_test.go @@ -64,7 +64,7 @@ func (suite *memberTestSuite) SetupTest() { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) ctx, cancel := context.WithCancel(context.Background()) suite.ctx = ctx - cluster, err := tests.NewTestAPICluster(suite.ctx, 1) + cluster, err := tests.NewTestPDServiceCluster(suite.ctx, 1) suite.cluster = cluster re.NoError(err) re.NoError(cluster.RunInitialServers()) diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index abace06bb78..9b6b3d95145 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -56,7 +56,7 @@ func (suite *apiTestSuite) TearDownSuite() { } func (suite *apiTestSuite) TestGetCheckerByName() { - suite.env.RunTestInAPIMode(suite.checkGetCheckerByName) + suite.env.RunTestInPDServiceMode(suite.checkGetCheckerByName) } func (suite *apiTestSuite) checkGetCheckerByName(cluster *tests.TestCluster) { @@ -102,7 +102,7 @@ func (suite *apiTestSuite) checkGetCheckerByName(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestAPIForward() { - suite.env.RunTestInAPIMode(suite.checkAPIForward) + suite.env.RunTestInPDServiceMode(suite.checkAPIForward) } func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { @@ -378,7 +378,7 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestConfig() { - suite.env.RunTestInAPIMode(suite.checkConfig) + suite.env.RunTestInPDServiceMode(suite.checkConfig) } func (suite *apiTestSuite) checkConfig(cluster *tests.TestCluster) { @@ -401,7 +401,7 @@ func (suite *apiTestSuite) checkConfig(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestConfigForward() { - suite.env.RunTestInAPIMode(suite.checkConfigForward) + suite.env.RunTestInPDServiceMode(suite.checkConfigForward) } func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { @@ -413,7 +413,7 @@ func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { urlPrefix := fmt.Sprintf("%s/pd/api/v1/config", addr) // Test config forward - // Expect to get same config in scheduling server and api server + // Expect to get same config in scheduling server and PD service testutil.Eventually(re, func() bool { testutil.ReadGetJSON(re, tests.TestDialClient, urlPrefix, &cfg) re.Equal(cfg["schedule"].(map[string]any)["leader-schedule-limit"], @@ -421,8 +421,8 @@ func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { return cfg["replication"].(map[string]any)["max-replicas"] == float64(opts.GetReplicationConfig().MaxReplicas) }) - // Test to change config in api server - // Expect to get new config in scheduling server and api server + // Test to change config in PD service + // Expect to get new config in scheduling server and PD service reqData, err := json.Marshal(map[string]any{ "max-replicas": 4, }) @@ -436,7 +436,7 @@ func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { }) // Test to change config only in scheduling server - // Expect to get new config in scheduling server but not old config in api server + // Expect to get new config in scheduling server but not old config in PD service scheCfg := opts.GetScheduleConfig().Clone() scheCfg.LeaderScheduleLimit = 100 opts.SetScheduleConfig(scheCfg) @@ -452,7 +452,7 @@ func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestAdminRegionCache() { - suite.env.RunTestInAPIMode(suite.checkAdminRegionCache) + suite.env.RunTestInPDServiceMode(suite.checkAdminRegionCache) } func (suite *apiTestSuite) checkAdminRegionCache(cluster *tests.TestCluster) { @@ -479,7 +479,7 @@ func (suite *apiTestSuite) checkAdminRegionCache(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestAdminRegionCacheForward() { - suite.env.RunTestInAPIMode(suite.checkAdminRegionCacheForward) + suite.env.RunTestInPDServiceMode(suite.checkAdminRegionCacheForward) } func (suite *apiTestSuite) checkAdminRegionCacheForward(cluster *tests.TestCluster) { @@ -491,22 +491,22 @@ func (suite *apiTestSuite) checkAdminRegionCacheForward(cluster *tests.TestClust r3 := core.NewTestRegionInfo(30, 1, []byte("c"), []byte(""), core.SetRegionConfVer(100), core.SetRegionVersion(100)) tests.MustPutRegionInfo(re, cluster, r3) - apiServer := cluster.GetLeaderServer().GetServer() + pdServer := cluster.GetLeaderServer().GetServer() schedulingServer := cluster.GetSchedulingPrimaryServer() re.Equal(3, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) - re.Equal(3, apiServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{})) + re.Equal(3, pdServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{})) addr := cluster.GetLeaderServer().GetAddr() urlPrefix := fmt.Sprintf("%s/pd/api/v1/admin/cache/region", addr) err := testutil.CheckDelete(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "30"), testutil.StatusOK(re)) re.NoError(err) re.Equal(2, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) - re.Equal(2, apiServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{})) + re.Equal(2, pdServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{})) err = testutil.CheckDelete(tests.TestDialClient, urlPrefix+"s", testutil.StatusOK(re)) re.NoError(err) re.Equal(0, schedulingServer.GetCluster().GetRegionCount([]byte{}, []byte{})) - re.Equal(0, apiServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{})) + re.Equal(0, pdServer.GetRaftCluster().GetRegionCount([]byte{}, []byte{})) } func (suite *apiTestSuite) TestFollowerForward() { @@ -520,7 +520,7 @@ func (suite *apiTestSuite) checkFollowerForward(cluster *tests.TestCluster) { leaderAddr := cluster.GetLeaderServer().GetAddr() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - follower, err := cluster.JoinAPIServer(ctx) + follower, err := cluster.JoinPDServer(ctx) re.NoError(err) re.NoError(follower.Run()) re.NotEmpty(cluster.WaitLeader()) @@ -558,7 +558,7 @@ func (suite *apiTestSuite) checkFollowerForward(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestMetrics() { - suite.env.RunTestInAPIMode(suite.checkMetrics) + suite.env.RunTestInPDServiceMode(suite.checkMetrics) } func (suite *apiTestSuite) checkMetrics(cluster *tests.TestCluster) { @@ -577,7 +577,7 @@ func (suite *apiTestSuite) checkMetrics(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestStatus() { - suite.env.RunTestInAPIMode(suite.checkStatus) + suite.env.RunTestInPDServiceMode(suite.checkStatus) } func (suite *apiTestSuite) checkStatus(cluster *tests.TestCluster) { @@ -600,7 +600,7 @@ func (suite *apiTestSuite) checkStatus(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestStores() { - suite.env.RunTestInAPIMode(suite.checkStores) + suite.env.RunTestInPDServiceMode(suite.checkStores) } func (suite *apiTestSuite) checkStores(cluster *tests.TestCluster) { @@ -647,8 +647,8 @@ func (suite *apiTestSuite) checkStores(cluster *tests.TestCluster) { tests.MustPutStore(re, cluster, store) } // Test /stores - apiServerAddr := cluster.GetLeaderServer().GetAddr() - urlPrefix := fmt.Sprintf("%s/pd/api/v1/stores", apiServerAddr) + pdServiceAddr := cluster.GetLeaderServer().GetAddr() + urlPrefix := fmt.Sprintf("%s/pd/api/v1/stores", pdServiceAddr) var resp map[string]any err := testutil.ReadGetJSON(re, tests.TestDialClient, urlPrefix, &resp) re.NoError(err) @@ -682,7 +682,7 @@ func (suite *apiTestSuite) checkStores(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestRegions() { - suite.env.RunTestInAPIMode(suite.checkRegions) + suite.env.RunTestInPDServiceMode(suite.checkRegions) } func (suite *apiTestSuite) checkRegions(cluster *tests.TestCluster) { @@ -691,8 +691,8 @@ func (suite *apiTestSuite) checkRegions(cluster *tests.TestCluster) { tests.MustPutRegion(re, cluster, 2, 2, []byte("c"), []byte("d")) tests.MustPutRegion(re, cluster, 3, 1, []byte("e"), []byte("f")) // Test /regions - apiServerAddr := cluster.GetLeaderServer().GetAddr() - urlPrefix := fmt.Sprintf("%s/pd/api/v1/regions", apiServerAddr) + pdServiceAddr := cluster.GetLeaderServer().GetAddr() + urlPrefix := fmt.Sprintf("%s/pd/api/v1/regions", pdServiceAddr) var resp map[string]any err := testutil.ReadGetJSON(re, tests.TestDialClient, urlPrefix, &resp) re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/config_test.go b/tests/integrations/mcs/scheduling/config_test.go index d7d200814bb..6c770d3e4c1 100644 --- a/tests/integrations/mcs/scheduling/config_test.go +++ b/tests/integrations/mcs/scheduling/config_test.go @@ -62,7 +62,7 @@ func (suite *configTestSuite) SetupSuite() { schedulers.Register() var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() re.NoError(err) @@ -132,7 +132,7 @@ func (suite *configTestSuite) TestConfigWatch() { watcher.Close() } -// Manually trigger the config persistence in the PD API server side. +// Manually trigger the config persistence in the PD service side. func persistConfig(re *require.Assertions, pdLeaderServer *tests.TestServer) { err := pdLeaderServer.GetPersistOptions().Persist(pdLeaderServer.GetServer().GetStorage()) re.NoError(err) @@ -152,19 +152,19 @@ func (suite *configTestSuite) TestSchedulerConfigWatch() { ) re.NoError(err) // Get all default scheduler names. - var namesFromAPIServer []string + var namesFromPDService []string testutil.Eventually(re, func() bool { - namesFromAPIServer, _, _ = suite.pdLeaderServer.GetRaftCluster().GetStorage().LoadAllSchedulerConfigs() - return len(namesFromAPIServer) == len(sc.DefaultSchedulers) + namesFromPDService, _, _ = suite.pdLeaderServer.GetRaftCluster().GetStorage().LoadAllSchedulerConfigs() + return len(namesFromPDService) == len(sc.DefaultSchedulers) }) // Check all default schedulers' configs. var namesFromSchedulingServer []string testutil.Eventually(re, func() bool { namesFromSchedulingServer, _, err = storage.LoadAllSchedulerConfigs() re.NoError(err) - return len(namesFromSchedulingServer) == len(namesFromAPIServer) + return len(namesFromSchedulingServer) == len(namesFromPDService) }) - re.Equal(namesFromAPIServer, namesFromSchedulingServer) + re.Equal(namesFromPDService, namesFromSchedulingServer) // Add a new scheduler. api.MustAddScheduler(re, suite.pdLeaderServer.GetAddr(), types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, diff --git a/tests/integrations/mcs/scheduling/meta_test.go b/tests/integrations/mcs/scheduling/meta_test.go index 4e0d5249fdb..8df576b82ca 100644 --- a/tests/integrations/mcs/scheduling/meta_test.go +++ b/tests/integrations/mcs/scheduling/meta_test.go @@ -53,7 +53,7 @@ func (suite *metaTestSuite) SetupSuite() { re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/rule_test.go b/tests/integrations/mcs/scheduling/rule_test.go index 880dfddbb16..706c5784831 100644 --- a/tests/integrations/mcs/scheduling/rule_test.go +++ b/tests/integrations/mcs/scheduling/rule_test.go @@ -54,7 +54,7 @@ func (suite *ruleTestSuite) SetupSuite() { var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() re.NoError(err) @@ -97,7 +97,7 @@ func (suite *ruleTestSuite) TestRuleWatch() { re.Equal(placement.DefaultGroupID, ruleGroups[0].ID) re.Equal(0, ruleGroups[0].Index) re.False(ruleGroups[0].Override) - // Set a new rule via the PD API server. + // Set a new rule via the PD service. apiRuleManager := suite.pdLeaderServer.GetRaftCluster().GetRuleManager() rule := &placement.Rule{ GroupID: "2", diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 3401fb880cb..9a3d33d1dcf 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -66,7 +66,7 @@ func (suite *serverTestSuite) SetupSuite() { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/scheduling/server/changeRunCollectWaitTime", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() @@ -220,7 +220,7 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { // Change back to the default value. conf.EnableSchedulingFallback = true leaderServer.SetMicroServiceConfig(*conf) - // API server will execute scheduling jobs since there is no scheduling server. + // PD service will execute scheduling jobs since there is no scheduling server. testutil.Eventually(re, func() bool { return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) @@ -229,7 +229,7 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) - // After scheduling server is started, API server will not execute scheduling jobs. + // After scheduling server is started, PD service will not execute scheduling jobs. testutil.Eventually(re, func() bool { return !suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) @@ -238,7 +238,7 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { return tc.GetPrimaryServer().GetCluster().IsBackgroundJobsRunning() }) tc.GetPrimaryServer().Close() - // Stop scheduling server. API server will execute scheduling jobs again. + // Stop scheduling server. PD service will execute scheduling jobs again. testutil.Eventually(re, func() bool { return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) @@ -246,7 +246,7 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { re.NoError(err) defer tc1.Destroy() tc1.WaitForPrimaryServing(re) - // After scheduling server is started, API server will not execute scheduling jobs. + // After scheduling server is started, PD service will not execute scheduling jobs. testutil.Eventually(re, func() bool { return !suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) @@ -259,21 +259,21 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { func (suite *serverTestSuite) TestDisableSchedulingServiceFallback() { re := suite.Require() - // API server will execute scheduling jobs since there is no scheduling server. + // PD service will execute scheduling jobs since there is no scheduling server. testutil.Eventually(re, func() bool { re.NotNil(suite.pdLeader.GetServer()) re.NotNil(suite.pdLeader.GetServer().GetRaftCluster()) return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) leaderServer := suite.pdLeader.GetServer() - // After Disabling scheduling service fallback, the API server will stop scheduling. + // After Disabling scheduling service fallback, the PD service will stop scheduling. conf := leaderServer.GetMicroServiceConfig().Clone() conf.EnableSchedulingFallback = false leaderServer.SetMicroServiceConfig(*conf) testutil.Eventually(re, func() bool { return !suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) - // Enable scheduling service fallback again, the API server will restart scheduling. + // Enable scheduling service fallback again, the PD service will restart scheduling. conf.EnableSchedulingFallback = true leaderServer.SetMicroServiceConfig(*conf) testutil.Eventually(re, func() bool { @@ -284,7 +284,7 @@ func (suite *serverTestSuite) TestDisableSchedulingServiceFallback() { re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) - // After scheduling server is started, API server will not execute scheduling jobs. + // After scheduling server is started, PD service will not execute scheduling jobs. testutil.Eventually(re, func() bool { return !suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) @@ -292,7 +292,7 @@ func (suite *serverTestSuite) TestDisableSchedulingServiceFallback() { testutil.Eventually(re, func() bool { return tc.GetPrimaryServer().GetCluster().IsBackgroundJobsRunning() }) - // Disable scheduling service fallback and stop scheduling server. API server won't execute scheduling jobs again. + // Disable scheduling service fallback and stop scheduling server. PD service won't execute scheduling jobs again. conf.EnableSchedulingFallback = false leaderServer.SetMicroServiceConfig(*conf) tc.GetPrimaryServer().Close() @@ -310,14 +310,14 @@ func (suite *serverTestSuite) TestSchedulerSync() { tc.WaitForPrimaryServing(re) schedulersController := tc.GetPrimaryServer().GetCluster().GetCoordinator().GetSchedulersController() checkEvictLeaderSchedulerExist(re, schedulersController, false) - // Add a new evict-leader-scheduler through the API server. + // Add a new evict-leader-scheduler through the PD service. api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) // Check if the evict-leader-scheduler is added. checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1}) - // Add a store_id to the evict-leader-scheduler through the API server. + // Add a store_id to the evict-leader-scheduler through the PD service. err = suite.pdLeader.GetServer().GetRaftCluster().PutMetaStore( &metapb.Store{ Id: 2, @@ -334,18 +334,18 @@ func (suite *serverTestSuite) TestSchedulerSync() { }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) - // Delete a store_id from the evict-leader-scheduler through the API server. + // Delete a store_id from the evict-leader-scheduler through the PD service. api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", types.EvictLeaderScheduler.String(), 1)) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{2}) - // Add a store_id to the evict-leader-scheduler through the API server by the scheduler handler. + // Add a store_id to the evict-leader-scheduler through the PD service by the scheduler handler. api.MustCallSchedulerConfigAPI(re, http.MethodPost, suite.backendEndpoints, types.EvictLeaderScheduler.String(), []string{"config"}, map[string]any{ "name": types.EvictLeaderScheduler.String(), "store_id": 1, }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) - // Delete a store_id from the evict-leader-scheduler through the API server by the scheduler handler. + // Delete a store_id from the evict-leader-scheduler through the PD service by the scheduler handler. api.MustCallSchedulerConfigAPI(re, http.MethodDelete, suite.backendEndpoints, types.EvictLeaderScheduler.String(), []string{"delete", "2"}, nil) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1}) @@ -354,7 +354,7 @@ func (suite *serverTestSuite) TestSchedulerSync() { // Check if the scheduler is removed. checkEvictLeaderSchedulerExist(re, schedulersController, false) - // Delete the evict-leader-scheduler through the API server by removing the last store_id. + // Delete the evict-leader-scheduler through the PD service by removing the last store_id. api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) @@ -363,7 +363,7 @@ func (suite *serverTestSuite) TestSchedulerSync() { api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", types.EvictLeaderScheduler.String(), 1)) checkEvictLeaderSchedulerExist(re, schedulersController, false) - // Delete the evict-leader-scheduler through the API server. + // Delete the evict-leader-scheduler through the PD service. api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) @@ -551,7 +551,7 @@ func (suite *serverTestSuite) TestStoreLimit() { leaderServer.GetRaftCluster().SetStoreLimit(1, storelimit.RemovePeer, 60) leaderServer.GetRaftCluster().SetStoreLimit(2, storelimit.AddPeer, 60) leaderServer.GetRaftCluster().SetStoreLimit(2, storelimit.RemovePeer, 60) - // There is a time window between setting store limit in API service side and capturing the change in scheduling service. + // There is a time window between setting store limit in PD service side and capturing the change in scheduling service. waitSyncFinish(re, tc, storelimit.AddPeer, 60) for i := uint64(1); i <= 5; i++ { op := operator.NewTestOperator(2, &metapb.RegionEpoch{}, operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: 100}) @@ -636,7 +636,7 @@ func (suite *multipleServerTestSuite) SetupSuite() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 2) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 2) re.NoError(err) err = suite.cluster.RunInitialServers() diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index 91614530ef1..dceb5ccdf7c 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -62,7 +62,7 @@ func (suite *tsoAPITestSuite) SetupTest() { var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.pdCluster, err = tests.NewTestAPICluster(suite.ctx, 1) + suite.pdCluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) re.NoError(err) err = suite.pdCluster.RunInitialServers() re.NoError(err) @@ -137,7 +137,7 @@ func TestTSOServerStartFirst(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - apiCluster, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { + apiCluster, err := tests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = []string{"k1", "k2"} }) defer apiCluster.Destroy() @@ -200,7 +200,7 @@ func TestForwardOnlyTSONoScheduling(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tc, err := tests.NewTestAPICluster(ctx, 1) + tc, err := tests.NewTestPDServiceCluster(ctx, 1) defer tc.Destroy() re.NoError(err) err = tc.RunInitialServers() @@ -227,7 +227,7 @@ func TestForwardOnlyTSONoScheduling(t *testing.T) { testutil.StatusOK(re), testutil.StringContain(re, "Reset ts successfully"), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) re.NoError(err) - // If close tso server, it should try forward to tso server, but return error in api mode. + // If close tso server, it should try forward to tso server, but return error in pd service mode. ttc.Destroy() err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "admin/reset-ts"), input, testutil.Status(re, http.StatusInternalServerError), testutil.StringContain(re, "[PD:apiutil:ErrRedirect]redirect failed")) diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index 2c19f6588e5..ecbc0295845 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -82,7 +82,7 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) SetupSuite() { var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() re.NoError(err) @@ -537,8 +537,8 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller", `return(true)`)) - // Init api server config but not start. - tc, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { + // Init PD service config but not start. + tc, err := tests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = []string{ "keyspace_a", "keyspace_b", } @@ -546,7 +546,7 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() - // Start api server and tso server. + // Start PD service and tso server. err = tc.RunInitialServers() re.NoError(err) defer tc.Destroy() @@ -734,8 +734,8 @@ func TestGetTSOImmediately(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller", `return(true)`)) - // Init api server config but not start. - tc, err := tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { + // Init PD service config but not start. + tc, err := tests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = []string{ "keyspace_a", "keyspace_b", } @@ -743,7 +743,7 @@ func TestGetTSOImmediately(t *testing.T) { re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() - // Start api server and tso server. + // Start PD service and tso server. err = tc.RunInitialServers() re.NoError(err) defer tc.Destroy() diff --git a/tests/integrations/mcs/tso/proxy_test.go b/tests/integrations/mcs/tso/proxy_test.go index b564076c1f0..50583ebbbb4 100644 --- a/tests/integrations/mcs/tso/proxy_test.go +++ b/tests/integrations/mcs/tso/proxy_test.go @@ -62,7 +62,7 @@ func (s *tsoProxyTestSuite) SetupSuite() { var err error s.ctx, s.cancel = context.WithCancel(context.Background()) // Create an API cluster with 1 server - s.apiCluster, err = tests.NewTestAPICluster(s.ctx, 1) + s.apiCluster, err = tests.NewTestPDServiceCluster(s.ctx, 1) re.NoError(err) err = s.apiCluster.RunInitialServers() re.NoError(err) diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 09a199c2d52..7416a314949 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -75,7 +75,7 @@ func (suite *tsoServerTestSuite) SetupSuite() { re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() @@ -156,19 +156,19 @@ func (suite *tsoServerTestSuite) TestParticipantStartWithAdvertiseListenAddr() { func TestTSOPath(t *testing.T) { re := require.New(t) - checkTSOPath(re, true /*isAPIServiceMode*/) - checkTSOPath(re, false /*isAPIServiceMode*/) + checkTSOPath(re, true /*isPDServiceMode*/) + checkTSOPath(re, false /*isPDServiceMode*/) } -func checkTSOPath(re *require.Assertions, isAPIServiceMode bool) { +func checkTSOPath(re *require.Assertions, isPDServiceMode bool) { var ( cluster *tests.TestCluster err error ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - if isAPIServiceMode { - cluster, err = tests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { + if isPDServiceMode { + cluster, err = tests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { conf.MicroService.EnableTSODynamicSwitching = false }) } else { @@ -184,7 +184,7 @@ func checkTSOPath(re *require.Assertions, isAPIServiceMode bool) { re.NoError(pdLeader.BootstrapCluster()) backendEndpoints := pdLeader.GetAddr() client := pdLeader.GetEtcdClient() - if isAPIServiceMode { + if isPDServiceMode { re.Equal(0, getEtcdTimestampKeyNum(re, client)) } else { re.Equal(1, getEtcdTimestampKeyNum(re, client)) @@ -217,7 +217,7 @@ func getEtcdTimestampKeyNum(re *require.Assertions, client *clientv3.Client) int return count } -type APIServerForward struct { +type PDServiceForward struct { re *require.Assertions ctx context.Context cancel context.CancelFunc @@ -227,13 +227,13 @@ type APIServerForward struct { pdClient pd.Client } -func NewAPIServerForward(re *require.Assertions) APIServerForward { - suite := APIServerForward{ +func NewPDServiceForward(re *require.Assertions) PDServiceForward { + suite := PDServiceForward{ re: re, } var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 3) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 3) re.NoError(err) err = suite.cluster.RunInitialServers() @@ -254,7 +254,7 @@ func NewAPIServerForward(re *require.Assertions) APIServerForward { return suite } -func (suite *APIServerForward) ShutDown() { +func (suite *PDServiceForward) ShutDown() { suite.pdClient.Close() re := suite.re @@ -273,7 +273,7 @@ func (suite *APIServerForward) ShutDown() { func TestForwardTSORelated(t *testing.T) { re := require.New(t) - suite := NewAPIServerForward(re) + suite := NewPDServiceForward(re) defer suite.ShutDown() leaderServer := suite.cluster.GetLeaderServer().GetServer() cfg := leaderServer.GetMicroServiceConfig().Clone() @@ -290,7 +290,7 @@ func TestForwardTSORelated(t *testing.T) { func TestForwardTSOWhenPrimaryChanged(t *testing.T) { re := require.New(t) - suite := NewAPIServerForward(re) + suite := NewPDServiceForward(re) defer suite.ShutDown() tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) @@ -330,7 +330,7 @@ func TestForwardTSOWhenPrimaryChanged(t *testing.T) { func TestResignTSOPrimaryForward(t *testing.T) { re := require.New(t) - suite := NewAPIServerForward(re) + suite := NewPDServiceForward(re) defer suite.ShutDown() // TODO: test random kill primary with 3 nodes tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) @@ -356,7 +356,7 @@ func TestResignTSOPrimaryForward(t *testing.T) { func TestResignAPIPrimaryForward(t *testing.T) { re := require.New(t) - suite := NewAPIServerForward(re) + suite := NewPDServiceForward(re) defer suite.ShutDown() tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) @@ -380,7 +380,7 @@ func TestResignAPIPrimaryForward(t *testing.T) { func TestForwardTSOUnexpectedToFollower1(t *testing.T) { re := require.New(t) - suite := NewAPIServerForward(re) + suite := NewPDServiceForward(re) defer suite.ShutDown() suite.checkForwardTSOUnexpectedToFollower(func() { // unary call will retry internally @@ -393,7 +393,7 @@ func TestForwardTSOUnexpectedToFollower1(t *testing.T) { func TestForwardTSOUnexpectedToFollower2(t *testing.T) { re := require.New(t) - suite := NewAPIServerForward(re) + suite := NewPDServiceForward(re) defer suite.ShutDown() suite.checkForwardTSOUnexpectedToFollower(func() { // unary call will retry internally @@ -407,7 +407,7 @@ func TestForwardTSOUnexpectedToFollower2(t *testing.T) { func TestForwardTSOUnexpectedToFollower3(t *testing.T) { re := require.New(t) - suite := NewAPIServerForward(re) + suite := NewPDServiceForward(re) defer suite.ShutDown() suite.checkForwardTSOUnexpectedToFollower(func() { _, _, err := suite.pdClient.GetTS(suite.ctx) @@ -415,7 +415,7 @@ func TestForwardTSOUnexpectedToFollower3(t *testing.T) { }) } -func (suite *APIServerForward) checkForwardTSOUnexpectedToFollower(checkTSO func()) { +func (suite *PDServiceForward) checkForwardTSOUnexpectedToFollower(checkTSO func()) { re := suite.re tc, err := tests.NewTestTSOCluster(suite.ctx, 2, suite.backendEndpoints) re.NoError(err) @@ -451,7 +451,7 @@ func (suite *APIServerForward) checkForwardTSOUnexpectedToFollower(checkTSO func tc.Destroy() } -func (suite *APIServerForward) addRegions() { +func (suite *PDServiceForward) addRegions() { leader := suite.cluster.GetServer(suite.cluster.WaitLeader()) rc := leader.GetServer().GetRaftCluster() for i := range 3 { @@ -465,7 +465,7 @@ func (suite *APIServerForward) addRegions() { } } -func (suite *APIServerForward) checkUnavailableTSO(re *require.Assertions) { +func (suite *PDServiceForward) checkUnavailableTSO(re *require.Assertions) { _, _, err := suite.pdClient.GetTS(suite.ctx) re.Error(err) // try to update gc safe point @@ -476,7 +476,7 @@ func (suite *APIServerForward) checkUnavailableTSO(re *require.Assertions) { re.Error(err) } -func (suite *APIServerForward) checkAvailableTSO(re *require.Assertions) { +func (suite *PDServiceForward) checkAvailableTSO(re *require.Assertions) { mcs.WaitForTSOServiceAvailable(suite.ctx, re, suite.pdClient) // try to get ts _, _, err := suite.pdClient.GetTS(suite.ctx) @@ -512,7 +512,7 @@ func (suite *CommonTestSuite) SetupSuite() { var err error re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() @@ -576,7 +576,7 @@ func (suite *CommonTestSuite) TestBootstrapDefaultKeyspaceGroup() { } check() - s, err := suite.cluster.JoinAPIServer(suite.ctx) + s, err := suite.cluster.JoinPDServer(suite.ctx) re.NoError(err) re.NoError(s.Run()) @@ -598,7 +598,7 @@ func TestTSOServiceSwitch(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tc, err := tests.NewTestAPICluster(ctx, 1, + tc, err := tests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { conf.MicroService.EnableTSODynamicSwitching = true }, diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index 2cda9f8734f..a06e44ed4ab 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -98,7 +98,7 @@ func (suite *tsoClientTestSuite) SetupSuite() { if suite.legacy { suite.cluster, err = tests.NewTestCluster(suite.ctx, serverCount) } else { - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, serverCount, func(conf *config.Config, _ string) { + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, serverCount, func(conf *config.Config, _ string) { conf.MicroService.EnableTSODynamicSwitching = false }) } @@ -510,7 +510,7 @@ func TestMixedTSODeployment(t *testing.T) { re.NotNil(leaderServer) backendEndpoints := leaderServer.GetAddr() - apiSvr, err := cluster.JoinAPIServer(ctx) + apiSvr, err := cluster.JoinPDServer(ctx) re.NoError(err) err = apiSvr.Run() re.NoError(err) @@ -544,7 +544,7 @@ func TestUpgradingAPIandTSOClusters(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) // Create an API cluster which has 3 servers - apiCluster, err := tests.NewTestAPICluster(ctx, 3) + apiCluster, err := tests.NewTestPDServiceCluster(ctx, 3) re.NoError(err) err = apiCluster.RunInitialServers() re.NoError(err) diff --git a/tests/integrations/tso/consistency_test.go b/tests/integrations/tso/consistency_test.go index 147f41a4591..b29ae696f26 100644 --- a/tests/integrations/tso/consistency_test.go +++ b/tests/integrations/tso/consistency_test.go @@ -76,7 +76,7 @@ func (suite *tsoConsistencyTestSuite) SetupSuite() { if suite.legacy { suite.cluster, err = tests.NewTestCluster(suite.ctx, serverCount) } else { - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, serverCount) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, serverCount) } re.NoError(err) err = suite.cluster.RunInitialServers() diff --git a/tests/integrations/tso/server_test.go b/tests/integrations/tso/server_test.go index f03db197b35..1428dbcd1a6 100644 --- a/tests/integrations/tso/server_test.go +++ b/tests/integrations/tso/server_test.go @@ -74,7 +74,7 @@ func (suite *tsoServerTestSuite) SetupSuite() { if suite.legacy { suite.cluster, err = tests.NewTestCluster(suite.ctx, serverCount) } else { - suite.cluster, err = tests.NewTestAPICluster(suite.ctx, serverCount) + suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, serverCount) } re.NoError(err) err = suite.cluster.RunInitialServers() diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index 1f76c469cfd..d1d5b06ceb4 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -55,7 +55,7 @@ func TestPDSchedulingTestSuite(t *testing.T) { func TestAPISchedulingTestSuite(t *testing.T) { suite.Run(t, &scheduleTestSuite{ - runMode: tests.APIMode, + runMode: tests.PDServiceMode, }) } diff --git a/tests/server/apiv2/handlers/tso_keyspace_group_test.go b/tests/server/apiv2/handlers/tso_keyspace_group_test.go index 796fd514eef..851df9b5fd1 100644 --- a/tests/server/apiv2/handlers/tso_keyspace_group_test.go +++ b/tests/server/apiv2/handlers/tso_keyspace_group_test.go @@ -42,7 +42,7 @@ func TestKeyspaceGroupTestSuite(t *testing.T) { func (suite *keyspaceGroupTestSuite) SetupTest() { re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) - cluster, err := tests.NewTestAPICluster(suite.ctx, 1) + cluster, err := tests.NewTestPDServiceCluster(suite.ctx, 1) suite.cluster = cluster re.NoError(err) re.NoError(cluster.RunInitialServers()) diff --git a/tests/server/server_test.go b/tests/server/server_test.go index 77cd7aa5158..06623b6f092 100644 --- a/tests/server/server_test.go +++ b/tests/server/server_test.go @@ -67,7 +67,7 @@ func TestUpdateAdvertiseUrls(t *testing.T) { for _, conf := range cluster.GetConfig().InitialServers { serverConf, err := conf.Generate() re.NoError(err) - s, err := tests.NewTestServer(ctx, serverConf) + s, err := tests.NewTestServer(ctx, serverConf, nil) re.NoError(err) cluster.GetServers()[conf.Name] = s } diff --git a/tests/testutil.go b/tests/testutil.go index 5e99b3dbeda..4bbfa8155b4 100644 --- a/tests/testutil.go +++ b/tests/testutil.go @@ -279,8 +279,8 @@ const ( Both SchedulerMode = iota // PDMode represents PD mode. PDMode - // APIMode represents API mode. - APIMode + // PDServiceMode represents API mode. + PDServiceMode ) // SchedulingTestEnvironment is used for test purpose. @@ -308,11 +308,11 @@ func (s *SchedulingTestEnvironment) RunTestBasedOnMode(test func(*TestCluster)) switch s.RunMode { case PDMode: s.RunTestInPDMode(test) - case APIMode: - s.RunTestInAPIMode(test) + case PDServiceMode: + s.RunTestInPDServiceMode(test) default: s.RunTestInPDMode(test) - s.RunTestInAPIMode(test) + s.RunTestInPDServiceMode(test) } } @@ -339,8 +339,8 @@ func getTestName() string { return "" } -// RunTestInAPIMode is to run test in api mode. -func (s *SchedulingTestEnvironment) RunTestInAPIMode(test func(*TestCluster)) { +// RunTestInPDServiceMode is to run test in pd service mode. +func (s *SchedulingTestEnvironment) RunTestInPDServiceMode(test func(*TestCluster)) { re := require.New(s.t) re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember", `return(true)`)) @@ -348,11 +348,11 @@ func (s *SchedulingTestEnvironment) RunTestInAPIMode(test func(*TestCluster)) { re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember")) re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) }() - s.t.Logf("start test %s in api mode", getTestName()) - if _, ok := s.clusters[APIMode]; !ok { - s.startCluster(APIMode) + s.t.Logf("start test %s in pd service mode", getTestName()) + if _, ok := s.clusters[PDServiceMode]; !ok { + s.startCluster(PDServiceMode) } - test(s.clusters[APIMode]) + test(s.clusters[PDServiceMode]) } // Cleanup is to cleanup the environment. @@ -379,8 +379,8 @@ func (s *SchedulingTestEnvironment) startCluster(m SchedulerMode) { leaderServer := cluster.GetServer(cluster.GetLeader()) re.NoError(leaderServer.BootstrapCluster()) s.clusters[PDMode] = cluster - case APIMode: - cluster, err := NewTestAPICluster(ctx, 1, s.opts...) + case PDServiceMode: + cluster, err := NewTestPDServiceCluster(ctx, 1, s.opts...) re.NoError(err) err = cluster.RunInitialServers() re.NoError(err) @@ -398,7 +398,7 @@ func (s *SchedulingTestEnvironment) startCluster(m SchedulerMode) { testutil.Eventually(re, func() bool { return cluster.GetLeaderServer().GetServer().IsServiceIndependent(constant.SchedulingServiceName) }) - s.clusters[APIMode] = cluster + s.clusters[PDServiceMode] = cluster } } diff --git a/tools/pd-ctl/pdctl/command/config_command.go b/tools/pd-ctl/pdctl/command/config_command.go index 2e9903db550..f89c63bc51c 100644 --- a/tools/pd-ctl/pdctl/command/config_command.go +++ b/tools/pd-ctl/pdctl/command/config_command.go @@ -49,8 +49,8 @@ const ( ruleBundlePrefix = "pd/api/v1/config/placement-rule" pdServerPrefix = "pd/api/v1/config/pd-server" serviceMiddlewareConfigPrefix = "pd/api/v1/service-middleware/config" - // flagFromAPIServer has no influence for pd mode, but it is useful for us to debug in api mode. - flagFromAPIServer = "from_api_server" + // flagFromPDService has no influence for pd mode, but it is useful for us to debug in pd service mode. + flagFromPDService = "from_pd_service" ) // NewConfigCommand return a config subcommand of rootCmd @@ -81,7 +81,7 @@ func NewShowConfigCommand() *cobra.Command { sc.AddCommand(newShowReplicationModeCommand()) sc.AddCommand(NewShowServerConfigCommand()) sc.AddCommand(NewShowServiceMiddlewareConfigCommand()) - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") + sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") return sc } @@ -92,7 +92,7 @@ func NewShowAllConfigCommand() *cobra.Command { Short: "show all config of PD", Run: showAllConfigCommandFunc, } - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") + sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") return sc } @@ -103,7 +103,7 @@ func NewShowScheduleConfigCommand() *cobra.Command { Short: "show schedule config of PD", Run: showScheduleConfigCommandFunc, } - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") + sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") return sc } @@ -114,7 +114,7 @@ func NewShowReplicationConfigCommand() *cobra.Command { Short: "show replication config of PD", Run: showReplicationConfigCommandFunc, } - sc.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") + sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") return sc } @@ -528,7 +528,7 @@ func NewPlacementRulesCommand() *cobra.Command { show.Flags().String("id", "", "rule id") show.Flags().String("region", "", "region id") show.Flags().Bool("detail", false, "detailed match info for region") - show.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") + show.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") load := &cobra.Command{ Use: "load", Short: "load placement rules to a file", @@ -538,7 +538,7 @@ func NewPlacementRulesCommand() *cobra.Command { load.Flags().String("id", "", "rule id") load.Flags().String("region", "", "region id") load.Flags().String("out", "rules.json", "the filename contains rules") - load.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") + load.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") save := &cobra.Command{ Use: "save", Short: "save rules from file", @@ -554,7 +554,7 @@ func NewPlacementRulesCommand() *cobra.Command { Short: "show rule group configuration(s)", Run: showRuleGroupFunc, } - ruleGroupShow.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") + ruleGroupShow.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") ruleGroupSet := &cobra.Command{ Use: "set ", Short: "update rule group configuration", @@ -577,7 +577,7 @@ func NewPlacementRulesCommand() *cobra.Command { Run: getRuleBundle, } ruleBundleGet.Flags().String("out", "", "the output file") - ruleBundleGet.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") + ruleBundleGet.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") ruleBundleSet := &cobra.Command{ Use: "set", Short: "set rule group config and its rules from file", @@ -596,7 +596,7 @@ func NewPlacementRulesCommand() *cobra.Command { Run: loadRuleBundle, } ruleBundleLoad.Flags().String("out", "rules.json", "the output file") - ruleBundleLoad.Flags().Bool(flagFromAPIServer, false, "read data from api server rather than micro service") + ruleBundleLoad.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") ruleBundleSave := &cobra.Command{ Use: "save", Short: "save all group configs and rules from file", @@ -895,7 +895,7 @@ func saveRuleBundle(cmd *cobra.Command, _ []string) { func buildHeader(cmd *cobra.Command) http.Header { header := http.Header{} - forbiddenRedirectToMicroService, err := cmd.Flags().GetBool(flagFromAPIServer) + forbiddenRedirectToMicroService, err := cmd.Flags().GetBool(flagFromPDService) if err == nil && forbiddenRedirectToMicroService { header.Add(apiutil.XForbiddenForwardToMicroServiceHeader, "true") } diff --git a/tools/pd-ctl/tests/config/config_test.go b/tools/pd-ctl/tests/config/config_test.go index b6c58fe2bc6..39820a6c7b7 100644 --- a/tools/pd-ctl/tests/config/config_test.go +++ b/tools/pd-ctl/tests/config/config_test.go @@ -383,9 +383,9 @@ func (suite *configTestSuite) checkConfigForwardControl(cluster *pdTests.TestClu f.Close() defer os.RemoveAll(fname) - checkScheduleConfig := func(scheduleCfg *sc.ScheduleConfig, isFromAPIServer bool) { + checkScheduleConfig := func(scheduleCfg *sc.ScheduleConfig, isFromPDService bool) { if schedulingServer := cluster.GetSchedulingPrimaryServer(); schedulingServer != nil { - if isFromAPIServer { + if isFromPDService { re.Equal(scheduleCfg.LeaderScheduleLimit, leaderServer.GetPersistOptions().GetLeaderScheduleLimit()) re.NotEqual(scheduleCfg.LeaderScheduleLimit, schedulingServer.GetPersistConfig().GetLeaderScheduleLimit()) } else { @@ -397,9 +397,9 @@ func (suite *configTestSuite) checkConfigForwardControl(cluster *pdTests.TestClu } } - checkReplicateConfig := func(replicationCfg *sc.ReplicationConfig, isFromAPIServer bool) { + checkReplicateConfig := func(replicationCfg *sc.ReplicationConfig, isFromPDService bool) { if schedulingServer := cluster.GetSchedulingPrimaryServer(); schedulingServer != nil { - if isFromAPIServer { + if isFromPDService { re.Equal(replicationCfg.MaxReplicas, uint64(leaderServer.GetPersistOptions().GetMaxReplicas())) re.NotEqual(int(replicationCfg.MaxReplicas), schedulingServer.GetPersistConfig().GetMaxReplicas()) } else { @@ -411,11 +411,11 @@ func (suite *configTestSuite) checkConfigForwardControl(cluster *pdTests.TestClu } } - checkRules := func(rules []*placement.Rule, isFromAPIServer bool) { + checkRules := func(rules []*placement.Rule, isFromPDService bool) { apiRules := leaderServer.GetRaftCluster().GetRuleManager().GetAllRules() if schedulingServer := cluster.GetSchedulingPrimaryServer(); schedulingServer != nil { schedulingRules := schedulingServer.GetCluster().GetRuleManager().GetAllRules() - if isFromAPIServer { + if isFromPDService { re.Len(apiRules, len(rules)) re.NotEqual(len(schedulingRules), len(rules)) } else { @@ -427,11 +427,11 @@ func (suite *configTestSuite) checkConfigForwardControl(cluster *pdTests.TestClu } } - checkGroup := func(group placement.RuleGroup, isFromAPIServer bool) { + checkGroup := func(group placement.RuleGroup, isFromPDService bool) { apiGroup := leaderServer.GetRaftCluster().GetRuleManager().GetRuleGroup(placement.DefaultGroupID) if schedulingServer := cluster.GetSchedulingPrimaryServer(); schedulingServer != nil { schedulingGroup := schedulingServer.GetCluster().GetRuleManager().GetRuleGroup(placement.DefaultGroupID) - if isFromAPIServer { + if isFromPDService { re.Equal(apiGroup.Index, group.Index) re.NotEqual(schedulingGroup.Index, group.Index) } else { @@ -444,28 +444,28 @@ func (suite *configTestSuite) checkConfigForwardControl(cluster *pdTests.TestClu } testConfig := func(options ...string) { - for _, isFromAPIServer := range []bool{true, false} { + for _, isFromPDService := range []bool{true, false} { cmd := ctl.GetRootCmd() args := []string{"-u", pdAddr, "config", "show"} args = append(args, options...) - if isFromAPIServer { - args = append(args, "--from_api_server") + if isFromPDService { + args = append(args, "--from_pd_service") } output, err := tests.ExecuteCommand(cmd, args...) re.NoError(err) if len(options) == 0 || options[0] == "all" { cfg := config.Config{} re.NoError(json.Unmarshal(output, &cfg)) - checkReplicateConfig(&cfg.Replication, isFromAPIServer) - checkScheduleConfig(&cfg.Schedule, isFromAPIServer) + checkReplicateConfig(&cfg.Replication, isFromPDService) + checkScheduleConfig(&cfg.Schedule, isFromPDService) } else if options[0] == "replication" { replicationCfg := &sc.ReplicationConfig{} re.NoError(json.Unmarshal(output, replicationCfg)) - checkReplicateConfig(replicationCfg, isFromAPIServer) + checkReplicateConfig(replicationCfg, isFromPDService) } else if options[0] == "schedule" { scheduleCfg := &sc.ScheduleConfig{} re.NoError(json.Unmarshal(output, scheduleCfg)) - checkScheduleConfig(scheduleCfg, isFromAPIServer) + checkScheduleConfig(scheduleCfg, isFromPDService) } else { re.Fail("no implement") } @@ -473,37 +473,37 @@ func (suite *configTestSuite) checkConfigForwardControl(cluster *pdTests.TestClu } testRules := func(options ...string) { - for _, isFromAPIServer := range []bool{true, false} { + for _, isFromPDService := range []bool{true, false} { cmd := ctl.GetRootCmd() args := []string{"-u", pdAddr, "config", "placement-rules"} args = append(args, options...) - if isFromAPIServer { - args = append(args, "--from_api_server") + if isFromPDService { + args = append(args, "--from_pd_service") } output, err := tests.ExecuteCommand(cmd, args...) re.NoError(err) if options[0] == "show" { var rules []*placement.Rule re.NoError(json.Unmarshal(output, &rules)) - checkRules(rules, isFromAPIServer) + checkRules(rules, isFromPDService) } else if options[0] == "load" { var rules []*placement.Rule b, _ := os.ReadFile(fname) re.NoError(json.Unmarshal(b, &rules)) - checkRules(rules, isFromAPIServer) + checkRules(rules, isFromPDService) } else if options[0] == "rule-group" { var group placement.RuleGroup re.NoError(json.Unmarshal(output, &group), string(output)) - checkGroup(group, isFromAPIServer) + checkGroup(group, isFromPDService) } else if options[0] == "rule-bundle" && options[1] == "get" { var bundle placement.GroupBundle re.NoError(json.Unmarshal(output, &bundle), string(output)) - checkRules(bundle.Rules, isFromAPIServer) + checkRules(bundle.Rules, isFromPDService) } else if options[0] == "rule-bundle" && options[1] == "load" { var bundles []placement.GroupBundle b, _ := os.ReadFile(fname) re.NoError(json.Unmarshal(b, &bundles), string(output)) - checkRules(bundles[0].Rules, isFromAPIServer) + checkRules(bundles[0].Rules, isFromPDService) } else { re.Fail("no implement") } @@ -522,13 +522,13 @@ func (suite *configTestSuite) checkConfigForwardControl(cluster *pdTests.TestClu re.Equal(uint64(233), sche.GetPersistConfig().GetLeaderScheduleLimit()) re.Equal(7, sche.GetPersistConfig().GetMaxReplicas()) } - // show config from api server rather than scheduling server + // show config from PD service rather than scheduling server testConfig() - // show all config from api server rather than scheduling server + // show all config from PD service rather than scheduling server testConfig("all") - // show replication config from api server rather than scheduling server + // show replication config from PD service rather than scheduling server testConfig("replication") - // show schedule config from api server rather than scheduling server + // show schedule config from PD service rather than scheduling server testConfig("schedule") // Test Rule diff --git a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go index fca00f2fd3c..fff95856931 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go @@ -41,7 +41,7 @@ func TestKeyspaceGroup(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tc, err := pdTests.NewTestAPICluster(ctx, 1) + tc, err := pdTests.NewTestPDServiceCluster(ctx, 1) re.NoError(err) defer tc.Destroy() err = tc.RunInitialServers() @@ -102,7 +102,7 @@ func TestSplitKeyspaceGroup(t *testing.T) { for i := range 129 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 3, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestPDServiceCluster(ctx, 3, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -157,7 +157,7 @@ func TestExternalAllocNodeWhenStart(t *testing.T) { for i := range 10 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -197,7 +197,7 @@ func TestSetNodeAndPriorityKeyspaceGroup(t *testing.T) { for i := range 10 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 3, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestPDServiceCluster(ctx, 3, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -301,7 +301,7 @@ func TestMergeKeyspaceGroup(t *testing.T) { for i := range 129 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -420,7 +420,7 @@ func TestKeyspaceGroupState(t *testing.T) { for i := range 10 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -511,7 +511,7 @@ func TestShowKeyspaceGroupPrimary(t *testing.T) { for i := range 10 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 1, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) diff --git a/tools/pd-ctl/tests/keyspace/keyspace_test.go b/tools/pd-ctl/tests/keyspace/keyspace_test.go index 23a1148cd66..6a523ced7b8 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_test.go @@ -49,7 +49,7 @@ func TestKeyspace(t *testing.T) { for i := 1; i < 10; i++ { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestAPICluster(ctx, 3, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestPDServiceCluster(ctx, 3, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -155,7 +155,7 @@ func (suite *keyspaceTestSuite) SetupTest() { suite.ctx, suite.cancel = context.WithCancel(context.Background()) re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) - tc, err := pdTests.NewTestAPICluster(suite.ctx, 1) + tc, err := pdTests.NewTestPDServiceCluster(suite.ctx, 1) re.NoError(err) re.NoError(tc.RunInitialServers()) tc.WaitLeader() diff --git a/tools/pd-simulator/simulator/drive.go b/tools/pd-simulator/simulator/drive.go index 0d81a2af1ab..d8a6094760e 100644 --- a/tools/pd-simulator/simulator/drive.go +++ b/tools/pd-simulator/simulator/drive.go @@ -165,7 +165,7 @@ func (d *Driver) allocID() error { func (d *Driver) updateNodesClient() error { urls := strings.Split(d.pdAddr, ",") ctx, cancel := context.WithCancel(context.Background()) - SD = sd.NewDefaultPDServiceDiscovery(ctx, cancel, urls, nil) + SD = sd.NewDefaultServiceDiscovery(ctx, cancel, urls, nil) if err := SD.Init(); err != nil { return err } From adddd4eaa62dcff0c0310b36a8d3687894a4bd5a Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 8 Jan 2025 20:12:37 +0800 Subject: [PATCH 10/33] *: fix typo (#8982) ref tikv/pd#4820 Signed-off-by: Ryan Leung --- .github/workflows/pd-tests.yaml | 4 +- client/http/api.go | 16 +-- client/http/interface.go | 24 ++-- client/http/request_info.go | 4 +- client/http/types.go | 4 +- pkg/errs/errno.go | 2 +- pkg/member/election_leader.go | 2 +- pkg/schedule/placement/rule_manager.go | 2 +- pkg/utils/apiutil/apiutil.go | 8 +- pkg/utils/apiutil/serverapi/middleware.go | 16 +-- pkg/utils/keypath/absolute_key_path.go | 2 +- server/api/config.go | 14 +-- server/apiv2/handlers/micro_service.go | 8 +- server/apiv2/router.go | 2 +- server/cluster/cluster.go | 4 +- server/config/config.go | 18 +-- server/config/persist_options.go | 20 ++-- server/server.go | 28 ++--- tests/integrations/mcs/members/member_test.go | 36 +++--- tests/integrations/mcs/scheduling/api_test.go | 112 +++++++++--------- .../mcs/scheduling/server_test.go | 12 +- tests/integrations/mcs/tso/api_test.go | 8 +- tests/integrations/mcs/tso/server_test.go | 16 +-- tests/integrations/tso/client_test.go | 2 +- tools/pd-ctl/pdctl/command/config_command.go | 24 ++-- tools/pd-ctl/pdctl/command/global.go | 6 +- tools/pd-ctl/tests/config/config_test.go | 12 +- 27 files changed, 203 insertions(+), 203 deletions(-) diff --git a/.github/workflows/pd-tests.yaml b/.github/workflows/pd-tests.yaml index 1464c696db3..2379f67d759 100644 --- a/.github/workflows/pd-tests.yaml +++ b/.github/workflows/pd-tests.yaml @@ -43,9 +43,9 @@ jobs: - worker_id: 8 name: 'TSO Integration Test' - worker_id: 9 - name: 'MicroService Integration(!TSO)' + name: 'Microservice Integration(!TSO)' - worker_id: 10 - name: 'MicroService Integration(TSO)' + name: 'Microservice Integration(TSO)' outputs: job-total: 10 steps: diff --git a/client/http/api.go b/client/http/api.go index 42d965ec17c..028af858904 100644 --- a/client/http/api.go +++ b/client/http/api.go @@ -80,8 +80,8 @@ const ( Version = "/pd/api/v1/version" operators = "/pd/api/v1/operators" safepoint = "/pd/api/v1/gc/safepoint" - // Micro Service - microServicePrefix = "/pd/api/v2/ms" + // Microservice + microservicePrefix = "/pd/api/v2/ms" // Keyspace KeyspaceConfig = "/pd/api/v2/keyspaces/%s/config" GetKeyspaceMetaByName = "/pd/api/v2/keyspaces/%s" @@ -198,14 +198,14 @@ func PProfGoroutineWithDebugLevel(level int) string { return fmt.Sprintf("%s?debug=%d", PProfGoroutine, level) } -// MicroServiceMembers returns the path of PD HTTP API to get the members of microservice. -func MicroServiceMembers(service string) string { - return fmt.Sprintf("%s/members/%s", microServicePrefix, service) +// MicroserviceMembers returns the path of PD HTTP API to get the members of microservice. +func MicroserviceMembers(service string) string { + return fmt.Sprintf("%s/members/%s", microservicePrefix, service) } -// MicroServicePrimary returns the path of PD HTTP API to get the primary of microservice. -func MicroServicePrimary(service string) string { - return fmt.Sprintf("%s/primary/%s", microServicePrefix, service) +// MicroservicePrimary returns the path of PD HTTP API to get the primary of microservice. +func MicroservicePrimary(service string) string { + return fmt.Sprintf("%s/primary/%s", microservicePrefix, service) } // GetUpdateKeyspaceConfigURL returns the path of PD HTTP API to update keyspace config. diff --git a/client/http/interface.go b/client/http/interface.go index 1aabd1ae331..3008824edb8 100644 --- a/client/http/interface.go +++ b/client/http/interface.go @@ -103,9 +103,9 @@ type Client interface { GetPDVersion(context.Context) (string, error) GetGCSafePoint(context.Context) (ListServiceGCSafepoint, error) DeleteGCSafePoint(context.Context, string) (string, error) - /* Micro Service interfaces */ - GetMicroServiceMembers(context.Context, string) ([]MicroServiceMember, error) - GetMicroServicePrimary(context.Context, string) (string, error) + /* Microservice interfaces */ + GetMicroserviceMembers(context.Context, string) ([]MicroserviceMember, error) + GetMicroservicePrimary(context.Context, string) (string, error) DeleteOperators(context.Context) error /* Keyspace interface */ @@ -937,12 +937,12 @@ func (c *client) GetMinResolvedTSByStoresIDs(ctx context.Context, storeIDs []uin return resp.MinResolvedTS, resp.StoresMinResolvedTS, nil } -// GetMicroServiceMembers gets the members of the microservice. -func (c *client) GetMicroServiceMembers(ctx context.Context, service string) ([]MicroServiceMember, error) { - var members []MicroServiceMember +// GetMicroserviceMembers gets the members of the microservice. +func (c *client) GetMicroserviceMembers(ctx context.Context, service string) ([]MicroserviceMember, error) { + var members []MicroserviceMember err := c.request(ctx, newRequestInfo(). - WithName(getMicroServiceMembersName). - WithURI(MicroServiceMembers(service)). + WithName(getMicroserviceMembersName). + WithURI(MicroserviceMembers(service)). WithMethod(http.MethodGet). WithResp(&members)) if err != nil { @@ -951,12 +951,12 @@ func (c *client) GetMicroServiceMembers(ctx context.Context, service string) ([] return members, nil } -// GetMicroServicePrimary gets the primary of the microservice. -func (c *client) GetMicroServicePrimary(ctx context.Context, service string) (string, error) { +// GetMicroservicePrimary gets the primary of the microservice. +func (c *client) GetMicroservicePrimary(ctx context.Context, service string) (string, error) { var primary string err := c.request(ctx, newRequestInfo(). - WithName(getMicroServicePrimaryName). - WithURI(MicroServicePrimary(service)). + WithName(getMicroservicePrimaryName). + WithURI(MicroservicePrimary(service)). WithMethod(http.MethodGet). WithResp(&primary)) return primary, err diff --git a/client/http/request_info.go b/client/http/request_info.go index d1930800304..2cc95e0215a 100644 --- a/client/http/request_info.go +++ b/client/http/request_info.go @@ -76,8 +76,8 @@ const ( accelerateScheduleName = "AccelerateSchedule" accelerateScheduleInBatchName = "AccelerateScheduleInBatch" getMinResolvedTSByStoresIDsName = "GetMinResolvedTSByStoresIDs" - getMicroServiceMembersName = "GetMicroServiceMembers" - getMicroServicePrimaryName = "GetMicroServicePrimary" + getMicroserviceMembersName = "GetMicroserviceMembers" + getMicroservicePrimaryName = "GetMicroservicePrimary" getPDVersionName = "GetPDVersion" resetTSName = "ResetTS" resetBaseAllocIDName = "ResetBaseAllocID" diff --git a/client/http/types.go b/client/http/types.go index 83e8badf334..75e9db127ff 100644 --- a/client/http/types.go +++ b/client/http/types.go @@ -605,8 +605,8 @@ type MembersInfo struct { EtcdLeader *pdpb.Member `json:"etcd_leader,omitempty"` } -// MicroServiceMember is the member info of a micro service. -type MicroServiceMember struct { +// MicroserviceMember is the member info of a microservice. +type MicroserviceMember struct { ServiceAddr string `json:"service-addr"` Version string `json:"version"` GitHash string `json:"git-hash"` diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go index 834bf4f824e..25f69af327f 100644 --- a/pkg/errs/errno.go +++ b/pkg/errs/errno.go @@ -542,7 +542,7 @@ var ( ErrInvalidGroup = errors.Normalize("invalid group settings, please check the group name, priority and the number of resources", errors.RFCCodeText("PD:resourcemanager:ErrInvalidGroup")) ) -// Micro service errors +// Microservice errors var ( ErrNotFoundSchedulingPrimary = errors.Normalize("cannot find scheduling primary", errors.RFCCodeText("PD:mcs:ErrNotFoundSchedulingPrimary")) ErrSchedulingServer = errors.Normalize("scheduling server meets %v", errors.RFCCodeText("PD:mcs:ErrSchedulingServer")) diff --git a/pkg/member/election_leader.go b/pkg/member/election_leader.go index 2e5769d7dc4..5cdc7b4cd9b 100644 --- a/pkg/member/election_leader.go +++ b/pkg/member/election_leader.go @@ -21,7 +21,7 @@ import ( ) // ElectionLeader defines the common interface of the leader, which is the pdpb.Member -// for in PD/PD service or the tsopb.Participant in the micro services. +// for in PD/PD service or the tsopb.Participant in the microservices. type ElectionLeader interface { // GetListenUrls returns the listen urls GetListenUrls() []string diff --git a/pkg/schedule/placement/rule_manager.go b/pkg/schedule/placement/rule_manager.go index 4470ff28424..7dc24de0f60 100644 --- a/pkg/schedule/placement/rule_manager.go +++ b/pkg/schedule/placement/rule_manager.go @@ -88,7 +88,7 @@ func (m *RuleManager) Initialize(maxReplica int, locationLabels []string, isolat if m.initialized { return nil } - // If RuleManager is initialized in micro service, + // If RuleManager is initialized in microservice, // it will load from etcd watcher and do not modify rule directly. if skipLoadRules { m.ruleList = ruleList{ diff --git a/pkg/utils/apiutil/apiutil.go b/pkg/utils/apiutil/apiutil.go index a743c543468..0b064b6d91a 100644 --- a/pkg/utils/apiutil/apiutil.go +++ b/pkg/utils/apiutil/apiutil.go @@ -64,10 +64,10 @@ const ( XRealIPHeader = "X-Real-Ip" // XCallerIDHeader is used to mark the caller ID. XCallerIDHeader = "X-Caller-ID" - // XForbiddenForwardToMicroServiceHeader is used to indicate that forwarding the request to a microservice is explicitly disallowed. - XForbiddenForwardToMicroServiceHeader = "X-Forbidden-Forward-To-MicroService" - // XForwardedToMicroServiceHeader is used to signal that the request has already been forwarded to a microservice. - XForwardedToMicroServiceHeader = "X-Forwarded-To-MicroService" + // XForbiddenForwardToMicroserviceHeader is used to indicate that forwarding the request to a microservice is explicitly disallowed. + XForbiddenForwardToMicroserviceHeader = "X-Forbidden-Forward-To-Microservice" + // XForwardedToMicroserviceHeader is used to signal that the request has already been forwarded to a microservice. + XForwardedToMicroserviceHeader = "X-Forwarded-To-Microservice" chunkSize = 4096 ) diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index 823deed64ea..0b03c787e6b 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -115,14 +115,14 @@ func MicroserviceRedirectRule(matchPath, targetPath, targetServiceName string, } } -func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, string) { +func (h *redirector) matchMicroserviceRedirectRules(r *http.Request) (bool, string) { if !h.s.IsPDServiceMode() { return false, "" } if len(h.microserviceRedirectRules) == 0 { return false, "" } - if r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) == "true" { + if r.Header.Get(apiutil.XForbiddenForwardToMicroserviceHeader) == "true" { return false, "" } // Remove trailing '/' from the URL path @@ -166,7 +166,7 @@ func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, stri } else { r.URL.Path = rule.targetPath } - log.Debug("redirect to micro service", zap.String("path", r.URL.Path), zap.String("origin-path", origin), + log.Debug("redirect to microservice", zap.String("path", r.URL.Path), zap.String("origin-path", origin), zap.String("target", addr), zap.String("method", r.Method)) return true, addr } @@ -175,7 +175,7 @@ func (h *redirector) matchMicroServiceRedirectRules(r *http.Request) (bool, stri } func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http.HandlerFunc) { - redirectToMicroService, targetAddr := h.matchMicroServiceRedirectRules(r) + redirectToMicroservice, targetAddr := h.matchMicroserviceRedirectRules(r) allowFollowerHandle := len(r.Header.Get(apiutil.PDAllowFollowerHandleHeader)) > 0 if h.s.IsClosed() { @@ -183,7 +183,7 @@ func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http return } - if (allowFollowerHandle || h.s.GetMember().IsLeader()) && !redirectToMicroService { + if (allowFollowerHandle || h.s.GetMember().IsLeader()) && !redirectToMicroservice { next(w, r) return } @@ -200,14 +200,14 @@ func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http } var clientUrls []string - if redirectToMicroService { + if redirectToMicroservice { if len(targetAddr) == 0 { http.Error(w, errs.ErrRedirect.FastGenByArgs().Error(), http.StatusInternalServerError) return } clientUrls = append(clientUrls, targetAddr) - // Add a header to the response, it is used to mark whether the request has been forwarded to the micro service. - w.Header().Add(apiutil.XForwardedToMicroServiceHeader, "true") + // Add a header to the response, it is used to mark whether the request has been forwarded to the microservice. + w.Header().Add(apiutil.XForwardedToMicroserviceHeader, "true") } else if name := r.Header.Get(apiutil.PDRedirectorHeader); len(name) == 0 { leader := h.waitForLeader(r) // The leader has not been elected yet. diff --git a/pkg/utils/keypath/absolute_key_path.go b/pkg/utils/keypath/absolute_key_path.go index 40692dc14bd..c3acbe6ea9c 100644 --- a/pkg/utils/keypath/absolute_key_path.go +++ b/pkg/utils/keypath/absolute_key_path.go @@ -44,7 +44,7 @@ const ( msTsoKespaceExpectedLeaderPathFormat = "/ms/%d/tso/keyspace_groups/election/%05d/primary/expected_primary" // "/ms/{cluster_id}/tso/keyspace_groups/election/{group_id}/primary" ) -// MsParam is the parameter of micro service. +// MsParam is the parameter of microservice. type MsParam struct { ServiceName string GroupID uint32 // only used for tso keyspace group diff --git a/server/api/config.go b/server/api/config.go index a27a1ed5e9b..ff4fe0add82 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -65,7 +65,7 @@ func newConfHandler(svr *server.Server, rd *render.Render) *confHandler { func (h *confHandler) GetConfig(w http.ResponseWriter, r *http.Request) { cfg := h.svr.GetConfig() if h.svr.IsServiceIndependent(constant.SchedulingServiceName) && - r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { + r.Header.Get(apiutil.XForbiddenForwardToMicroserviceHeader) != "true" { schedulingServerConfig, err := h.getSchedulingServerConfig() if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) @@ -188,7 +188,7 @@ func (h *confHandler) updateConfig(cfg *config.Config, key string, value any) er case "keyspace": return h.updateKeyspaceConfig(cfg, kp[len(kp)-1], value) case "micro-service": - return h.updateMicroServiceConfig(cfg, kp[len(kp)-1], value) + return h.updateMicroserviceConfig(cfg, kp[len(kp)-1], value) } return errors.Errorf("config prefix %s not found", kp[0]) } @@ -209,8 +209,8 @@ func (h *confHandler) updateKeyspaceConfig(config *config.Config, key string, va return err } -func (h *confHandler) updateMicroServiceConfig(config *config.Config, key string, value any) error { - updated, found, err := jsonutil.AddKeyValue(&config.MicroService, key, value) +func (h *confHandler) updateMicroserviceConfig(config *config.Config, key string, value any) error { + updated, found, err := jsonutil.AddKeyValue(&config.Microservice, key, value) if err != nil { return err } @@ -220,7 +220,7 @@ func (h *confHandler) updateMicroServiceConfig(config *config.Config, key string } if updated { - err = h.svr.SetMicroServiceConfig(config.MicroService) + err = h.svr.SetMicroserviceConfig(config.Microservice) } return err } @@ -339,7 +339,7 @@ func getConfigMap(cfg map[string]any, key []string, value any) map[string]any { // @Router /config/schedule [get] func (h *confHandler) GetScheduleConfig(w http.ResponseWriter, r *http.Request) { if h.svr.IsServiceIndependent(constant.SchedulingServiceName) && - r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { + r.Header.Get(apiutil.XForbiddenForwardToMicroserviceHeader) != "true" { cfg, err := h.getSchedulingServerConfig() if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) @@ -412,7 +412,7 @@ func (h *confHandler) SetScheduleConfig(w http.ResponseWriter, r *http.Request) // @Router /config/replicate [get] func (h *confHandler) GetReplicationConfig(w http.ResponseWriter, r *http.Request) { if h.svr.IsServiceIndependent(constant.SchedulingServiceName) && - r.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) != "true" { + r.Header.Get(apiutil.XForbiddenForwardToMicroserviceHeader) != "true" { cfg, err := h.getSchedulingServerConfig() if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) diff --git a/server/apiv2/handlers/micro_service.go b/server/apiv2/handlers/micro_service.go index b4d3d6bbe89..3a0d6a2c2a6 100644 --- a/server/apiv2/handlers/micro_service.go +++ b/server/apiv2/handlers/micro_service.go @@ -24,8 +24,8 @@ import ( "github.com/tikv/pd/server/apiv2/middlewares" ) -// RegisterMicroService registers microservice handler to the router. -func RegisterMicroService(r *gin.RouterGroup) { +// RegisterMicroservice registers microservice handler to the router. +func RegisterMicroservice(r *gin.RouterGroup) { router := r.Group("ms") router.GET("members/:service", GetMembers) router.GET("primary/:service", GetPrimary) @@ -40,7 +40,7 @@ func RegisterMicroService(r *gin.RouterGroup) { func GetMembers(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) if !svr.IsPDServiceMode() { - c.AbortWithStatusJSON(http.StatusNotFound, "not support micro service") + c.AbortWithStatusJSON(http.StatusNotFound, "not support microservice") return } @@ -66,7 +66,7 @@ func GetMembers(c *gin.Context) { func GetPrimary(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) if !svr.IsPDServiceMode() { - c.AbortWithStatusJSON(http.StatusNotFound, "not support micro service") + c.AbortWithStatusJSON(http.StatusNotFound, "not support microservice") return } diff --git a/server/apiv2/router.go b/server/apiv2/router.go index 7f927dd9705..8df34e0a742 100644 --- a/server/apiv2/router.go +++ b/server/apiv2/router.go @@ -56,6 +56,6 @@ func NewV2Handler(_ context.Context, svr *server.Server) (http.Handler, apiutil. root.GET("ready", handlers.Ready) handlers.RegisterKeyspace(root) handlers.RegisterTSOKeyspaceGroup(root) - handlers.RegisterMicroService(root) + handlers.RegisterMicroservice(root) return router, group, nil } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index e482b7b8b68..4ad97dfd0cc 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -406,7 +406,7 @@ func (c *RaftCluster) Start(s Server, bootstrap bool) (err error) { func (c *RaftCluster) checkSchedulingService() { if c.isPDServiceMode { servers, err := discovery.Discover(c.etcdClient, constant.SchedulingServiceName) - if c.opt.GetMicroServiceConfig().IsSchedulingFallbackEnabled() && (err != nil || len(servers) == 0) { + if c.opt.GetMicroserviceConfig().IsSchedulingFallbackEnabled() && (err != nil || len(servers) == 0) { c.startSchedulingJobs(c, c.hbstreams) c.UnsetServiceIndependent(constant.SchedulingServiceName) } else { @@ -426,7 +426,7 @@ func (c *RaftCluster) checkSchedulingService() { // checkTSOService checks the TSO service. func (c *RaftCluster) checkTSOService() { if c.isPDServiceMode { - if c.opt.GetMicroServiceConfig().IsTSODynamicSwitchingEnabled() { + if c.opt.GetMicroserviceConfig().IsTSODynamicSwitchingEnabled() { servers, err := discovery.Discover(c.etcdClient, constant.TSOServiceName) if err != nil || len(servers) == 0 { if err := c.startTSOJobsIfNeeded(); err != nil { diff --git a/server/config/config.go b/server/config/config.go index 69cd76409bc..3d0d16bc376 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -162,7 +162,7 @@ type Config struct { Keyspace KeyspaceConfig `toml:"keyspace" json:"keyspace"` - MicroService MicroServiceConfig `toml:"micro-service" json:"micro-service"` + Microservice MicroserviceConfig `toml:"micro-service" json:"micro-service"` Controller rm.ControllerConfig `toml:"controller" json:"controller"` } @@ -457,7 +457,7 @@ func (c *Config) Adjust(meta *toml.MetaData, reloading bool) error { c.Keyspace.adjust(configMetaData.Child("keyspace")) - c.MicroService.adjust(configMetaData.Child("micro-service")) + c.Microservice.adjust(configMetaData.Child("micro-service")) if err := c.Security.Encryption.Adjust(); err != nil { return err @@ -839,13 +839,13 @@ func (c *DRAutoSyncReplicationConfig) adjust(meta *configutil.ConfigMetaData) { } } -// MicroServiceConfig is the configuration for micro service. -type MicroServiceConfig struct { +// MicroserviceConfig is the configuration for microservice. +type MicroserviceConfig struct { EnableSchedulingFallback bool `toml:"enable-scheduling-fallback" json:"enable-scheduling-fallback,string"` EnableTSODynamicSwitching bool `toml:"enable-tso-dynamic-switching" json:"enable-tso-dynamic-switching,string"` } -func (c *MicroServiceConfig) adjust(meta *configutil.ConfigMetaData) { +func (c *MicroserviceConfig) adjust(meta *configutil.ConfigMetaData) { if !meta.IsDefined("enable-scheduling-fallback") { c.EnableSchedulingFallback = defaultEnableSchedulingFallback } @@ -854,19 +854,19 @@ func (c *MicroServiceConfig) adjust(meta *configutil.ConfigMetaData) { } } -// Clone returns a copy of micro service config. -func (c *MicroServiceConfig) Clone() *MicroServiceConfig { +// Clone returns a copy of microservice config. +func (c *MicroserviceConfig) Clone() *MicroserviceConfig { cfg := *c return &cfg } // IsSchedulingFallbackEnabled returns whether to enable scheduling service fallback to PD service. -func (c *MicroServiceConfig) IsSchedulingFallbackEnabled() bool { +func (c *MicroserviceConfig) IsSchedulingFallbackEnabled() bool { return c.EnableSchedulingFallback } // IsTSODynamicSwitchingEnabled returns whether to enable TSO dynamic switching. -func (c *MicroServiceConfig) IsTSODynamicSwitchingEnabled() bool { +func (c *MicroserviceConfig) IsTSODynamicSwitchingEnabled() bool { return c.EnableTSODynamicSwitching } diff --git a/server/config/persist_options.go b/server/config/persist_options.go index 59d42383743..5ce24ac3f56 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -55,7 +55,7 @@ type PersistOptions struct { replicationMode atomic.Value labelProperty atomic.Value keyspace atomic.Value - microService atomic.Value + microservice atomic.Value storeConfig atomic.Value clusterVersion unsafe.Pointer } @@ -69,7 +69,7 @@ func NewPersistOptions(cfg *Config) *PersistOptions { o.replicationMode.Store(&cfg.ReplicationMode) o.labelProperty.Store(cfg.LabelProperty) o.keyspace.Store(&cfg.Keyspace) - o.microService.Store(&cfg.MicroService) + o.microservice.Store(&cfg.Microservice) // storeConfig will be fetched from TiKV later, // set it to an empty config here first. o.storeConfig.Store(&sc.StoreConfig{}) @@ -138,14 +138,14 @@ func (o *PersistOptions) SetKeyspaceConfig(cfg *KeyspaceConfig) { o.keyspace.Store(cfg) } -// GetMicroServiceConfig returns the micro service configuration. -func (o *PersistOptions) GetMicroServiceConfig() *MicroServiceConfig { - return o.microService.Load().(*MicroServiceConfig) +// GetMicroserviceConfig returns the microservice configuration. +func (o *PersistOptions) GetMicroserviceConfig() *MicroserviceConfig { + return o.microservice.Load().(*MicroserviceConfig) } -// SetMicroServiceConfig sets the micro service configuration. -func (o *PersistOptions) SetMicroServiceConfig(cfg *MicroServiceConfig) { - o.microService.Store(cfg) +// SetMicroserviceConfig sets the microservice configuration. +func (o *PersistOptions) SetMicroserviceConfig(cfg *MicroserviceConfig) { + o.microservice.Store(cfg) } // GetStoreConfig returns the store config. @@ -791,7 +791,7 @@ func (o *PersistOptions) Persist(storage endpoint.ConfigStorage) error { ReplicationMode: *o.GetReplicationModeConfig(), LabelProperty: o.GetLabelPropertyConfig(), Keyspace: *o.GetKeyspaceConfig(), - MicroService: *o.GetMicroServiceConfig(), + Microservice: *o.GetMicroserviceConfig(), ClusterVersion: *o.GetClusterVersion(), }, StoreConfig: *o.GetStoreConfig(), @@ -825,7 +825,7 @@ func (o *PersistOptions) Reload(storage endpoint.ConfigStorage) error { o.replicationMode.Store(&cfg.ReplicationMode) o.labelProperty.Store(cfg.LabelProperty) o.keyspace.Store(&cfg.Keyspace) - o.microService.Store(&cfg.MicroService) + o.microservice.Store(&cfg.Microservice) o.storeConfig.Store(&cfg.StoreConfig) o.SetClusterVersion(&cfg.ClusterVersion) } diff --git a/server/server.go b/server/server.go index 8de2a5c57f3..7ecf309b1d9 100644 --- a/server/server.go +++ b/server/server.go @@ -300,7 +300,7 @@ func CreateServer(ctx context.Context, cfg *config.Config, services []string, le }) s.registry.RegisterService("MetaStorage", ms_server.NewService) s.registry.RegisterService("ResourceManager", rm_server.NewService[*Server]) - // Register the micro services REST path. + // Register the microservices REST path. s.registry.InstallAllRESTHandler(s, etcdCfg.UserHandlers) etcdCfg.ServiceRegister = func(gs *grpc.Server) { @@ -308,7 +308,7 @@ func CreateServer(ctx context.Context, cfg *config.Config, services []string, le pdpb.RegisterPDServer(gs, grpcServer) keyspacepb.RegisterKeyspaceServer(gs, &KeyspaceServer{GrpcServer: grpcServer}) diagnosticspb.RegisterDiagnosticsServer(gs, s) - // Register the micro services GRPC service. + // Register the microservices GRPC service. s.registry.InstallAllGRPCServices(s, gs) s.grpcServer = gs } @@ -948,7 +948,7 @@ func (s *Server) GetConfig() *config.Config { cfg.PDServerCfg = *s.persistOptions.GetPDServerConfig().Clone() cfg.ReplicationMode = *s.persistOptions.GetReplicationModeConfig() cfg.Keyspace = *s.persistOptions.GetKeyspaceConfig().Clone() - cfg.MicroService = *s.persistOptions.GetMicroServiceConfig().Clone() + cfg.Microservice = *s.persistOptions.GetMicroserviceConfig().Clone() cfg.LabelProperty = s.persistOptions.GetLabelPropertyConfig().Clone() cfg.ClusterVersion = *s.persistOptions.GetClusterVersion() return cfg @@ -979,24 +979,24 @@ func (s *Server) SetKeyspaceConfig(cfg config.KeyspaceConfig) error { return nil } -// GetMicroServiceConfig gets the micro service config information. -func (s *Server) GetMicroServiceConfig() *config.MicroServiceConfig { - return s.persistOptions.GetMicroServiceConfig().Clone() +// GetMicroserviceConfig gets the microservice config information. +func (s *Server) GetMicroserviceConfig() *config.MicroserviceConfig { + return s.persistOptions.GetMicroserviceConfig().Clone() } -// SetMicroServiceConfig sets the micro service config information. -func (s *Server) SetMicroServiceConfig(cfg config.MicroServiceConfig) error { - old := s.persistOptions.GetMicroServiceConfig() - s.persistOptions.SetMicroServiceConfig(&cfg) +// SetMicroserviceConfig sets the microservice config information. +func (s *Server) SetMicroserviceConfig(cfg config.MicroserviceConfig) error { + old := s.persistOptions.GetMicroserviceConfig() + s.persistOptions.SetMicroserviceConfig(&cfg) if err := s.persistOptions.Persist(s.storage); err != nil { - s.persistOptions.SetMicroServiceConfig(old) - log.Error("failed to update micro service config", + s.persistOptions.SetMicroserviceConfig(old) + log.Error("failed to update microservice config", zap.Reflect("new", cfg), zap.Reflect("old", old), errs.ZapError(err)) return err } - log.Info("micro service config is updated", zap.Reflect("new", cfg), zap.Reflect("old", old)) + log.Info("microservice config is updated", zap.Reflect("new", cfg), zap.Reflect("old", old)) return nil } @@ -1391,7 +1391,7 @@ func (s *Server) GetRaftCluster() *cluster.RaftCluster { // IsServiceIndependent returns whether the service is independent. func (s *Server) IsServiceIndependent(name string) bool { if s.mode == PDServiceMode && !s.IsClosed() { - if name == constant.TSOServiceName && !s.GetMicroServiceConfig().IsTSODynamicSwitchingEnabled() { + if name == constant.TSOServiceName && !s.GetMicroserviceConfig().IsTSODynamicSwitchingEnabled() { return true } return s.cluster.IsServiceIndependent(name) diff --git a/tests/integrations/mcs/members/member_test.go b/tests/integrations/mcs/members/member_test.go index 7e83ea570b9..0ec0b323d7a 100644 --- a/tests/integrations/mcs/members/member_test.go +++ b/tests/integrations/mcs/members/member_test.go @@ -126,29 +126,29 @@ func (suite *memberTestSuite) TearDownTest() { func (suite *memberTestSuite) TestMembers() { re := suite.Require() - members, err := suite.pdClient.GetMicroServiceMembers(suite.ctx, "tso") + members, err := suite.pdClient.GetMicroserviceMembers(suite.ctx, "tso") re.NoError(err) re.Len(members, 3) - members, err = suite.pdClient.GetMicroServiceMembers(suite.ctx, "scheduling") + members, err = suite.pdClient.GetMicroserviceMembers(suite.ctx, "scheduling") re.NoError(err) re.Len(members, 3) } func (suite *memberTestSuite) TestPrimary() { re := suite.Require() - primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, "tso") + primary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, "tso") re.NoError(err) re.NotEmpty(primary) - primary, err = suite.pdClient.GetMicroServicePrimary(suite.ctx, "scheduling") + primary, err = suite.pdClient.GetMicroservicePrimary(suite.ctx, "scheduling") re.NoError(err) re.NotEmpty(primary) } func (suite *memberTestSuite) TestPrimaryWorkWhileOtherServerClose() { re := suite.Require() - primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, "tso") + primary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, "tso") re.NoError(err) re.NotEmpty(primary) @@ -162,7 +162,7 @@ func (suite *memberTestSuite) TestPrimaryWorkWhileOtherServerClose() { nodes = suite.schedulingNodes } - primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + primary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, service) re.NoError(err) // Close non-primary node. @@ -174,7 +174,7 @@ func (suite *memberTestSuite) TestPrimaryWorkWhileOtherServerClose() { tests.WaitForPrimaryServing(re, nodes) // primary should be same with before. - curPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + curPrimary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, service) re.NoError(err) re.Equal(primary, curPrimary) } @@ -193,7 +193,7 @@ func (suite *memberTestSuite) TestTransferPrimary() { } // Test resign primary by random - primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + primary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, service) re.NoError(err) newPrimaryData := make(map[string]any) @@ -214,7 +214,7 @@ func (suite *memberTestSuite) TestTransferPrimary() { return false }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) - primary, err = suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + primary, err = suite.pdClient.GetMicroservicePrimary(suite.ctx, service) re.NoError(err) // Test transfer primary to a specific node @@ -240,7 +240,7 @@ func (suite *memberTestSuite) TestTransferPrimary() { return nodes[newPrimary].IsServing() }, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond)) - primary, err = suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + primary, err = suite.pdClient.GetMicroservicePrimary(suite.ctx, service) re.NoError(err) re.Equal(primary, newPrimary) @@ -268,7 +268,7 @@ func (suite *memberTestSuite) TestCampaignPrimaryAfterTransfer() { nodes = suite.schedulingNodes } - primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + primary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, service) re.NoError(err) // Test transfer primary to a specific node @@ -292,7 +292,7 @@ func (suite *memberTestSuite) TestCampaignPrimaryAfterTransfer() { resp.Body.Close() tests.WaitForPrimaryServing(re, nodes) - newPrimary, err = suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + newPrimary, err = suite.pdClient.GetMicroservicePrimary(suite.ctx, service) re.NoError(err) re.NotEqual(primary, newPrimary) @@ -300,7 +300,7 @@ func (suite *memberTestSuite) TestCampaignPrimaryAfterTransfer() { nodes[newPrimary].Close() tests.WaitForPrimaryServing(re, nodes) // Primary should be different with before - anotherPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + anotherPrimary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, service) re.NoError(err) re.NotEqual(newPrimary, anotherPrimary) } @@ -308,7 +308,7 @@ func (suite *memberTestSuite) TestCampaignPrimaryAfterTransfer() { func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpired() { re := suite.Require() - primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, "tso") + primary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, "tso") re.NoError(err) re.NotEmpty(primary) @@ -322,7 +322,7 @@ func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpired() { nodes = suite.schedulingNodes } - primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + primary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, service) re.NoError(err) // Test transfer primary to a specific node @@ -363,7 +363,7 @@ func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpired() { // TestTransferPrimaryWhileLeaseExpiredAndServerDown tests transfer primary while lease expired and server down func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpiredAndServerDown() { re := suite.Require() - primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, "tso") + primary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, "tso") re.NoError(err) re.NotEmpty(primary) @@ -377,7 +377,7 @@ func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpiredAndServerDown( nodes = suite.schedulingNodes } - primary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + primary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, service) re.NoError(err) // Test transfer primary to a specific node @@ -415,7 +415,7 @@ func (suite *memberTestSuite) TestTransferPrimaryWhileLeaseExpiredAndServerDown( tests.WaitForPrimaryServing(re, nodes) // Primary should be different with before - onlyPrimary, err := suite.pdClient.GetMicroServicePrimary(suite.ctx, service) + onlyPrimary, err := suite.pdClient.GetMicroservicePrimary(suite.ctx, service) re.NoError(err) re.NotEqual(newPrimary, onlyPrimary) } diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 9b6b3d95145..e52ced10011 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -118,29 +118,29 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { // Test operators err := testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators"), &respSlice, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) re.Empty(respSlice) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators"), []byte(``), - testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators/2"), nil, - testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckDelete(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators/2"), - testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators/records"), nil, - testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) // Test checker err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "checker/merge"), &resp, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) re.False(resp["paused"].(bool)) @@ -151,7 +151,7 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { pauseArgs, err := json.Marshal(input) re.NoError(err) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "checker/merge"), pauseArgs, - testutil.StatusOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) } postChecker(30) @@ -170,7 +170,7 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { // "/schedulers/{name}", http.MethodDelete testutil.Eventually(re, func() bool { err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers"), &respSlice, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) return slice.Contains(respSlice, "balance-leader-scheduler") }) @@ -181,18 +181,18 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { pauseArgs, err := json.Marshal(input) re.NoError(err) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers/balance-leader-scheduler"), pauseArgs, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) } postScheduler(30) postScheduler(0) err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers/diagnostic/balance-leader-scheduler"), &resp, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "scheduler-config"), &resp, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) re.Contains(resp, "balance-leader-scheduler") re.Contains(resp, "balance-hot-region-scheduler") @@ -203,16 +203,16 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { } for _, schedulerName := range schedulers { err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s/%s/%s", urlPrefix, "scheduler-config", schedulerName, "list"), &resp, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) } err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers"), nil, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) err = testutil.CheckDelete(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers/balance-leader-scheduler"), - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) input := make(map[string]any) @@ -220,74 +220,74 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { b, err := json.Marshal(input) re.NoError(err) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "schedulers"), b, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) // Test hotspot var hotRegions statistics.StoreHotPeersInfos err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "hotspot/regions/write"), &hotRegions, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "hotspot/regions/read"), &hotRegions, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) var stores handler.HotStoreStats err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "hotspot/stores"), &stores, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) var buckets handler.HotBucketsResponse err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "hotspot/buckets"), &buckets, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) var history storage.HistoryHotRegions err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "hotspot/regions/history"), &history, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) // Test region label var labelRules []*labeler.LabelRule err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/region-label/rules"), &labelRules, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.ReadGetJSONWithBody(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/region-label/rules/ids"), []byte(`["rule1", "rule3"]`), - &labelRules, testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + &labelRules, testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/region-label/rule/rule1"), nil, - testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusNotOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "region/id/1"), nil, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "region/id/1/label/key"), nil, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "region/id/1/labels"), nil, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) // Test Region body := fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3"))) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "regions/accelerate-schedule"), []byte(body), - testutil.StatusOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) body = fmt.Sprintf(`[{"start_key":"%s", "end_key": "%s"}, {"start_key":"%s", "end_key": "%s"}]`, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a3")), hex.EncodeToString([]byte("a4")), hex.EncodeToString([]byte("a6"))) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "regions/accelerate-schedule/batch"), []byte(body), - testutil.StatusOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) body = fmt.Sprintf(`{"start_key":"%s", "end_key": "%s"}`, hex.EncodeToString([]byte("b1")), hex.EncodeToString([]byte("b3"))) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "regions/scatter"), []byte(body), - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) body = fmt.Sprintf(`{"retry_limit":%v, "split_keys": ["%s","%s","%s"]}`, 3, hex.EncodeToString([]byte("bbb")), hex.EncodeToString([]byte("ccc")), hex.EncodeToString([]byte("ddd"))) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "regions/split"), []byte(body), - testutil.StatusOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf(`%s/regions/replicated?startKey=%s&endKey=%s`, urlPrefix, hex.EncodeToString([]byte("a1")), hex.EncodeToString([]byte("a2"))), nil, - testutil.StatusOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusOK(re), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) // Test rules: only forward `GET` request var rules []*placement.Rule @@ -305,76 +305,76 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { re.NoError(err) err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules"), &rules, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules"), rulesArgs, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules/batch"), rulesArgs, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules/group/pd"), &rules, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules/region/2"), &rules, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) var fit placement.RegionFit err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules/region/2/detail"), &fit, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules/key/0000000000000001"), &rules, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule/pd/2"), nil, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckDelete(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule/pd/2"), - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule"), rulesArgs, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule_group/pd"), nil, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckDelete(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule_group/pd"), - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule_group"), rulesArgs, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rule_groups"), nil, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule"), nil, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule"), rulesArgs, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule/pd"), nil, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) err = testutil.CheckDelete(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule/pd"), - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule/pd"), rulesArgs, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) // test redirect is disabled err = testutil.CheckGetJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule/pd"), nil, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) req, err := http.NewRequest(http.MethodGet, fmt.Sprintf("%s/%s", urlPrefix, "config/placement-rule/pd"), http.NoBody) re.NoError(err) - req.Header.Set(apiutil.XForbiddenForwardToMicroServiceHeader, "true") + req.Header.Set(apiutil.XForbiddenForwardToMicroserviceHeader, "true") httpResp, err := tests.TestDialClient.Do(req) re.NoError(err) re.Equal(http.StatusOK, httpResp.StatusCode) defer httpResp.Body.Close() - re.Empty(httpResp.Header.Get(apiutil.XForwardedToMicroServiceHeader)) + re.Empty(httpResp.Header.Get(apiutil.XForwardedToMicroserviceHeader)) } func (suite *apiTestSuite) TestConfig() { @@ -536,14 +536,14 @@ func (suite *apiTestSuite) checkFollowerForward(cluster *tests.TestCluster) { // follower will forward to scheduling server directly re.NotEqual(cluster.GetLeaderServer().GetAddr(), followerAddr) err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules"), &rules, - testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true"), + testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true"), ) re.NoError(err) } else { // follower will forward to leader server re.NotEqual(cluster.GetLeaderServer().GetAddr(), followerAddr) err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config/rules"), &rules, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader), + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader), ) re.NoError(err) } @@ -552,7 +552,7 @@ func (suite *apiTestSuite) checkFollowerForward(cluster *tests.TestCluster) { re.NotEqual(cluster.GetLeaderServer().GetAddr(), followerAddr) results := make(map[string]any) err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "config"), &results, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader), + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader), ) re.NoError(err) } diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 9a3d33d1dcf..7271111bd79 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -216,10 +216,10 @@ func (suite *serverTestSuite) TestForwardStoreHeartbeat() { func (suite *serverTestSuite) TestSchedulingServiceFallback() { re := suite.Require() leaderServer := suite.pdLeader.GetServer() - conf := leaderServer.GetMicroServiceConfig().Clone() + conf := leaderServer.GetMicroserviceConfig().Clone() // Change back to the default value. conf.EnableSchedulingFallback = true - leaderServer.SetMicroServiceConfig(*conf) + leaderServer.SetMicroserviceConfig(*conf) // PD service will execute scheduling jobs since there is no scheduling server. testutil.Eventually(re, func() bool { return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() @@ -267,15 +267,15 @@ func (suite *serverTestSuite) TestDisableSchedulingServiceFallback() { }) leaderServer := suite.pdLeader.GetServer() // After Disabling scheduling service fallback, the PD service will stop scheduling. - conf := leaderServer.GetMicroServiceConfig().Clone() + conf := leaderServer.GetMicroserviceConfig().Clone() conf.EnableSchedulingFallback = false - leaderServer.SetMicroServiceConfig(*conf) + leaderServer.SetMicroserviceConfig(*conf) testutil.Eventually(re, func() bool { return !suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) // Enable scheduling service fallback again, the PD service will restart scheduling. conf.EnableSchedulingFallback = true - leaderServer.SetMicroServiceConfig(*conf) + leaderServer.SetMicroserviceConfig(*conf) testutil.Eventually(re, func() bool { return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) @@ -294,7 +294,7 @@ func (suite *serverTestSuite) TestDisableSchedulingServiceFallback() { }) // Disable scheduling service fallback and stop scheduling server. PD service won't execute scheduling jobs again. conf.EnableSchedulingFallback = false - leaderServer.SetMicroServiceConfig(*conf) + leaderServer.SetMicroserviceConfig(*conf) tc.GetPrimaryServer().Close() time.Sleep(time.Second) testutil.Eventually(re, func() bool { diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index dceb5ccdf7c..97f56cbcee1 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -107,13 +107,13 @@ func (suite *tsoAPITestSuite) TestForwardResetTS() { // Test reset ts input := []byte(`{"tso":"121312", "force-use-larger":true}`) err := testutil.CheckPostJSON(tests.TestDialClient, url, input, - testutil.StatusOK(re), testutil.StringContain(re, "Reset ts successfully"), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusOK(re), testutil.StringContain(re, "Reset ts successfully"), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) // Test reset ts with invalid tso input = []byte(`{}`) err = testutil.CheckPostJSON(tests.TestDialClient, url, input, - testutil.StatusNotOK(re), testutil.StringContain(re, "invalid tso value"), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusNotOK(re), testutil.StringContain(re, "invalid tso value"), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) } @@ -217,14 +217,14 @@ func TestForwardOnlyTSONoScheduling(t *testing.T) { // Test /operators, it should not forward when there is no scheduling server. var slice []string err = testutil.ReadGetJSON(re, tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "operators"), &slice, - testutil.WithoutHeader(re, apiutil.XForwardedToMicroServiceHeader)) + testutil.WithoutHeader(re, apiutil.XForwardedToMicroserviceHeader)) re.NoError(err) re.Empty(slice) // Test admin/reset-ts, it should forward to tso server. input := []byte(`{"tso":"121312", "force-use-larger":true}`) err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "admin/reset-ts"), input, - testutil.StatusOK(re), testutil.StringContain(re, "Reset ts successfully"), testutil.WithHeader(re, apiutil.XForwardedToMicroServiceHeader, "true")) + testutil.StatusOK(re), testutil.StringContain(re, "Reset ts successfully"), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) // If close tso server, it should try forward to tso server, but return error in pd service mode. diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 7416a314949..4c9bb4248e5 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -169,7 +169,7 @@ func checkTSOPath(re *require.Assertions, isPDServiceMode bool) { defer cancel() if isPDServiceMode { cluster, err = tests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { - conf.MicroService.EnableTSODynamicSwitching = false + conf.Microservice.EnableTSODynamicSwitching = false }) } else { cluster, err = tests.NewTestCluster(ctx, 1) @@ -276,9 +276,9 @@ func TestForwardTSORelated(t *testing.T) { suite := NewPDServiceForward(re) defer suite.ShutDown() leaderServer := suite.cluster.GetLeaderServer().GetServer() - cfg := leaderServer.GetMicroServiceConfig().Clone() + cfg := leaderServer.GetMicroserviceConfig().Clone() cfg.EnableTSODynamicSwitching = false - leaderServer.SetMicroServiceConfig(*cfg) + leaderServer.SetMicroserviceConfig(*cfg) // Unable to use the tso-related interface without tso server suite.checkUnavailableTSO(re) tc, err := tests.NewTestTSOCluster(suite.ctx, 1, suite.backendEndpoints) @@ -600,7 +600,7 @@ func TestTSOServiceSwitch(t *testing.T) { tc, err := tests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { - conf.MicroService.EnableTSODynamicSwitching = true + conf.Microservice.EnableTSODynamicSwitching = true }, ) re.NoError(err) @@ -638,9 +638,9 @@ func TestTSOServiceSwitch(t *testing.T) { re.NoError(err) // Disable TSO switching - cfg := pdLeader.GetServer().GetMicroServiceConfig().Clone() + cfg := pdLeader.GetServer().GetMicroserviceConfig().Clone() cfg.EnableTSODynamicSwitching = false - pdLeader.GetServer().SetMicroServiceConfig(*cfg) + pdLeader.GetServer().SetMicroserviceConfig(*cfg) tsoCluster.Destroy() @@ -654,10 +654,10 @@ func TestTSOServiceSwitch(t *testing.T) { } // Now enable TSO switching - cfg = pdLeader.GetServer().GetMicroServiceConfig().Clone() + cfg = pdLeader.GetServer().GetMicroserviceConfig().Clone() cfg.EnableTSODynamicSwitching = true - pdLeader.GetServer().SetMicroServiceConfig(*cfg) + pdLeader.GetServer().SetMicroserviceConfig(*cfg) // Wait for PD to detect the change time.Sleep(300 * time.Millisecond) diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index a06e44ed4ab..87ddd9e226b 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -99,7 +99,7 @@ func (suite *tsoClientTestSuite) SetupSuite() { suite.cluster, err = tests.NewTestCluster(suite.ctx, serverCount) } else { suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, serverCount, func(conf *config.Config, _ string) { - conf.MicroService.EnableTSODynamicSwitching = false + conf.Microservice.EnableTSODynamicSwitching = false }) } re.NoError(err) diff --git a/tools/pd-ctl/pdctl/command/config_command.go b/tools/pd-ctl/pdctl/command/config_command.go index f89c63bc51c..c49b0e3b9e5 100644 --- a/tools/pd-ctl/pdctl/command/config_command.go +++ b/tools/pd-ctl/pdctl/command/config_command.go @@ -81,7 +81,7 @@ func NewShowConfigCommand() *cobra.Command { sc.AddCommand(newShowReplicationModeCommand()) sc.AddCommand(NewShowServerConfigCommand()) sc.AddCommand(NewShowServiceMiddlewareConfigCommand()) - sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") + sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") return sc } @@ -92,7 +92,7 @@ func NewShowAllConfigCommand() *cobra.Command { Short: "show all config of PD", Run: showAllConfigCommandFunc, } - sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") + sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") return sc } @@ -103,7 +103,7 @@ func NewShowScheduleConfigCommand() *cobra.Command { Short: "show schedule config of PD", Run: showScheduleConfigCommandFunc, } - sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") + sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") return sc } @@ -114,7 +114,7 @@ func NewShowReplicationConfigCommand() *cobra.Command { Short: "show replication config of PD", Run: showReplicationConfigCommandFunc, } - sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") + sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") return sc } @@ -528,7 +528,7 @@ func NewPlacementRulesCommand() *cobra.Command { show.Flags().String("id", "", "rule id") show.Flags().String("region", "", "region id") show.Flags().Bool("detail", false, "detailed match info for region") - show.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") + show.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") load := &cobra.Command{ Use: "load", Short: "load placement rules to a file", @@ -538,7 +538,7 @@ func NewPlacementRulesCommand() *cobra.Command { load.Flags().String("id", "", "rule id") load.Flags().String("region", "", "region id") load.Flags().String("out", "rules.json", "the filename contains rules") - load.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") + load.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") save := &cobra.Command{ Use: "save", Short: "save rules from file", @@ -554,7 +554,7 @@ func NewPlacementRulesCommand() *cobra.Command { Short: "show rule group configuration(s)", Run: showRuleGroupFunc, } - ruleGroupShow.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") + ruleGroupShow.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") ruleGroupSet := &cobra.Command{ Use: "set ", Short: "update rule group configuration", @@ -577,7 +577,7 @@ func NewPlacementRulesCommand() *cobra.Command { Run: getRuleBundle, } ruleBundleGet.Flags().String("out", "", "the output file") - ruleBundleGet.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") + ruleBundleGet.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") ruleBundleSet := &cobra.Command{ Use: "set", Short: "set rule group config and its rules from file", @@ -596,7 +596,7 @@ func NewPlacementRulesCommand() *cobra.Command { Run: loadRuleBundle, } ruleBundleLoad.Flags().String("out", "rules.json", "the output file") - ruleBundleLoad.Flags().Bool(flagFromPDService, false, "read data from PD service rather than micro service") + ruleBundleLoad.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") ruleBundleSave := &cobra.Command{ Use: "save", Short: "save all group configs and rules from file", @@ -895,9 +895,9 @@ func saveRuleBundle(cmd *cobra.Command, _ []string) { func buildHeader(cmd *cobra.Command) http.Header { header := http.Header{} - forbiddenRedirectToMicroService, err := cmd.Flags().GetBool(flagFromPDService) - if err == nil && forbiddenRedirectToMicroService { - header.Add(apiutil.XForbiddenForwardToMicroServiceHeader, "true") + forbiddenRedirectToMicroservice, err := cmd.Flags().GetBool(flagFromPDService) + if err == nil && forbiddenRedirectToMicroservice { + header.Add(apiutil.XForbiddenForwardToMicroserviceHeader, "true") } return header } diff --git a/tools/pd-ctl/pdctl/command/global.go b/tools/pd-ctl/pdctl/command/global.go index ce79277db1b..4e91f0de514 100644 --- a/tools/pd-ctl/pdctl/command/global.go +++ b/tools/pd-ctl/pdctl/command/global.go @@ -207,9 +207,9 @@ func dial(req *http.Request) (string, error) { if err != nil { return "", err } - if req.Header.Get(apiutil.XForbiddenForwardToMicroServiceHeader) == "true" { - if resp.Header.Get(apiutil.XForwardedToMicroServiceHeader) == "true" { - return string(content), errors.Errorf("the request is forwarded to micro service unexpectedly") + if req.Header.Get(apiutil.XForbiddenForwardToMicroserviceHeader) == "true" { + if resp.Header.Get(apiutil.XForwardedToMicroserviceHeader) == "true" { + return string(content), errors.Errorf("the request is forwarded to microservice unexpectedly") } } return string(content), nil diff --git a/tools/pd-ctl/tests/config/config_test.go b/tools/pd-ctl/tests/config/config_test.go index 39820a6c7b7..3b622beecbf 100644 --- a/tools/pd-ctl/tests/config/config_test.go +++ b/tools/pd-ctl/tests/config/config_test.go @@ -1233,11 +1233,11 @@ func (suite *configTestSuite) checkPDServerConfig(cluster *pdTests.TestCluster) re.Equal(int(3), conf.FlowRoundByDigit) } -func (suite *configTestSuite) TestMicroServiceConfig() { - suite.env.RunTestBasedOnMode(suite.checkMicroServiceConfig) +func (suite *configTestSuite) TestMicroserviceConfig() { + suite.env.RunTestBasedOnMode(suite.checkMicroserviceConfig) } -func (suite *configTestSuite) checkMicroServiceConfig(cluster *pdTests.TestCluster) { +func (suite *configTestSuite) checkMicroserviceConfig(cluster *pdTests.TestCluster) { re := suite.Require() leaderServer := cluster.GetLeaderServer() pdAddr := leaderServer.GetAddr() @@ -1254,13 +1254,13 @@ func (suite *configTestSuite) checkMicroServiceConfig(cluster *pdTests.TestClust re.NoError(err) cfg := config.Config{} re.NoError(json.Unmarshal(output, &cfg)) - re.True(svr.GetMicroServiceConfig().EnableSchedulingFallback) - re.True(cfg.MicroService.EnableSchedulingFallback) + re.True(svr.GetMicroserviceConfig().EnableSchedulingFallback) + re.True(cfg.Microservice.EnableSchedulingFallback) // config set enable-scheduling-fallback args := []string{"-u", pdAddr, "config", "set", "enable-scheduling-fallback", "false"} _, err = tests.ExecuteCommand(cmd, args...) re.NoError(err) - re.False(svr.GetMicroServiceConfig().EnableSchedulingFallback) + re.False(svr.GetMicroserviceConfig().EnableSchedulingFallback) } func (suite *configTestSuite) TestRegionRules() { From 31a0ad6b2b784f3041ddc6bb2d1393c7606863a6 Mon Sep 17 00:00:00 2001 From: ShuNing Date: Thu, 9 Jan 2025 17:40:25 +0800 Subject: [PATCH 11/33] config: completely remove the deprecated field from the PD server config (#8981) close tikv/pd#8980 server/config: completely remove the deprecated field from the PD server config Signed-off-by: nolouch --- server/config/config.go | 30 +++++------------------- server/config/config_test.go | 2 +- server/config/persist_options.go | 1 - tools/pd-ctl/tests/config/config_test.go | 1 - 4 files changed, 7 insertions(+), 27 deletions(-) diff --git a/server/config/config.go b/server/config/config.go index 3d0d16bc376..aa3483bd7fe 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -18,7 +18,6 @@ import ( "crypto/tls" "encoding/json" "fmt" - "math" "net/url" "os" "path/filepath" @@ -507,9 +506,6 @@ type PDServerConfig struct { MetricStorage string `toml:"metric-storage" json:"metric-storage"` // There are some values supported: "auto", "none", or a specific address, default: "auto" DashboardAddress string `toml:"dashboard-address" json:"dashboard-address"` - // TraceRegionFlow the option to update flow information of regions. - // WARN: TraceRegionFlow is deprecated. - TraceRegionFlow bool `toml:"trace-region-flow" json:"trace-region-flow,string,omitempty"` // FlowRoundByDigit used to discretization processing flow information. FlowRoundByDigit int `toml:"flow-round-by-digit" json:"flow-round-by-digit"` // MinResolvedTSPersistenceInterval is the interval to save the min resolved ts. @@ -573,37 +569,23 @@ func (c *PDServerConfig) adjust(meta *configutil.ConfigMetaData) error { } else if c.GCTunerThreshold > maxGCTunerThreshold { c.GCTunerThreshold = maxGCTunerThreshold } - if err := c.migrateConfigurationFromFile(meta); err != nil { + if err := migrateConfigurationFromFile(meta); err != nil { return err } return c.Validate() } -func (c *PDServerConfig) migrateConfigurationFromFile(meta *configutil.ConfigMetaData) error { +func migrateConfigurationFromFile(meta *configutil.ConfigMetaData) error { oldName, newName := "trace-region-flow", "flow-round-by-digit" - defineOld, defineNew := meta.IsDefined(oldName), meta.IsDefined(newName) + defineOld := meta.IsDefined(oldName) switch { - case defineOld && defineNew: - if c.TraceRegionFlow && (c.FlowRoundByDigit == defaultFlowRoundByDigit) { - return errors.Errorf("config item %s and %s(deprecated) are conflict", newName, oldName) - } - case defineOld && !defineNew: - if !c.TraceRegionFlow { - c.FlowRoundByDigit = math.MaxInt8 - } + case defineOld: + return errors.Errorf("config item %s and %s(deprecated) are conflict", newName, oldName) + default: } return nil } -// MigrateDeprecatedFlags updates new flags according to deprecated flags. -func (c *PDServerConfig) MigrateDeprecatedFlags() { - if !c.TraceRegionFlow { - c.FlowRoundByDigit = math.MaxInt8 - } - // json omity the false. next time will not persist to the kv. - c.TraceRegionFlow = false -} - // Clone returns a cloned PD server config. func (c *PDServerConfig) Clone() *PDServerConfig { runtimeServices := append(c.RuntimeServices[:0:0], c.RuntimeServices...) diff --git a/server/config/config_test.go b/server/config/config_test.go index cd9447a5824..b035ab22178 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -277,7 +277,7 @@ func TestMigrateFlags(t *testing.T) { } cfg, err := load(` [pd-server] -trace-region-flow = false +flow-round-by-digit = 127 [schedule] disable-remove-down-replica = true enable-make-up-replica = false diff --git a/server/config/persist_options.go b/server/config/persist_options.go index 5ce24ac3f56..1328971de59 100644 --- a/server/config/persist_options.go +++ b/server/config/persist_options.go @@ -817,7 +817,6 @@ func (o *PersistOptions) Reload(storage endpoint.ConfigStorage) error { adjustScheduleCfg(&cfg.Schedule) // Some fields may not be stored in the storage, we need to calculate them manually. cfg.StoreConfig.Adjust() - cfg.PDServerCfg.MigrateDeprecatedFlags() if isExist { o.schedule.Store(&cfg.Schedule) o.replication.Store(&cfg.Replication) diff --git a/tools/pd-ctl/tests/config/config_test.go b/tools/pd-ctl/tests/config/config_test.go index 3b622beecbf..aeb3352eccd 100644 --- a/tools/pd-ctl/tests/config/config_test.go +++ b/tools/pd-ctl/tests/config/config_test.go @@ -150,7 +150,6 @@ func (suite *configTestSuite) checkConfig(cluster *pdTests.TestCluster) { args = []string{"-u", pdAddr, "config", "set", "trace-region-flow", "false"} _, err = tests.ExecuteCommand(cmd, args...) re.NoError(err) - re.False(svr.GetPDServerConfig().TraceRegionFlow) origin := svr.GetPDServerConfig().FlowRoundByDigit args = []string{"-u", pdAddr, "config", "set", "flow-round-by-digit", "10"} From 50c1bbb9e9a9901be82062530501c5da6a9e0576 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Mon, 13 Jan 2025 11:27:25 +0800 Subject: [PATCH 12/33] *: no need to distinguish PD mode (#8984) ref tikv/pd#8477 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/servicediscovery/service_discovery.go | 4 +- cmd/pd-server/main.go | 6 +- pkg/mcs/registry/registry.go | 8 +- pkg/mcs/scheduling/server/cluster.go | 4 +- pkg/mcs/scheduling/server/config/config.go | 6 +- pkg/mcs/scheduling/server/config/watcher.go | 4 +- pkg/mcs/scheduling/server/grpc_service.go | 2 +- pkg/mcs/scheduling/server/meta/watcher.go | 4 +- pkg/mcs/scheduling/server/rule/watcher.go | 4 +- pkg/mcs/scheduling/server/server.go | 8 +- pkg/member/election_leader.go | 2 +- pkg/schedule/coordinator.go | 2 +- .../schedulers/scheduler_controller.go | 4 +- pkg/tso/keyspace_group_manager.go | 2 +- pkg/utils/apiutil/serverapi/middleware.go | 4 +- server/api/admin.go | 2 +- server/api/server.go | 4 +- server/apiv2/handlers/micro_service.go | 4 +- server/cluster/cluster.go | 22 ++--- server/config/config.go | 2 +- .../service_middleware_persist_options.go | 10 +-- server/grpc_service.go | 6 +- server/server.go | 59 +++++++------- server/server_test.go | 2 +- tests/cluster.go | 16 ++-- tests/integrations/client/client_test.go | 2 +- .../mcs/discovery/register_test.go | 9 +-- .../mcs/keyspace/tso_keyspace_group_test.go | 2 +- tests/integrations/mcs/members/member_test.go | 2 +- tests/integrations/mcs/scheduling/api_test.go | 40 +++++----- .../mcs/scheduling/config_test.go | 4 +- .../integrations/mcs/scheduling/meta_test.go | 2 +- .../integrations/mcs/scheduling/rule_test.go | 4 +- .../mcs/scheduling/server_test.go | 38 ++++----- tests/integrations/mcs/tso/api_test.go | 6 +- .../mcs/tso/keyspace_group_manager_test.go | 14 ++-- tests/integrations/mcs/tso/proxy_test.go | 2 +- tests/integrations/mcs/tso/server_test.go | 22 ++--- tests/integrations/tso/client_test.go | 24 +++--- tests/integrations/tso/consistency_test.go | 2 +- tests/integrations/tso/server_test.go | 2 +- tests/server/api/checker_test.go | 2 +- tests/server/api/operator_test.go | 10 +-- tests/server/api/region_test.go | 12 +-- tests/server/api/rule_test.go | 34 ++++---- tests/server/api/scheduler_test.go | 26 +++--- .../apiv2/handlers/tso_keyspace_group_test.go | 2 +- tests/server/config/config_test.go | 16 ++-- tests/testutil.go | 80 +++++++++---------- tools/pd-ctl/pdctl/command/config_command.go | 24 +++--- tools/pd-ctl/tests/config/config_test.go | 32 ++++---- tools/pd-ctl/tests/hot/hot_test.go | 8 +- .../tests/keyspace/keyspace_group_test.go | 14 ++-- tools/pd-ctl/tests/keyspace/keyspace_test.go | 4 +- tools/pd-ctl/tests/operator/operator_test.go | 2 +- .../pd-ctl/tests/scheduler/scheduler_test.go | 14 ++-- 56 files changed, 321 insertions(+), 325 deletions(-) diff --git a/client/servicediscovery/service_discovery.go b/client/servicediscovery/service_discovery.go index bef80e28a37..f5ac665b7cd 100644 --- a/client/servicediscovery/service_discovery.go +++ b/client/servicediscovery/service_discovery.go @@ -74,7 +74,7 @@ const ( type serviceType int const ( - apiService serviceType = iota + pdService serviceType = iota tsoService ) @@ -678,7 +678,7 @@ func (*serviceDiscovery) GetKeyspaceGroupID() uint32 { // DiscoverMicroservice discovers the microservice with the specified type and returns the server urls. func (c *serviceDiscovery) discoverMicroservice(svcType serviceType) (urls []string, err error) { switch svcType { - case apiService: + case pdService: urls = c.GetServiceURLs() case tsoService: leaderURL := c.getLeaderURL() diff --git a/cmd/pd-server/main.go b/cmd/pd-server/main.go index 24ca46e7d5e..e9abef1fd4a 100644 --- a/cmd/pd-server/main.go +++ b/cmd/pd-server/main.go @@ -218,11 +218,7 @@ func start(cmd *cobra.Command, args []string, services ...string) { // Flushing any buffered log entries defer log.Sync() memory.InitMemoryHook() - if len(services) != 0 { - versioninfo.Log(server.PDServiceMode) - } else { - versioninfo.Log(server.PDMode) - } + versioninfo.Log(server.PD) for _, msg := range cfg.WarningMsgs { log.Warn(msg) diff --git a/pkg/mcs/registry/registry.go b/pkg/mcs/registry/registry.go index 2ffa04b1bf9..dc0fafbbd33 100644 --- a/pkg/mcs/registry/registry.go +++ b/pkg/mcs/registry/registry.go @@ -85,18 +85,18 @@ func (r *ServiceRegistry) InstallAllRESTHandler(srv bs.Server, h map[string]http serviceName := createServiceName(prefix, name) if l, ok := r.services[serviceName]; ok { if err := l.RegisterRESTHandler(h); err != nil { - log.Error("register restful PD service failed", zap.String("prefix", prefix), zap.String("service-name", name), zap.Error(err)) + log.Error("register restful PD failed", zap.String("prefix", prefix), zap.String("service-name", name), zap.Error(err)) } else { - log.Info("restful PD service already registered", zap.String("prefix", prefix), zap.String("service-name", name)) + log.Info("restful PD already registered", zap.String("prefix", prefix), zap.String("service-name", name)) } continue } l := builder(srv) r.services[serviceName] = l if err := l.RegisterRESTHandler(h); err != nil { - log.Error("register restful PD service failed", zap.String("prefix", prefix), zap.String("service-name", name), zap.Error(err)) + log.Error("register restful PD failed", zap.String("prefix", prefix), zap.String("service-name", name), zap.Error(err)) } else { - log.Info("restful PD service registered successfully", zap.String("prefix", prefix), zap.String("service-name", name)) + log.Info("restful PD registered successfully", zap.String("prefix", prefix), zap.String("service-name", name)) } } } diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 6f80572673c..43e0179412e 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -257,8 +257,8 @@ func (c *Cluster) triggerMembershipCheck() { } } -// SwitchPDServiceLeader switches the PD service leader. -func (c *Cluster) SwitchPDServiceLeader(new pdpb.PDClient) bool { +// SwitchPDLeader switches the PD leader. +func (c *Cluster) SwitchPDLeader(new pdpb.PDClient) bool { old := c.pdLeader.Load() return c.pdLeader.CompareAndSwap(old, new) } diff --git a/pkg/mcs/scheduling/server/config/config.go b/pkg/mcs/scheduling/server/config/config.go index 413a6c601cc..2b3e350552f 100644 --- a/pkg/mcs/scheduling/server/config/config.go +++ b/pkg/mcs/scheduling/server/config/config.go @@ -243,7 +243,7 @@ func NewPersistConfig(cfg *Config, ttl *cache.TTLString) *PersistConfig { o.SetClusterVersion(&cfg.ClusterVersion) o.schedule.Store(&cfg.Schedule) o.replication.Store(&cfg.Replication) - // storeConfig will be fetched from TiKV by PD service, + // storeConfig will be fetched from TiKV by PD, // so we just set an empty value here first. o.storeConfig.Store(&sc.StoreConfig{}) o.ttl = ttl @@ -748,11 +748,11 @@ func (o *PersistConfig) IsRaftKV2() bool { // TODO: implement the following methods // AddSchedulerCfg adds the scheduler configurations. -// This method is a no-op since we only use configurations derived from one-way synchronization from PD service now. +// This method is a no-op since we only use configurations derived from one-way synchronization from PD now. func (*PersistConfig) AddSchedulerCfg(types.CheckerSchedulerType, []string) {} // RemoveSchedulerCfg removes the scheduler configurations. -// This method is a no-op since we only use configurations derived from one-way synchronization from PD service now. +// This method is a no-op since we only use configurations derived from one-way synchronization from PD now. func (*PersistConfig) RemoveSchedulerCfg(types.CheckerSchedulerType) {} // CheckLabelProperty checks if the label property is satisfied. diff --git a/pkg/mcs/scheduling/server/config/watcher.go b/pkg/mcs/scheduling/server/config/watcher.go index 9db2d47d0f4..1c87076429e 100644 --- a/pkg/mcs/scheduling/server/config/watcher.go +++ b/pkg/mcs/scheduling/server/config/watcher.go @@ -36,7 +36,7 @@ import ( "github.com/tikv/pd/pkg/utils/keypath" ) -// Watcher is used to watch the PD service for any configuration changes. +// Watcher is used to watch the PD for any configuration changes. type Watcher struct { wg sync.WaitGroup ctx context.Context @@ -76,7 +76,7 @@ type persistedConfig struct { Store sc.StoreConfig `json:"store"` } -// NewWatcher creates a new watcher to watch the config meta change from PD service. +// NewWatcher creates a new watcher to watch the config meta change from PD. func NewWatcher( ctx context.Context, etcdClient *clientv3.Client, diff --git a/pkg/mcs/scheduling/server/grpc_service.go b/pkg/mcs/scheduling/server/grpc_service.go index bd2cc40c21d..033397e3b50 100644 --- a/pkg/mcs/scheduling/server/grpc_service.go +++ b/pkg/mcs/scheduling/server/grpc_service.go @@ -159,7 +159,7 @@ func (s *Service) RegionHeartbeat(stream schedulingpb.Scheduling_RegionHeartbeat region := core.RegionFromHeartbeat(request, 0) err = c.HandleRegionHeartbeat(region) if err != nil { - // TODO: if we need to send the error back to PD service. + // TODO: if we need to send the error back to PD. log.Error("failed handle region heartbeat", zap.Error(err)) continue } diff --git a/pkg/mcs/scheduling/server/meta/watcher.go b/pkg/mcs/scheduling/server/meta/watcher.go index c51f10027d7..048e2716cd1 100644 --- a/pkg/mcs/scheduling/server/meta/watcher.go +++ b/pkg/mcs/scheduling/server/meta/watcher.go @@ -33,7 +33,7 @@ import ( "github.com/tikv/pd/pkg/utils/keypath" ) -// Watcher is used to watch the PD service for any meta changes. +// Watcher is used to watch the PD for any meta changes. type Watcher struct { wg sync.WaitGroup ctx context.Context @@ -48,7 +48,7 @@ type Watcher struct { storeWatcher *etcdutil.LoopWatcher } -// NewWatcher creates a new watcher to watch the meta change from PD service. +// NewWatcher creates a new watcher to watch the meta change from PD. func NewWatcher( ctx context.Context, etcdClient *clientv3.Client, diff --git a/pkg/mcs/scheduling/server/rule/watcher.go b/pkg/mcs/scheduling/server/rule/watcher.go index 014a3abc2be..f20c4fa7f99 100644 --- a/pkg/mcs/scheduling/server/rule/watcher.go +++ b/pkg/mcs/scheduling/server/rule/watcher.go @@ -34,7 +34,7 @@ import ( "github.com/tikv/pd/pkg/utils/keypath" ) -// Watcher is used to watch the PD service for any Placement Rule changes. +// Watcher is used to watch the PD for any Placement Rule changes. type Watcher struct { ctx context.Context cancel context.CancelFunc @@ -74,7 +74,7 @@ type Watcher struct { patch *placement.RuleConfigPatch } -// NewWatcher creates a new watcher to watch the Placement Rule change from PD service. +// NewWatcher creates a new watcher to watch the Placement Rule change from PD. func NewWatcher( ctx context.Context, etcdClient *clientv3.Client, diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index 80156c1e26b..482bc48d2b4 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -110,7 +110,7 @@ type Server struct { hbStreams *hbstream.HeartbeatStreams storage *endpoint.StorageEndpoint - // for watching the PD service meta info updates that are related to the scheduling. + // for watching the PD meta info updates that are related to the scheduling. configWatcher *config.Watcher ruleWatcher *rule.Watcher metaWatcher *meta.Watcher @@ -169,10 +169,10 @@ func (s *Server) startServerLoop() { s.serverLoopCtx, s.serverLoopCancel = context.WithCancel(s.Context()) s.serverLoopWg.Add(2) go s.primaryElectionLoop() - go s.updatePDServiceMemberLoop() + go s.updatePDMemberLoop() } -func (s *Server) updatePDServiceMemberLoop() { +func (s *Server) updatePDMemberLoop() { defer logutil.LogPanic() defer s.serverLoopWg.Done() @@ -220,7 +220,7 @@ func (s *Server) updatePDServiceMemberLoop() { // double check break } - if s.cluster.SwitchPDServiceLeader(pdpb.NewPDClient(cc)) { + if s.cluster.SwitchPDLeader(pdpb.NewPDClient(cc)) { if status.Leader != curLeader { log.Info("switch leader", zap.String("leader-id", fmt.Sprintf("%x", ep.ID)), zap.String("endpoint", ep.ClientURLs[0])) } diff --git a/pkg/member/election_leader.go b/pkg/member/election_leader.go index 5cdc7b4cd9b..76d977eddbd 100644 --- a/pkg/member/election_leader.go +++ b/pkg/member/election_leader.go @@ -21,7 +21,7 @@ import ( ) // ElectionLeader defines the common interface of the leader, which is the pdpb.Member -// for in PD/PD service or the tsopb.Participant in the microservices. +// for in PD or the tsopb.Participant in the microservices. type ElectionLeader interface { // GetListenUrls returns the listen urls GetListenUrls() []string diff --git a/pkg/schedule/coordinator.go b/pkg/schedule/coordinator.go index 80299bf1e25..0f09b119939 100644 --- a/pkg/schedule/coordinator.go +++ b/pkg/schedule/coordinator.go @@ -389,7 +389,7 @@ func (c *Coordinator) LoadPlugin(pluginPath string, ch chan string) { return } log.Info("create scheduler", zap.String("scheduler-name", s.GetName())) - // TODO: handle the plugin in PD service mode. + // TODO: handle the plugin in microservice env. if err = c.schedulers.AddScheduler(s); err != nil { log.Error("can't add scheduler", zap.String("scheduler-name", s.GetName()), errs.ZapError(err)) return diff --git a/pkg/schedule/schedulers/scheduler_controller.go b/pkg/schedule/schedulers/scheduler_controller.go index 5f461d326c5..a6d37c9c834 100644 --- a/pkg/schedule/schedulers/scheduler_controller.go +++ b/pkg/schedule/schedulers/scheduler_controller.go @@ -52,10 +52,10 @@ type Controller struct { cluster sche.SchedulerCluster storage endpoint.ConfigStorage // schedulers are used to manage all schedulers, which will only be initialized - // and used in the PD leader service mode now. + // and used in the non-microservice env now. schedulers map[string]*ScheduleController // schedulerHandlers is used to manage the HTTP handlers of schedulers, - // which will only be initialized and used in the PD service mode now. + // which will only be initialized and used in the microservice env now. schedulerHandlers map[string]http.Handler opController *operator.Controller } diff --git a/pkg/tso/keyspace_group_manager.go b/pkg/tso/keyspace_group_manager.go index 9793939fa17..4817aeeeddf 100644 --- a/pkg/tso/keyspace_group_manager.go +++ b/pkg/tso/keyspace_group_manager.go @@ -334,7 +334,7 @@ type KeyspaceGroupManager struct { // Value: discover.ServiceRegistryEntry tsoServiceKey string // legacySvcRootPath defines the legacy root path for all etcd paths which derives from - // the PD/PD service. It's in the format of "/pd/{cluster_id}". + // the PD. It's in the format of "/pd/{cluster_id}". // The main paths for different usages include: // 1. The path, used by the default keyspace group, for LoadTimestamp/SaveTimestamp in the // storage endpoint. diff --git a/pkg/utils/apiutil/serverapi/middleware.go b/pkg/utils/apiutil/serverapi/middleware.go index 0b03c787e6b..089acd47470 100644 --- a/pkg/utils/apiutil/serverapi/middleware.go +++ b/pkg/utils/apiutil/serverapi/middleware.go @@ -116,7 +116,7 @@ func MicroserviceRedirectRule(matchPath, targetPath, targetServiceName string, } func (h *redirector) matchMicroserviceRedirectRules(r *http.Request) (bool, string) { - if !h.s.IsPDServiceMode() { + if !h.s.IsKeyspaceGroupEnabled() { return false, "" } if len(h.microserviceRedirectRules) == 0 { @@ -223,7 +223,7 @@ func (h *redirector) ServeHTTP(w http.ResponseWriter, r *http.Request, next http clientUrls = leader.GetClientUrls() r.Header.Set(apiutil.PDRedirectorHeader, h.s.Name()) } else { - // Prevent more than one redirection among PD/PD service. + // Prevent more than one redirection among PD. log.Error("redirect but server is not leader", zap.String("from", name), zap.String("server", h.s.Name()), errs.ZapError(errs.ErrRedirectToNotLeader)) http.Error(w, errs.ErrRedirectToNotLeader.FastGenByArgs().Error(), http.StatusInternalServerError) return diff --git a/server/api/admin.go b/server/api/admin.go index 561f4ec4bff..b5612999dd0 100644 --- a/server/api/admin.go +++ b/server/api/admin.go @@ -254,5 +254,5 @@ func (h *adminHandler) deleteRegionCacheInSchedulingServer(id ...uint64) error { } func buildMsg(err error) string { - return fmt.Sprintf("This operation was executed in PD service but needs to be re-executed on scheduling server due to the following error: %s", err.Error()) + return fmt.Sprintf("This operation was executed in PD but needs to be re-executed on scheduling server due to the following error: %s", err.Error()) } diff --git a/server/api/server.go b/server/api/server.go index 8e352b6a36e..a9464bdb2e3 100644 --- a/server/api/server.go +++ b/server/api/server.go @@ -63,7 +63,7 @@ func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.AP // Following requests are **not** redirected: // "/schedulers", http.MethodPost // "/schedulers/{name}", http.MethodDelete - // Because the writing of all the config of the scheduling service is in the PD service, + // Because the writing of all the config of the scheduling service is in the PD, // we should not post and delete the scheduler directly in the scheduling service. router.PathPrefix(apiPrefix).Handler(negroni.New( serverapi.NewRuntimeServiceValidator(svr, group), @@ -153,7 +153,7 @@ func NewHandler(_ context.Context, svr *server.Server) (http.Handler, apiutil.AP scheapi.APIPathPrefix+"/config/placement-rule", constant.SchedulingServiceName, []string{http.MethodGet}), - // because the writing of all the meta information of the scheduling service is in the PD service, + // because the writing of all the meta information of the scheduling service is in the PD, // we should not post and delete the scheduler directly in the scheduling service. serverapi.MicroserviceRedirectRule( prefix+"/schedulers", diff --git a/server/apiv2/handlers/micro_service.go b/server/apiv2/handlers/micro_service.go index 3a0d6a2c2a6..25a2b17b18c 100644 --- a/server/apiv2/handlers/micro_service.go +++ b/server/apiv2/handlers/micro_service.go @@ -39,7 +39,7 @@ func RegisterMicroservice(r *gin.RouterGroup) { // @Router /ms/members/{service} [get] func GetMembers(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) - if !svr.IsPDServiceMode() { + if !svr.IsKeyspaceGroupEnabled() { c.AbortWithStatusJSON(http.StatusNotFound, "not support microservice") return } @@ -65,7 +65,7 @@ func GetMembers(c *gin.Context) { // @Router /ms/primary/{service} [get] func GetPrimary(c *gin.Context) { svr := c.MustGet(middlewares.ServerContextKey).(*server.Server) - if !svr.IsPDServiceMode() { + if !svr.IsKeyspaceGroupEnabled() { c.AbortWithStatusJSON(http.StatusNotFound, "not support microservice") return } diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 4ad97dfd0cc..8923efef9a6 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -131,7 +131,7 @@ type Server interface { GetMembers() ([]*pdpb.Member, error) ReplicateFileToMember(ctx context.Context, member *pdpb.Member, name string, data []byte) error GetKeyspaceGroupManager() *keyspace.GroupManager - IsPDServiceMode() bool + IsKeyspaceGroupEnabled() bool GetSafePointV2Manager() *gc.SafePointV2Manager } @@ -156,12 +156,12 @@ type RaftCluster struct { etcdClient *clientv3.Client httpClient *http.Client - running bool - isPDServiceMode bool - meta *metapb.Cluster - storage storage.Storage - minResolvedTS atomic.Value // Store as uint64 - externalTS atomic.Value // Store as uint64 + running bool + isKeyspaceGroupEnabled bool + meta *metapb.Cluster + storage storage.Storage + minResolvedTS atomic.Value // Store as uint64 + externalTS atomic.Value // Store as uint64 // Keep the previous store limit settings when removing a store. prevStoreLimit map[uint64]map[storelimit.Type]float64 @@ -325,7 +325,7 @@ func (c *RaftCluster) Start(s Server, bootstrap bool) (err error) { log.Warn("raft cluster has already been started") return nil } - c.isPDServiceMode = s.IsPDServiceMode() + c.isKeyspaceGroupEnabled = s.IsKeyspaceGroupEnabled() err = c.InitCluster(s.GetAllocator(), s.GetPersistOptions(), s.GetHBStreams(), s.GetKeyspaceGroupManager()) if err != nil { return err @@ -376,7 +376,7 @@ func (c *RaftCluster) Start(s Server, bootstrap bool) (err error) { c.loadExternalTS() c.loadMinResolvedTS() - if c.isPDServiceMode { + if c.isKeyspaceGroupEnabled { // bootstrap keyspace group manager after starting other parts successfully. // This order avoids a stuck goroutine in keyspaceGroupManager when it fails to create raftcluster. err = c.keyspaceGroupManager.Bootstrap(c.ctx) @@ -404,7 +404,7 @@ func (c *RaftCluster) Start(s Server, bootstrap bool) (err error) { } func (c *RaftCluster) checkSchedulingService() { - if c.isPDServiceMode { + if c.isKeyspaceGroupEnabled { servers, err := discovery.Discover(c.etcdClient, constant.SchedulingServiceName) if c.opt.GetMicroserviceConfig().IsSchedulingFallbackEnabled() && (err != nil || len(servers) == 0) { c.startSchedulingJobs(c, c.hbstreams) @@ -425,7 +425,7 @@ func (c *RaftCluster) checkSchedulingService() { // checkTSOService checks the TSO service. func (c *RaftCluster) checkTSOService() { - if c.isPDServiceMode { + if c.isKeyspaceGroupEnabled { if c.opt.GetMicroserviceConfig().IsTSODynamicSwitchingEnabled() { servers, err := discovery.Discover(c.etcdClient, constant.TSOServiceName) if err != nil || len(servers) == 0 { diff --git a/server/config/config.go b/server/config/config.go index aa3483bd7fe..62aec798e8c 100644 --- a/server/config/config.go +++ b/server/config/config.go @@ -842,7 +842,7 @@ func (c *MicroserviceConfig) Clone() *MicroserviceConfig { return &cfg } -// IsSchedulingFallbackEnabled returns whether to enable scheduling service fallback to PD service. +// IsSchedulingFallbackEnabled returns whether to enable scheduling service fallback to PD. func (c *MicroserviceConfig) IsSchedulingFallbackEnabled() bool { return c.EnableSchedulingFallback } diff --git a/server/config/service_middleware_persist_options.go b/server/config/service_middleware_persist_options.go index ae4c0ef9b7a..d59d23146f7 100644 --- a/server/config/service_middleware_persist_options.go +++ b/server/config/service_middleware_persist_options.go @@ -45,7 +45,7 @@ func (o *ServiceMiddlewarePersistOptions) GetAuditConfig() *AuditConfig { return o.audit.Load().(*AuditConfig) } -// SetAuditConfig sets the PD service middleware configuration. +// SetAuditConfig sets the PD middleware configuration. func (o *ServiceMiddlewarePersistOptions) SetAuditConfig(cfg *AuditConfig) { o.audit.Store(cfg) } @@ -55,12 +55,12 @@ func (o *ServiceMiddlewarePersistOptions) IsAuditEnabled() bool { return o.GetAuditConfig().EnableAudit } -// GetRateLimitConfig returns pd service middleware configurations. +// GetRateLimitConfig returns PD middleware configurations. func (o *ServiceMiddlewarePersistOptions) GetRateLimitConfig() *RateLimitConfig { return o.rateLimit.Load().(*RateLimitConfig) } -// SetRateLimitConfig sets the PD service middleware configuration. +// SetRateLimitConfig sets the PD middleware configuration. func (o *ServiceMiddlewarePersistOptions) SetRateLimitConfig(cfg *RateLimitConfig) { o.rateLimit.Store(cfg) } @@ -70,12 +70,12 @@ func (o *ServiceMiddlewarePersistOptions) IsRateLimitEnabled() bool { return o.GetRateLimitConfig().EnableRateLimit } -// GetGRPCRateLimitConfig returns pd service middleware configurations. +// GetGRPCRateLimitConfig returns PD middleware configurations. func (o *ServiceMiddlewarePersistOptions) GetGRPCRateLimitConfig() *GRPCRateLimitConfig { return o.grpcRateLimit.Load().(*GRPCRateLimitConfig) } -// SetGRPCRateLimitConfig sets the PD service middleware configuration. +// SetGRPCRateLimitConfig sets the PD middleware configuration. func (o *ServiceMiddlewarePersistOptions) SetGRPCRateLimitConfig(cfg *GRPCRateLimitConfig) { o.grpcRateLimit.Store(cfg) } diff --git a/server/grpc_service.go b/server/grpc_service.go index 118b3d84748..55639474ad0 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -271,9 +271,9 @@ func (s *GrpcServer) GetClusterInfo(context.Context, *pdpb.GetClusterInfoRequest }, nil } -// GetMinTS implements gRPC PDServer. In PD mode, it simply returns a timestamp. -// In PD service mode, it queries all tso servers and gets the minimum timestamp across -// all keyspace groups. +// GetMinTS implements gRPC PDServer. In non-microservice env, it simply returns a timestamp. +// In microservice env, if the tso server exist, it queries all tso servers and gets the minimum timestamp across +// all keyspace groups. Otherwise, it generates a timestamp locally. func (s *GrpcServer) GetMinTS( ctx context.Context, request *pdpb.GetMinTSRequest, ) (*pdpb.GetMinTSResponse, error) { diff --git a/server/server.go b/server/server.go index 7ecf309b1d9..8e6e3422977 100644 --- a/server/server.go +++ b/server/server.go @@ -99,10 +99,8 @@ const ( recoveringMarkPath = "cluster/markers/snapshot-recovering" - // PDMode represents that server is in PD mode. - PDMode = "PD" - // PDServiceMode represents that server is in PD service mode which is in microservice architecture. - PDServiceMode = "PD Service" + // PD is name of member. + PD = "PD" // maxRetryTimesGetServicePrimary is the max retry times for getting primary addr. // Note: it need to be less than client.defaultPDTimeout @@ -227,7 +225,7 @@ type Server struct { auditBackends []audit.Backend registry *registry.ServiceRegistry - mode string + isKeyspaceGroupEnabled bool servicePrimaryMap sync.Map /* Store as map[string]string */ tsoPrimaryWatcher *etcdutil.LoopWatcher schedulingPrimaryWatcher *etcdutil.LoopWatcher @@ -241,13 +239,16 @@ type HandlerBuilder func(context.Context, *Server) (http.Handler, apiutil.APISer // CreateServer creates the UNINITIALIZED pd server with given configuration. func CreateServer(ctx context.Context, cfg *config.Config, services []string, legacyServiceBuilders ...HandlerBuilder) (*Server, error) { - var mode string - if len(services) != 0 { - mode = PDServiceMode - } else { - mode = PDMode - } - log.Info(fmt.Sprintf("%s config", mode), zap.Reflect("config", cfg)) + // TODO: Currently, whether we enable microservice or not is determined by the service list. + // It's equal to whether we enable the keyspace group or not. + // But indeed the keyspace group is independent of the microservice. + // There could be the following scenarios: + // 1. Enable microservice but disable keyspace group. (non-serverless scenario) + // 2. Enable microservice and enable keyspace group. (serverless scenario) + // 3. Disable microservice and disable keyspace group. (both serverless scenario and non-serverless scenario) + // We should separate the keyspace group from the microservice later. + isKeyspaceGroupEnabled := len(services) != 0 + log.Info("PD config", zap.Bool("enable-keyspace-group", isKeyspaceGroupEnabled), zap.Reflect("config", cfg)) serviceMiddlewareCfg := config.NewServiceMiddlewareConfig() s := &Server{ @@ -259,7 +260,7 @@ func CreateServer(ctx context.Context, cfg *config.Config, services []string, le ctx: ctx, startTimestamp: time.Now().Unix(), DiagnosticsServer: sysutil.NewDiagnosticsServer(cfg.Log.File.Filename), - mode: mode, + isKeyspaceGroupEnabled: isKeyspaceGroupEnabled, tsoClientPool: struct { syncutil.RWMutex clients map[string]tsopb.TSO_TsoClient @@ -478,7 +479,7 @@ func (s *Server) startServer(ctx context.Context) error { Member: s.member.MemberValue(), Step: keyspace.AllocStep, }) - if s.IsPDServiceMode() { + if s.IsKeyspaceGroupEnabled() { s.keyspaceGroupManager = keyspace.NewKeyspaceGroupManager(s.ctx, s.storage, s.client) } s.keyspaceManager = keyspace.NewKeyspaceManager(s.ctx, s.storage, s.cluster, keyspaceIDAllocator, &s.cfg.Keyspace, s.keyspaceGroupManager) @@ -530,7 +531,7 @@ func (s *Server) Close() { s.cgMonitor.StopMonitor() s.stopServerLoop() - if s.IsPDServiceMode() { + if s.IsKeyspaceGroupEnabled() { s.keyspaceGroupManager.Close() } @@ -641,7 +642,7 @@ func (s *Server) startServerLoop(ctx context.Context) { go s.etcdLeaderLoop() go s.serverMetricsLoop() go s.encryptionKeyManagerLoop() - if s.IsPDServiceMode() { + if s.IsKeyspaceGroupEnabled() { s.initTSOPrimaryWatcher() s.initSchedulingPrimaryWatcher() } @@ -788,9 +789,9 @@ func (s *Server) stopRaftCluster() { s.cluster.Stop() } -// IsPDServiceMode return whether the server is in PD service mode. -func (s *Server) IsPDServiceMode() bool { - return s.mode == PDServiceMode +// IsKeyspaceGroupEnabled return whether the server is in PD. +func (s *Server) IsKeyspaceGroupEnabled() bool { + return s.isKeyspaceGroupEnabled } // GetAddr returns the server urls for clients. @@ -1390,7 +1391,7 @@ func (s *Server) GetRaftCluster() *cluster.RaftCluster { // IsServiceIndependent returns whether the service is independent. func (s *Server) IsServiceIndependent(name string) bool { - if s.mode == PDServiceMode && !s.IsClosed() { + if s.isKeyspaceGroupEnabled && !s.IsClosed() { if name == constant.TSOServiceName && !s.GetMicroserviceConfig().IsTSODynamicSwitchingEnabled() { return true } @@ -1595,7 +1596,7 @@ func (s *Server) leaderLoop() { for { if s.IsClosed() { - log.Info(fmt.Sprintf("server is closed, return %s leader loop", s.mode)) + log.Info("server is closed, return PD leader loop") return } @@ -1664,13 +1665,13 @@ func (s *Server) leaderLoop() { } func (s *Server) campaignLeader() { - log.Info(fmt.Sprintf("start to campaign %s leader", s.mode), zap.String("campaign-leader-name", s.Name())) + log.Info("start to campaign PD leader", zap.String("campaign-leader-name", s.Name())) if err := s.member.CampaignLeader(s.ctx, s.cfg.LeaderLease); err != nil { if err.Error() == errs.ErrEtcdTxnConflict.Error() { - log.Info(fmt.Sprintf("campaign %s leader meets error due to txn conflict, another PD/PD service may campaign successfully", s.mode), + log.Info("campaign PD leader meets error due to txn conflict, another PD may campaign successfully", zap.String("campaign-leader-name", s.Name())) } else { - log.Error(fmt.Sprintf("campaign %s leader meets error due to etcd error", s.mode), + log.Error("campaign PD leader meets error due to etcd error", zap.String("campaign-leader-name", s.Name()), errs.ZapError(err)) } @@ -1690,7 +1691,7 @@ func (s *Server) campaignLeader() { // maintain the PD leadership, after this, TSO can be service. s.member.KeepLeader(ctx) - log.Info(fmt.Sprintf("campaign %s leader ok", s.mode), zap.String("campaign-leader-name", s.Name())) + log.Info("campaign PD leader ok", zap.String("campaign-leader-name", s.Name())) if err := s.reloadConfigFromKV(); err != nil { log.Error("failed to reload configuration", errs.ZapError(err)) @@ -1727,17 +1728,17 @@ func (s *Server) campaignLeader() { } // EnableLeader to accept the remaining service, such as GetStore, GetRegion. s.member.EnableLeader() - member.ServiceMemberGauge.WithLabelValues(s.mode).Set(1) + member.ServiceMemberGauge.WithLabelValues(PD).Set(1) defer resetLeaderOnce.Do(func() { // as soon as cancel the leadership keepalive, then other member have chance // to be new leader. cancel() s.member.ResetLeader() - member.ServiceMemberGauge.WithLabelValues(s.mode).Set(0) + member.ServiceMemberGauge.WithLabelValues(PD).Set(0) }) CheckPDVersionWithClusterVersion(s.persistOptions) - log.Info(fmt.Sprintf("%s leader is ready to serve", s.mode), zap.String("leader-name", s.Name())) + log.Info("PD leader is ready to serve", zap.String("leader-name", s.Name())) leaderTicker := time.NewTicker(constant.LeaderTickInterval) defer leaderTicker.Stop() @@ -1746,7 +1747,7 @@ func (s *Server) campaignLeader() { select { case <-leaderTicker.C: if !s.member.IsLeader() { - log.Info("no longer a leader because lease has expired, pd leader will step down") + log.Info("no longer a leader because lease has expired, PD leader will step down") return } // add failpoint to test exit leader, failpoint judge the member is the give value, then break diff --git a/server/server_test.go b/server/server_test.go index 28839b89389..2c7e1fedef2 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -272,7 +272,7 @@ func TestAPIService(t *testing.T) { err = svr.Run() re.NoError(err) MustWaitLeader(re, []*Server{svr}) - re.True(svr.IsPDServiceMode()) + re.True(svr.IsKeyspaceGroupEnabled()) } func TestIsPathInDirectory(t *testing.T) { diff --git a/tests/cluster.go b/tests/cluster.go index 4189b43902a..66d8ce60e7b 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -429,8 +429,8 @@ func NewTestCluster(ctx context.Context, initialServerCount int, opts ...ConfigO return createTestCluster(ctx, initialServerCount, nil, opts...) } -// NewTestPDServiceCluster creates a new TestCluster with PD service. -func NewTestPDServiceCluster(ctx context.Context, initialServerCount int, opts ...ConfigOption) (*TestCluster, error) { +// NewTestClusterWithKeyspaceGroup creates a new TestCluster with PD. +func NewTestClusterWithKeyspaceGroup(ctx context.Context, initialServerCount int, opts ...ConfigOption) (*TestCluster, error) { return createTestCluster(ctx, initialServerCount, []string{constant.PDServiceName}, opts...) } @@ -461,13 +461,13 @@ func createTestCluster(ctx context.Context, initialServerCount int, services []s }, nil } -// RestartTestAPICluster restarts the API test cluster. -func RestartTestAPICluster(ctx context.Context, cluster *TestCluster) (*TestCluster, error) { +// RestartTestPDCluster restarts the PD test cluster. +func RestartTestPDCluster(ctx context.Context, cluster *TestCluster) (*TestCluster, error) { return restartTestCluster(ctx, cluster, true) } func restartTestCluster( - ctx context.Context, cluster *TestCluster, isPDServiceMode bool, + ctx context.Context, cluster *TestCluster, isKeyspaceGroupEnabled bool, ) (newTestCluster *TestCluster, err error) { schedulers.Register() newTestCluster = &TestCluster{ @@ -494,7 +494,7 @@ func restartTestCluster( newServer *TestServer serverErr error ) - if isPDServiceMode { + if isKeyspaceGroupEnabled { newServer, serverErr = NewTestServer(ctx, serverCfg, []string{constant.PDServiceName}) } else { newServer, serverErr = NewTestServer(ctx, serverCfg, nil) @@ -729,8 +729,8 @@ func (c *TestCluster) Join(ctx context.Context, opts ...ConfigOption) (*TestServ return s, nil } -// JoinPDServer is used to add a new TestServer into the cluster. -func (c *TestCluster) JoinPDServer(ctx context.Context, opts ...ConfigOption) (*TestServer, error) { +// JoinWithKeyspaceGroup is used to add a new TestServer into the cluster with keyspace group enabled. +func (c *TestCluster) JoinWithKeyspaceGroup(ctx context.Context, opts ...ConfigOption) (*TestServer, error) { conf, err := c.config.join().Generate(opts...) if err != nil { return nil, err diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index ab3874a33a7..6bd25567f81 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -361,7 +361,7 @@ func TestTSOFollowerProxyWithTSOService(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/client/servicediscovery/fastUpdateServiceMode", `return(true)`)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - cluster, err := tests.NewTestPDServiceCluster(ctx, 1) + cluster, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 1) re.NoError(err) defer cluster.Destroy() err = cluster.RunInitialServers() diff --git a/tests/integrations/mcs/discovery/register_test.go b/tests/integrations/mcs/discovery/register_test.go index eb8933e10d8..217d8137e5b 100644 --- a/tests/integrations/mcs/discovery/register_test.go +++ b/tests/integrations/mcs/discovery/register_test.go @@ -54,7 +54,7 @@ func (suite *serverRegisterTestSuite) SetupSuite() { re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() @@ -84,8 +84,7 @@ func (suite *serverRegisterTestSuite) checkServerRegister(serviceName string) { addr := s.GetAddr() client := suite.pdLeader.GetEtcdClient() - // test PD service discovery - + // test PD discovery endpoints, err := discovery.Discover(client, serviceName) re.NoError(err) returnedEntry := &discovery.ServiceRegistryEntry{} @@ -98,7 +97,7 @@ func (suite *serverRegisterTestSuite) checkServerRegister(serviceName string) { re.True(exist) re.Equal(expectedPrimary, primary) - // test PD service discovery after unregister + // test PD discovery after unregister cleanup() endpoints, err = discovery.Discover(client, serviceName) re.NoError(err) @@ -140,7 +139,7 @@ func (suite *serverRegisterTestSuite) checkServerPrimaryChange(serviceName strin delete(serverMap, primary) expectedPrimary = tests.WaitForPrimaryServing(re, serverMap) - // test PD service discovery + // test PD discovery client := suite.pdLeader.GetEtcdClient() endpoints, err := discovery.Discover(client, serviceName) re.NoError(err) diff --git a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go index b31d919324d..4644b2131d1 100644 --- a/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go +++ b/tests/integrations/mcs/keyspace/tso_keyspace_group_test.go @@ -60,7 +60,7 @@ func (suite *keyspaceGroupTestSuite) SetupTest() { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) ctx, cancel := context.WithCancel(context.Background()) suite.ctx = ctx - cluster, err := tests.NewTestPDServiceCluster(suite.ctx, 1) + cluster, err := tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) suite.cluster = cluster re.NoError(err) re.NoError(cluster.RunInitialServers()) diff --git a/tests/integrations/mcs/members/member_test.go b/tests/integrations/mcs/members/member_test.go index 0ec0b323d7a..7da0bf652bb 100644 --- a/tests/integrations/mcs/members/member_test.go +++ b/tests/integrations/mcs/members/member_test.go @@ -64,7 +64,7 @@ func (suite *memberTestSuite) SetupTest() { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) ctx, cancel := context.WithCancel(context.Background()) suite.ctx = ctx - cluster, err := tests.NewTestPDServiceCluster(suite.ctx, 1) + cluster, err := tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) suite.cluster = cluster re.NoError(err) re.NoError(cluster.RunInitialServers()) diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index e52ced10011..55407831df2 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -56,7 +56,7 @@ func (suite *apiTestSuite) TearDownSuite() { } func (suite *apiTestSuite) TestGetCheckerByName() { - suite.env.RunTestInPDServiceMode(suite.checkGetCheckerByName) + suite.env.RunTestInMicroserviceEnv(suite.checkGetCheckerByName) } func (suite *apiTestSuite) checkGetCheckerByName(cluster *tests.TestCluster) { @@ -102,7 +102,7 @@ func (suite *apiTestSuite) checkGetCheckerByName(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestAPIForward() { - suite.env.RunTestInPDServiceMode(suite.checkAPIForward) + suite.env.RunTestInMicroserviceEnv(suite.checkAPIForward) } func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { @@ -378,7 +378,7 @@ func (suite *apiTestSuite) checkAPIForward(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestConfig() { - suite.env.RunTestInPDServiceMode(suite.checkConfig) + suite.env.RunTestInMicroserviceEnv(suite.checkConfig) } func (suite *apiTestSuite) checkConfig(cluster *tests.TestCluster) { @@ -401,7 +401,7 @@ func (suite *apiTestSuite) checkConfig(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestConfigForward() { - suite.env.RunTestInPDServiceMode(suite.checkConfigForward) + suite.env.RunTestInMicroserviceEnv(suite.checkConfigForward) } func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { @@ -413,7 +413,7 @@ func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { urlPrefix := fmt.Sprintf("%s/pd/api/v1/config", addr) // Test config forward - // Expect to get same config in scheduling server and PD service + // Expect to get same config in scheduling server and PD testutil.Eventually(re, func() bool { testutil.ReadGetJSON(re, tests.TestDialClient, urlPrefix, &cfg) re.Equal(cfg["schedule"].(map[string]any)["leader-schedule-limit"], @@ -421,8 +421,8 @@ func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { return cfg["replication"].(map[string]any)["max-replicas"] == float64(opts.GetReplicationConfig().MaxReplicas) }) - // Test to change config in PD service - // Expect to get new config in scheduling server and PD service + // Test to change config in PD + // Expect to get new config in scheduling server and PD reqData, err := json.Marshal(map[string]any{ "max-replicas": 4, }) @@ -436,7 +436,7 @@ func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { }) // Test to change config only in scheduling server - // Expect to get new config in scheduling server but not old config in PD service + // Expect to get new config in scheduling server but not old config in PD scheCfg := opts.GetScheduleConfig().Clone() scheCfg.LeaderScheduleLimit = 100 opts.SetScheduleConfig(scheCfg) @@ -452,7 +452,7 @@ func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestAdminRegionCache() { - suite.env.RunTestInPDServiceMode(suite.checkAdminRegionCache) + suite.env.RunTestInMicroserviceEnv(suite.checkAdminRegionCache) } func (suite *apiTestSuite) checkAdminRegionCache(cluster *tests.TestCluster) { @@ -479,7 +479,7 @@ func (suite *apiTestSuite) checkAdminRegionCache(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestAdminRegionCacheForward() { - suite.env.RunTestInPDServiceMode(suite.checkAdminRegionCacheForward) + suite.env.RunTestInMicroserviceEnv(suite.checkAdminRegionCacheForward) } func (suite *apiTestSuite) checkAdminRegionCacheForward(cluster *tests.TestCluster) { @@ -510,7 +510,7 @@ func (suite *apiTestSuite) checkAdminRegionCacheForward(cluster *tests.TestClust } func (suite *apiTestSuite) TestFollowerForward() { - suite.env.RunTestBasedOnMode(suite.checkFollowerForward) + suite.env.RunTest(suite.checkFollowerForward) suite.TearDownSuite() suite.SetupSuite() } @@ -520,7 +520,7 @@ func (suite *apiTestSuite) checkFollowerForward(cluster *tests.TestCluster) { leaderAddr := cluster.GetLeaderServer().GetAddr() ctx, cancel := context.WithCancel(context.Background()) defer cancel() - follower, err := cluster.JoinPDServer(ctx) + follower, err := cluster.Join(ctx) re.NoError(err) re.NoError(follower.Run()) re.NotEmpty(cluster.WaitLeader()) @@ -558,7 +558,7 @@ func (suite *apiTestSuite) checkFollowerForward(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestMetrics() { - suite.env.RunTestInPDServiceMode(suite.checkMetrics) + suite.env.RunTestInMicroserviceEnv(suite.checkMetrics) } func (suite *apiTestSuite) checkMetrics(cluster *tests.TestCluster) { @@ -577,7 +577,7 @@ func (suite *apiTestSuite) checkMetrics(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestStatus() { - suite.env.RunTestInPDServiceMode(suite.checkStatus) + suite.env.RunTestInMicroserviceEnv(suite.checkStatus) } func (suite *apiTestSuite) checkStatus(cluster *tests.TestCluster) { @@ -600,7 +600,7 @@ func (suite *apiTestSuite) checkStatus(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestStores() { - suite.env.RunTestInPDServiceMode(suite.checkStores) + suite.env.RunTestInMicroserviceEnv(suite.checkStores) } func (suite *apiTestSuite) checkStores(cluster *tests.TestCluster) { @@ -647,8 +647,8 @@ func (suite *apiTestSuite) checkStores(cluster *tests.TestCluster) { tests.MustPutStore(re, cluster, store) } // Test /stores - pdServiceAddr := cluster.GetLeaderServer().GetAddr() - urlPrefix := fmt.Sprintf("%s/pd/api/v1/stores", pdServiceAddr) + pdAddr := cluster.GetLeaderServer().GetAddr() + urlPrefix := fmt.Sprintf("%s/pd/api/v1/stores", pdAddr) var resp map[string]any err := testutil.ReadGetJSON(re, tests.TestDialClient, urlPrefix, &resp) re.NoError(err) @@ -682,7 +682,7 @@ func (suite *apiTestSuite) checkStores(cluster *tests.TestCluster) { } func (suite *apiTestSuite) TestRegions() { - suite.env.RunTestInPDServiceMode(suite.checkRegions) + suite.env.RunTestInMicroserviceEnv(suite.checkRegions) } func (suite *apiTestSuite) checkRegions(cluster *tests.TestCluster) { @@ -691,8 +691,8 @@ func (suite *apiTestSuite) checkRegions(cluster *tests.TestCluster) { tests.MustPutRegion(re, cluster, 2, 2, []byte("c"), []byte("d")) tests.MustPutRegion(re, cluster, 3, 1, []byte("e"), []byte("f")) // Test /regions - pdServiceAddr := cluster.GetLeaderServer().GetAddr() - urlPrefix := fmt.Sprintf("%s/pd/api/v1/regions", pdServiceAddr) + pdAddr := cluster.GetLeaderServer().GetAddr() + urlPrefix := fmt.Sprintf("%s/pd/api/v1/regions", pdAddr) var resp map[string]any err := testutil.ReadGetJSON(re, tests.TestDialClient, urlPrefix, &resp) re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/config_test.go b/tests/integrations/mcs/scheduling/config_test.go index 6c770d3e4c1..caf86a665cb 100644 --- a/tests/integrations/mcs/scheduling/config_test.go +++ b/tests/integrations/mcs/scheduling/config_test.go @@ -62,7 +62,7 @@ func (suite *configTestSuite) SetupSuite() { schedulers.Register() var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() re.NoError(err) @@ -132,7 +132,7 @@ func (suite *configTestSuite) TestConfigWatch() { watcher.Close() } -// Manually trigger the config persistence in the PD service side. +// Manually trigger the config persistence in the PD side. func persistConfig(re *require.Assertions, pdLeaderServer *tests.TestServer) { err := pdLeaderServer.GetPersistOptions().Persist(pdLeaderServer.GetServer().GetStorage()) re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/meta_test.go b/tests/integrations/mcs/scheduling/meta_test.go index 8df576b82ca..b9c371bf12c 100644 --- a/tests/integrations/mcs/scheduling/meta_test.go +++ b/tests/integrations/mcs/scheduling/meta_test.go @@ -53,7 +53,7 @@ func (suite *metaTestSuite) SetupSuite() { re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() re.NoError(err) diff --git a/tests/integrations/mcs/scheduling/rule_test.go b/tests/integrations/mcs/scheduling/rule_test.go index 706c5784831..b741f147061 100644 --- a/tests/integrations/mcs/scheduling/rule_test.go +++ b/tests/integrations/mcs/scheduling/rule_test.go @@ -54,7 +54,7 @@ func (suite *ruleTestSuite) SetupSuite() { var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() re.NoError(err) @@ -97,7 +97,7 @@ func (suite *ruleTestSuite) TestRuleWatch() { re.Equal(placement.DefaultGroupID, ruleGroups[0].ID) re.Equal(0, ruleGroups[0].Index) re.False(ruleGroups[0].Override) - // Set a new rule via the PD service. + // Set a new rule via the PD. apiRuleManager := suite.pdLeaderServer.GetRaftCluster().GetRuleManager() rule := &placement.Rule{ GroupID: "2", diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index 7271111bd79..d79bcb47228 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -66,7 +66,7 @@ func (suite *serverTestSuite) SetupSuite() { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/scheduling/server/changeRunCollectWaitTime", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() @@ -220,7 +220,7 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { // Change back to the default value. conf.EnableSchedulingFallback = true leaderServer.SetMicroserviceConfig(*conf) - // PD service will execute scheduling jobs since there is no scheduling server. + // PD will execute scheduling jobs since there is no scheduling server. testutil.Eventually(re, func() bool { return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) @@ -229,7 +229,7 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) - // After scheduling server is started, PD service will not execute scheduling jobs. + // After scheduling server is started, PD will not execute scheduling jobs. testutil.Eventually(re, func() bool { return !suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) @@ -238,7 +238,7 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { return tc.GetPrimaryServer().GetCluster().IsBackgroundJobsRunning() }) tc.GetPrimaryServer().Close() - // Stop scheduling server. PD service will execute scheduling jobs again. + // Stop scheduling server. PD will execute scheduling jobs again. testutil.Eventually(re, func() bool { return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) @@ -246,7 +246,7 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { re.NoError(err) defer tc1.Destroy() tc1.WaitForPrimaryServing(re) - // After scheduling server is started, PD service will not execute scheduling jobs. + // After scheduling server is started, PD will not execute scheduling jobs. testutil.Eventually(re, func() bool { return !suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) @@ -259,21 +259,21 @@ func (suite *serverTestSuite) TestSchedulingServiceFallback() { func (suite *serverTestSuite) TestDisableSchedulingServiceFallback() { re := suite.Require() - // PD service will execute scheduling jobs since there is no scheduling server. + // PD will execute scheduling jobs since there is no scheduling server. testutil.Eventually(re, func() bool { re.NotNil(suite.pdLeader.GetServer()) re.NotNil(suite.pdLeader.GetServer().GetRaftCluster()) return suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) leaderServer := suite.pdLeader.GetServer() - // After Disabling scheduling service fallback, the PD service will stop scheduling. + // After Disabling scheduling service fallback, the PD will stop scheduling. conf := leaderServer.GetMicroserviceConfig().Clone() conf.EnableSchedulingFallback = false leaderServer.SetMicroserviceConfig(*conf) testutil.Eventually(re, func() bool { return !suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) - // Enable scheduling service fallback again, the PD service will restart scheduling. + // Enable scheduling service fallback again, the PD will restart scheduling. conf.EnableSchedulingFallback = true leaderServer.SetMicroserviceConfig(*conf) testutil.Eventually(re, func() bool { @@ -284,7 +284,7 @@ func (suite *serverTestSuite) TestDisableSchedulingServiceFallback() { re.NoError(err) defer tc.Destroy() tc.WaitForPrimaryServing(re) - // After scheduling server is started, PD service will not execute scheduling jobs. + // After scheduling server is started, PD will not execute scheduling jobs. testutil.Eventually(re, func() bool { return !suite.pdLeader.GetServer().GetRaftCluster().IsSchedulingControllerRunning() }) @@ -292,7 +292,7 @@ func (suite *serverTestSuite) TestDisableSchedulingServiceFallback() { testutil.Eventually(re, func() bool { return tc.GetPrimaryServer().GetCluster().IsBackgroundJobsRunning() }) - // Disable scheduling service fallback and stop scheduling server. PD service won't execute scheduling jobs again. + // Disable scheduling service fallback and stop scheduling server. PD won't execute scheduling jobs again. conf.EnableSchedulingFallback = false leaderServer.SetMicroserviceConfig(*conf) tc.GetPrimaryServer().Close() @@ -310,14 +310,14 @@ func (suite *serverTestSuite) TestSchedulerSync() { tc.WaitForPrimaryServing(re) schedulersController := tc.GetPrimaryServer().GetCluster().GetCoordinator().GetSchedulersController() checkEvictLeaderSchedulerExist(re, schedulersController, false) - // Add a new evict-leader-scheduler through the PD service. + // Add a new evict-leader-scheduler through the PD. api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) // Check if the evict-leader-scheduler is added. checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1}) - // Add a store_id to the evict-leader-scheduler through the PD service. + // Add a store_id to the evict-leader-scheduler through the PD. err = suite.pdLeader.GetServer().GetRaftCluster().PutMetaStore( &metapb.Store{ Id: 2, @@ -334,18 +334,18 @@ func (suite *serverTestSuite) TestSchedulerSync() { }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) - // Delete a store_id from the evict-leader-scheduler through the PD service. + // Delete a store_id from the evict-leader-scheduler through the PD. api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", types.EvictLeaderScheduler.String(), 1)) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{2}) - // Add a store_id to the evict-leader-scheduler through the PD service by the scheduler handler. + // Add a store_id to the evict-leader-scheduler through the PD by the scheduler handler. api.MustCallSchedulerConfigAPI(re, http.MethodPost, suite.backendEndpoints, types.EvictLeaderScheduler.String(), []string{"config"}, map[string]any{ "name": types.EvictLeaderScheduler.String(), "store_id": 1, }) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2}) - // Delete a store_id from the evict-leader-scheduler through the PD service by the scheduler handler. + // Delete a store_id from the evict-leader-scheduler through the PD by the scheduler handler. api.MustCallSchedulerConfigAPI(re, http.MethodDelete, suite.backendEndpoints, types.EvictLeaderScheduler.String(), []string{"delete", "2"}, nil) checkEvictLeaderSchedulerExist(re, schedulersController, true) checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1}) @@ -354,7 +354,7 @@ func (suite *serverTestSuite) TestSchedulerSync() { // Check if the scheduler is removed. checkEvictLeaderSchedulerExist(re, schedulersController, false) - // Delete the evict-leader-scheduler through the PD service by removing the last store_id. + // Delete the evict-leader-scheduler through the PD by removing the last store_id. api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) @@ -363,7 +363,7 @@ func (suite *serverTestSuite) TestSchedulerSync() { api.MustDeleteScheduler(re, suite.backendEndpoints, fmt.Sprintf("%s-%d", types.EvictLeaderScheduler.String(), 1)) checkEvictLeaderSchedulerExist(re, schedulersController, false) - // Delete the evict-leader-scheduler through the PD service. + // Delete the evict-leader-scheduler through the PD. api.MustAddScheduler(re, suite.backendEndpoints, types.EvictLeaderScheduler.String(), map[string]any{ "store_id": 1, }) @@ -551,7 +551,7 @@ func (suite *serverTestSuite) TestStoreLimit() { leaderServer.GetRaftCluster().SetStoreLimit(1, storelimit.RemovePeer, 60) leaderServer.GetRaftCluster().SetStoreLimit(2, storelimit.AddPeer, 60) leaderServer.GetRaftCluster().SetStoreLimit(2, storelimit.RemovePeer, 60) - // There is a time window between setting store limit in PD service side and capturing the change in scheduling service. + // There is a time window between setting store limit in PD side and capturing the change in scheduling service. waitSyncFinish(re, tc, storelimit.AddPeer, 60) for i := uint64(1); i <= 5; i++ { op := operator.NewTestOperator(2, &metapb.RegionEpoch{}, operator.OpRegion, operator.AddPeer{ToStore: 2, PeerID: 100}) @@ -636,7 +636,7 @@ func (suite *multipleServerTestSuite) SetupSuite() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 2) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 2) re.NoError(err) err = suite.cluster.RunInitialServers() diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index 97f56cbcee1..aa153a7eb9f 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -62,7 +62,7 @@ func (suite *tsoAPITestSuite) SetupTest() { var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.pdCluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) + suite.pdCluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) re.NoError(err) err = suite.pdCluster.RunInitialServers() re.NoError(err) @@ -137,7 +137,7 @@ func TestTSOServerStartFirst(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - apiCluster, err := tests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { + apiCluster, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = []string{"k1", "k2"} }) defer apiCluster.Destroy() @@ -200,7 +200,7 @@ func TestForwardOnlyTSONoScheduling(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tc, err := tests.NewTestPDServiceCluster(ctx, 1) + tc, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 1) defer tc.Destroy() re.NoError(err) err = tc.RunInitialServers() diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index ecbc0295845..51ace0f08c4 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -82,7 +82,7 @@ func (suite *tsoKeyspaceGroupManagerTestSuite) SetupSuite() { var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() re.NoError(err) @@ -537,8 +537,8 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller", `return(true)`)) - // Init PD service config but not start. - tc, err := tests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { + // Init PD config but not start. + tc, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = []string{ "keyspace_a", "keyspace_b", } @@ -546,7 +546,7 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() - // Start PD service and tso server. + // Start PD and tso server. err = tc.RunInitialServers() re.NoError(err) defer tc.Destroy() @@ -734,8 +734,8 @@ func TestGetTSOImmediately(t *testing.T) { re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/acceleratedAllocNodes", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/tso/fastGroupSplitPatroller", `return(true)`)) - // Init PD service config but not start. - tc, err := tests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { + // Init PD config but not start. + tc, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = []string{ "keyspace_a", "keyspace_b", } @@ -743,7 +743,7 @@ func TestGetTSOImmediately(t *testing.T) { re.NoError(err) pdAddr := tc.GetConfig().GetClientURL() - // Start PD service and tso server. + // Start PD and tso server. err = tc.RunInitialServers() re.NoError(err) defer tc.Destroy() diff --git a/tests/integrations/mcs/tso/proxy_test.go b/tests/integrations/mcs/tso/proxy_test.go index 50583ebbbb4..99213742e6c 100644 --- a/tests/integrations/mcs/tso/proxy_test.go +++ b/tests/integrations/mcs/tso/proxy_test.go @@ -62,7 +62,7 @@ func (s *tsoProxyTestSuite) SetupSuite() { var err error s.ctx, s.cancel = context.WithCancel(context.Background()) // Create an API cluster with 1 server - s.apiCluster, err = tests.NewTestPDServiceCluster(s.ctx, 1) + s.apiCluster, err = tests.NewTestClusterWithKeyspaceGroup(s.ctx, 1) re.NoError(err) err = s.apiCluster.RunInitialServers() re.NoError(err) diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 4c9bb4248e5..168741204de 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -75,7 +75,7 @@ func (suite *tsoServerTestSuite) SetupSuite() { re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() @@ -156,19 +156,19 @@ func (suite *tsoServerTestSuite) TestParticipantStartWithAdvertiseListenAddr() { func TestTSOPath(t *testing.T) { re := require.New(t) - checkTSOPath(re, true /*isPDServiceMode*/) - checkTSOPath(re, false /*isPDServiceMode*/) + checkTSOPath(re, true /*isKeyspaceGroupEnabled*/) + checkTSOPath(re, false /*isKeyspaceGroupEnabled*/) } -func checkTSOPath(re *require.Assertions, isPDServiceMode bool) { +func checkTSOPath(re *require.Assertions, isKeyspaceGroupEnabled bool) { var ( cluster *tests.TestCluster err error ) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - if isPDServiceMode { - cluster, err = tests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { + if isKeyspaceGroupEnabled { + cluster, err = tests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Microservice.EnableTSODynamicSwitching = false }) } else { @@ -184,7 +184,7 @@ func checkTSOPath(re *require.Assertions, isPDServiceMode bool) { re.NoError(pdLeader.BootstrapCluster()) backendEndpoints := pdLeader.GetAddr() client := pdLeader.GetEtcdClient() - if isPDServiceMode { + if isKeyspaceGroupEnabled { re.Equal(0, getEtcdTimestampKeyNum(re, client)) } else { re.Equal(1, getEtcdTimestampKeyNum(re, client)) @@ -233,7 +233,7 @@ func NewPDServiceForward(re *require.Assertions) PDServiceForward { } var err error suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 3) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 3) re.NoError(err) err = suite.cluster.RunInitialServers() @@ -512,7 +512,7 @@ func (suite *CommonTestSuite) SetupSuite() { var err error re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, 1) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) re.NoError(err) err = suite.cluster.RunInitialServers() @@ -576,7 +576,7 @@ func (suite *CommonTestSuite) TestBootstrapDefaultKeyspaceGroup() { } check() - s, err := suite.cluster.JoinPDServer(suite.ctx) + s, err := suite.cluster.JoinWithKeyspaceGroup(suite.ctx) re.NoError(err) re.NoError(s.Run()) @@ -598,7 +598,7 @@ func TestTSOServiceSwitch(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tc, err := tests.NewTestPDServiceCluster(ctx, 1, + tc, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Microservice.EnableTSODynamicSwitching = true }, diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index 87ddd9e226b..4822d6439ca 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -98,7 +98,7 @@ func (suite *tsoClientTestSuite) SetupSuite() { if suite.legacy { suite.cluster, err = tests.NewTestCluster(suite.ctx, serverCount) } else { - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, serverCount, func(conf *config.Config, _ string) { + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, serverCount, func(conf *config.Config, _ string) { conf.Microservice.EnableTSODynamicSwitching = false }) } @@ -510,9 +510,9 @@ func TestMixedTSODeployment(t *testing.T) { re.NotNil(leaderServer) backendEndpoints := leaderServer.GetAddr() - apiSvr, err := cluster.JoinPDServer(ctx) + pdSvr, err := cluster.Join(ctx) re.NoError(err) - err = apiSvr.Run() + err = pdSvr.Run() re.NoError(err) s, cleanup := tests.StartSingleTSOTestServer(ctx, re, backendEndpoints, tempurl.Alloc()) @@ -537,20 +537,20 @@ func TestMixedTSODeployment(t *testing.T) { wg.Wait() } -// TestUpgradingAPIandTSOClusters tests the scenario that after we restart the API cluster +// TestUpgradingPDAndTSOClusters tests the scenario that after we restart the PD cluster // then restart the TSO cluster, the TSO service can still serve TSO requests normally. -func TestUpgradingAPIandTSOClusters(t *testing.T) { +func TestUpgradingPDAndTSOClusters(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) - // Create an API cluster which has 3 servers - apiCluster, err := tests.NewTestPDServiceCluster(ctx, 3) + // Create an PD cluster which has 3 servers + pdCluster, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 3) re.NoError(err) - err = apiCluster.RunInitialServers() + err = pdCluster.RunInitialServers() re.NoError(err) - leaderName := apiCluster.WaitLeader() + leaderName := pdCluster.WaitLeader() re.NotEmpty(leaderName) - pdLeader := apiCluster.GetServer(leaderName) + pdLeader := pdCluster.GetServer(leaderName) backendEndpoints := pdLeader.GetAddr() // Create a pd client in PD mode to let the API leader to forward requests to the TSO cluster. @@ -569,7 +569,7 @@ func TestUpgradingAPIandTSOClusters(t *testing.T) { mcs.WaitForTSOServiceAvailable(ctx, re, pdClient) // Restart the API cluster - apiCluster, err = tests.RestartTestAPICluster(ctx, apiCluster) + pdCluster, err = tests.RestartTestPDCluster(ctx, pdCluster) re.NoError(err) // The TSO service should be eventually healthy mcs.WaitForTSOServiceAvailable(ctx, re, pdClient) @@ -581,7 +581,7 @@ func TestUpgradingAPIandTSOClusters(t *testing.T) { mcs.WaitForTSOServiceAvailable(ctx, re, pdClient) tsoCluster.Destroy() - apiCluster.Destroy() + pdCluster.Destroy() cancel() re.NoError(failpoint.Disable("github.com/tikv/pd/client/servicediscovery/usePDServiceMode")) } diff --git a/tests/integrations/tso/consistency_test.go b/tests/integrations/tso/consistency_test.go index b29ae696f26..b4c3ffef790 100644 --- a/tests/integrations/tso/consistency_test.go +++ b/tests/integrations/tso/consistency_test.go @@ -76,7 +76,7 @@ func (suite *tsoConsistencyTestSuite) SetupSuite() { if suite.legacy { suite.cluster, err = tests.NewTestCluster(suite.ctx, serverCount) } else { - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, serverCount) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, serverCount) } re.NoError(err) err = suite.cluster.RunInitialServers() diff --git a/tests/integrations/tso/server_test.go b/tests/integrations/tso/server_test.go index 1428dbcd1a6..3860f054c31 100644 --- a/tests/integrations/tso/server_test.go +++ b/tests/integrations/tso/server_test.go @@ -74,7 +74,7 @@ func (suite *tsoServerTestSuite) SetupSuite() { if suite.legacy { suite.cluster, err = tests.NewTestCluster(suite.ctx, serverCount) } else { - suite.cluster, err = tests.NewTestPDServiceCluster(suite.ctx, serverCount) + suite.cluster, err = tests.NewTestClusterWithKeyspaceGroup(suite.ctx, serverCount) } re.NoError(err) err = suite.cluster.RunInitialServers() diff --git a/tests/server/api/checker_test.go b/tests/server/api/checker_test.go index 14077ec30ae..5444fbf730b 100644 --- a/tests/server/api/checker_test.go +++ b/tests/server/api/checker_test.go @@ -45,7 +45,7 @@ func (suite *checkerTestSuite) TearDownSuite() { } func (suite *checkerTestSuite) TestAPI() { - suite.env.RunTestBasedOnMode(suite.checkAPI) + suite.env.RunTest(suite.checkAPI) } func (suite *checkerTestSuite) checkAPI(cluster *tests.TestCluster) { diff --git a/tests/server/api/operator_test.go b/tests/server/api/operator_test.go index 54abdf2d236..9e87215b490 100644 --- a/tests/server/api/operator_test.go +++ b/tests/server/api/operator_test.go @@ -57,7 +57,7 @@ func (suite *operatorTestSuite) TearDownSuite() { } func (suite *operatorTestSuite) TestAddRemovePeer() { - suite.env.RunTestBasedOnMode(suite.checkAddRemovePeer) + suite.env.RunTest(suite.checkAddRemovePeer) } func (suite *operatorTestSuite) checkAddRemovePeer(cluster *tests.TestCluster) { @@ -167,7 +167,7 @@ func (suite *operatorTestSuite) checkAddRemovePeer(cluster *tests.TestCluster) { } func (suite *operatorTestSuite) TestMergeRegionOperator() { - suite.env.RunTestBasedOnMode(suite.checkMergeRegionOperator) + suite.env.RunTest(suite.checkMergeRegionOperator) } func (suite *operatorTestSuite) checkMergeRegionOperator(cluster *tests.TestCluster) { @@ -227,7 +227,7 @@ func (suite *operatorTestSuite) TestTransferRegionWithPlacementRule() { func(conf *config.Config, _ string) { conf.Replication.MaxReplicas = 3 }) - env.RunTestBasedOnMode(suite.checkTransferRegionWithPlacementRule) + env.RunTest(suite.checkTransferRegionWithPlacementRule) env.Cleanup() } @@ -507,7 +507,7 @@ func (suite *operatorTestSuite) TestGetOperatorsAsObject() { func(conf *config.Config, _ string) { conf.Replication.MaxReplicas = 1 }) - env.RunTestBasedOnMode(suite.checkGetOperatorsAsObject) + env.RunTest(suite.checkGetOperatorsAsObject) env.Cleanup() } @@ -604,7 +604,7 @@ func (suite *operatorTestSuite) checkGetOperatorsAsObject(cluster *tests.TestClu } func (suite *operatorTestSuite) TestRemoveOperators() { - suite.env.RunTestBasedOnMode(suite.checkRemoveOperators) + suite.env.RunTest(suite.checkRemoveOperators) } func (suite *operatorTestSuite) checkRemoveOperators(cluster *tests.TestCluster) { diff --git a/tests/server/api/region_test.go b/tests/server/api/region_test.go index b98b4419a80..ccd70aaae87 100644 --- a/tests/server/api/region_test.go +++ b/tests/server/api/region_test.go @@ -93,13 +93,13 @@ func (suite *regionTestSuite) TearDownTest() { return true }) } - suite.env.RunTestBasedOnMode(cleanFunc) + suite.env.RunTest(cleanFunc) } func (suite *regionTestSuite) TestSplitRegions() { // use a new environment to avoid affecting other tests env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestBasedOnMode(suite.checkSplitRegions) + env.RunTest(suite.checkSplitRegions) env.Cleanup() } @@ -142,7 +142,7 @@ func (suite *regionTestSuite) checkSplitRegions(cluster *tests.TestCluster) { func (suite *regionTestSuite) TestAccelerateRegionsScheduleInRange() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/checker/skipCheckSuspectRanges", "return(true)")) - suite.env.RunTestBasedOnMode(suite.checkAccelerateRegionsScheduleInRange) + suite.env.RunTest(suite.checkAccelerateRegionsScheduleInRange) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/checker/skipCheckSuspectRanges")) } @@ -180,7 +180,7 @@ func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRange(cluster *tes func (suite *regionTestSuite) TestAccelerateRegionsScheduleInRanges() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/checker/skipCheckSuspectRanges", "return(true)")) - suite.env.RunTestBasedOnMode(suite.checkAccelerateRegionsScheduleInRanges) + suite.env.RunTest(suite.checkAccelerateRegionsScheduleInRanges) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/checker/skipCheckSuspectRanges")) } @@ -219,7 +219,7 @@ func (suite *regionTestSuite) checkAccelerateRegionsScheduleInRanges(cluster *te func (suite *regionTestSuite) TestScatterRegions() { // use a new environment to avoid affecting other tests env := tests.NewSchedulingTestEnvironment(suite.T()) - env.RunTestBasedOnMode(suite.checkScatterRegions) + env.RunTest(suite.checkScatterRegions) env.Cleanup() } @@ -266,7 +266,7 @@ func (suite *regionTestSuite) checkScatterRegions(cluster *tests.TestCluster) { } func (suite *regionTestSuite) TestCheckRegionsReplicated() { - suite.env.RunTestBasedOnMode(suite.checkRegionsReplicated) + suite.env.RunTest(suite.checkRegionsReplicated) } func (suite *regionTestSuite) checkRegionsReplicated(cluster *tests.TestCluster) { diff --git a/tests/server/api/rule_test.go b/tests/server/api/rule_test.go index 3982e61b6ea..58832c26b42 100644 --- a/tests/server/api/rule_test.go +++ b/tests/server/api/rule_test.go @@ -76,11 +76,11 @@ func (suite *ruleTestSuite) TearDownTest() { err = tu.CheckPostJSON(tests.TestDialClient, urlPrefix+"/pd/api/v1/config/placement-rule", data, tu.StatusOK(re)) re.NoError(err) } - suite.env.RunTestBasedOnMode(cleanFunc) + suite.env.RunTest(cleanFunc) } func (suite *ruleTestSuite) TestSet() { - suite.env.RunTestBasedOnMode(suite.checkSet) + suite.env.RunTest(suite.checkSet) } func (suite *ruleTestSuite) checkSet(cluster *tests.TestCluster) { @@ -196,7 +196,7 @@ func (suite *ruleTestSuite) checkSet(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestGet() { - suite.env.RunTestBasedOnMode(suite.checkGet) + suite.env.RunTest(suite.checkGet) } func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) { @@ -247,7 +247,7 @@ func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestGetAll() { - suite.env.RunTestBasedOnMode(suite.checkGetAll) + suite.env.RunTest(suite.checkGetAll) } func (suite *ruleTestSuite) checkGetAll(cluster *tests.TestCluster) { @@ -269,7 +269,7 @@ func (suite *ruleTestSuite) checkGetAll(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestSetAll() { - suite.env.RunTestBasedOnMode(suite.checkSetAll) + suite.env.RunTest(suite.checkSetAll) } func (suite *ruleTestSuite) checkSetAll(cluster *tests.TestCluster) { @@ -385,7 +385,7 @@ func (suite *ruleTestSuite) checkSetAll(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestGetAllByGroup() { - suite.env.RunTestBasedOnMode(suite.checkGetAllByGroup) + suite.env.RunTest(suite.checkGetAllByGroup) } func (suite *ruleTestSuite) checkGetAllByGroup(cluster *tests.TestCluster) { @@ -442,7 +442,7 @@ func (suite *ruleTestSuite) checkGetAllByGroup(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestGetAllByRegion() { - suite.env.RunTestBasedOnMode(suite.checkGetAllByRegion) + suite.env.RunTest(suite.checkGetAllByRegion) } func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { @@ -507,7 +507,7 @@ func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestGetAllByKey() { - suite.env.RunTestBasedOnMode(suite.checkGetAllByKey) + suite.env.RunTest(suite.checkGetAllByKey) } func (suite *ruleTestSuite) checkGetAllByKey(cluster *tests.TestCluster) { @@ -566,7 +566,7 @@ func (suite *ruleTestSuite) checkGetAllByKey(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestDelete() { - suite.env.RunTestBasedOnMode(suite.checkDelete) + suite.env.RunTest(suite.checkDelete) } func (suite *ruleTestSuite) checkDelete(cluster *tests.TestCluster) { @@ -632,7 +632,7 @@ func (suite *ruleTestSuite) checkDelete(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestBatch() { - suite.env.RunTestBasedOnMode(suite.checkBatch) + suite.env.RunTest(suite.checkBatch) } func (suite *ruleTestSuite) checkBatch(cluster *tests.TestCluster) { @@ -761,7 +761,7 @@ func (suite *ruleTestSuite) checkBatch(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestBundle() { - suite.env.RunTestBasedOnMode(suite.checkBundle) + suite.env.RunTest(suite.checkBundle) } func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { @@ -871,7 +871,7 @@ func (suite *ruleTestSuite) checkBundle(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestBundleBadRequest() { - suite.env.RunTestBasedOnMode(suite.checkBundleBadRequest) + suite.env.RunTest(suite.checkBundleBadRequest) } func (suite *ruleTestSuite) checkBundleBadRequest(cluster *tests.TestCluster) { @@ -902,7 +902,7 @@ func (suite *ruleTestSuite) checkBundleBadRequest(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestLeaderAndVoter() { - suite.env.RunTestBasedOnMode(suite.checkLeaderAndVoter) + suite.env.RunTest(suite.checkLeaderAndVoter) } func (suite *ruleTestSuite) checkLeaderAndVoter(cluster *tests.TestCluster) { @@ -993,7 +993,7 @@ func (suite *ruleTestSuite) checkLeaderAndVoter(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestDeleteAndUpdate() { - suite.env.RunTestBasedOnMode(suite.checkDeleteAndUpdate) + suite.env.RunTest(suite.checkDeleteAndUpdate) } func (suite *ruleTestSuite) checkDeleteAndUpdate(cluster *tests.TestCluster) { @@ -1074,7 +1074,7 @@ func (suite *ruleTestSuite) checkDeleteAndUpdate(cluster *tests.TestCluster) { } func (suite *ruleTestSuite) TestConcurrency() { - suite.env.RunTestBasedOnMode(suite.checkConcurrency) + suite.env.RunTest(suite.checkConcurrency) } func (suite *ruleTestSuite) checkConcurrency(cluster *tests.TestCluster) { @@ -1163,7 +1163,7 @@ func (suite *ruleTestSuite) checkConcurrencyWith(cluster *tests.TestCluster, } func (suite *ruleTestSuite) TestLargeRules() { - suite.env.RunTestBasedOnMode(suite.checkLargeRules) + suite.env.RunTest(suite.checkLargeRules) } func (suite *ruleTestSuite) checkLargeRules(cluster *tests.TestCluster) { @@ -1292,7 +1292,7 @@ func (suite *regionRuleTestSuite) TearDownSuite() { } func (suite *regionRuleTestSuite) TestRegionPlacementRule() { - suite.env.RunTestBasedOnMode(suite.checkRegionPlacementRule) + suite.env.RunTest(suite.checkRegionPlacementRule) } func (suite *regionRuleTestSuite) checkRegionPlacementRule(cluster *tests.TestCluster) { diff --git a/tests/server/api/scheduler_test.go b/tests/server/api/scheduler_test.go index d1d5b06ceb4..b01e569e2f0 100644 --- a/tests/server/api/scheduler_test.go +++ b/tests/server/api/scheduler_test.go @@ -43,19 +43,19 @@ const apiPrefix = "/pd" type scheduleTestSuite struct { suite.Suite - env *tests.SchedulingTestEnvironment - runMode tests.SchedulerMode + te *tests.SchedulingTestEnvironment + env tests.Env } -func TestPDSchedulingTestSuite(t *testing.T) { +func TestNonMicroserviceSchedulingTestSuite(t *testing.T) { suite.Run(t, &scheduleTestSuite{ - runMode: tests.PDMode, + env: tests.NonMicroserviceEnv, }) } -func TestAPISchedulingTestSuite(t *testing.T) { +func TestMicroserviceSchedulingTestSuite(t *testing.T) { suite.Run(t, &scheduleTestSuite{ - runMode: tests.PDServiceMode, + env: tests.MicroserviceEnv, }) } @@ -63,19 +63,19 @@ func (suite *scheduleTestSuite) SetupSuite() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/changeCoordinatorTicker", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/skipStoreConfigSync", `return(true)`)) - suite.env = tests.NewSchedulingTestEnvironment(suite.T()) - suite.env.RunMode = suite.runMode + suite.te = tests.NewSchedulingTestEnvironment(suite.T()) + suite.te.Env = suite.env } func (suite *scheduleTestSuite) TearDownSuite() { re := suite.Require() - suite.env.Cleanup() + suite.te.Cleanup() re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/skipStoreConfigSync")) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/changeCoordinatorTicker")) } func (suite *scheduleTestSuite) TestOriginAPI() { - suite.env.RunTestBasedOnMode(suite.checkOriginAPI) + suite.te.RunTest(suite.checkOriginAPI) } func (suite *scheduleTestSuite) checkOriginAPI(cluster *tests.TestCluster) { @@ -157,7 +157,7 @@ func (suite *scheduleTestSuite) checkOriginAPI(cluster *tests.TestCluster) { } func (suite *scheduleTestSuite) TestAPI() { - suite.env.RunTestBasedOnMode(suite.checkAPI) + suite.te.RunTest(suite.checkAPI) } func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { @@ -654,7 +654,7 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) { } func (suite *scheduleTestSuite) TestDisable() { - suite.env.RunTestBasedOnMode(suite.checkDisable) + suite.te.RunTest(suite.checkDisable) } func (suite *scheduleTestSuite) checkDisable(cluster *tests.TestCluster) { @@ -765,7 +765,7 @@ func (suite *scheduleTestSuite) testPauseOrResume(re *require.Assertions, urlPre } func (suite *scheduleTestSuite) TestEmptySchedulers() { - suite.env.RunTestBasedOnMode(suite.checkEmptySchedulers) + suite.te.RunTest(suite.checkEmptySchedulers) } func (suite *scheduleTestSuite) checkEmptySchedulers(cluster *tests.TestCluster) { diff --git a/tests/server/apiv2/handlers/tso_keyspace_group_test.go b/tests/server/apiv2/handlers/tso_keyspace_group_test.go index 851df9b5fd1..17f326f4a4d 100644 --- a/tests/server/apiv2/handlers/tso_keyspace_group_test.go +++ b/tests/server/apiv2/handlers/tso_keyspace_group_test.go @@ -42,7 +42,7 @@ func TestKeyspaceGroupTestSuite(t *testing.T) { func (suite *keyspaceGroupTestSuite) SetupTest() { re := suite.Require() suite.ctx, suite.cancel = context.WithCancel(context.Background()) - cluster, err := tests.NewTestPDServiceCluster(suite.ctx, 1) + cluster, err := tests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) suite.cluster = cluster re.NoError(err) re.NoError(cluster.RunInitialServers()) diff --git a/tests/server/config/config_test.go b/tests/server/config/config_test.go index bb387b68030..ced0a6b8261 100644 --- a/tests/server/config/config_test.go +++ b/tests/server/config/config_test.go @@ -92,7 +92,7 @@ func (suite *configTestSuite) TearDownSuite() { } func (suite *configTestSuite) TestConfigAll() { - suite.env.RunTestBasedOnMode(suite.checkConfigAll) + suite.env.RunTest(suite.checkConfigAll) } func (suite *configTestSuite) checkConfigAll(cluster *tests.TestCluster) { @@ -213,7 +213,7 @@ func (suite *configTestSuite) checkConfigAll(cluster *tests.TestCluster) { } func (suite *configTestSuite) TestConfigSchedule() { - suite.env.RunTestBasedOnMode(suite.checkConfigSchedule) + suite.env.RunTest(suite.checkConfigSchedule) } func (suite *configTestSuite) checkConfigSchedule(cluster *tests.TestCluster) { @@ -239,7 +239,7 @@ func (suite *configTestSuite) checkConfigSchedule(cluster *tests.TestCluster) { } func (suite *configTestSuite) TestConfigReplication() { - suite.env.RunTestBasedOnMode(suite.checkConfigReplication) + suite.env.RunTest(suite.checkConfigReplication) } func (suite *configTestSuite) checkConfigReplication(cluster *tests.TestCluster) { @@ -282,7 +282,7 @@ func (suite *configTestSuite) checkConfigReplication(cluster *tests.TestCluster) } func (suite *configTestSuite) TestConfigLabelProperty() { - suite.env.RunTestBasedOnMode(suite.checkConfigLabelProperty) + suite.env.RunTest(suite.checkConfigLabelProperty) } func (suite *configTestSuite) checkConfigLabelProperty(cluster *tests.TestCluster) { @@ -334,7 +334,7 @@ func (suite *configTestSuite) checkConfigLabelProperty(cluster *tests.TestCluste } func (suite *configTestSuite) TestConfigDefault() { - suite.env.RunTestBasedOnMode(suite.checkConfigDefault) + suite.env.RunTest(suite.checkConfigDefault) } func (suite *configTestSuite) checkConfigDefault(cluster *tests.TestCluster) { @@ -378,7 +378,7 @@ func (suite *configTestSuite) checkConfigDefault(cluster *tests.TestCluster) { } func (suite *configTestSuite) TestConfigPDServer() { - suite.env.RunTestBasedOnMode(suite.checkConfigPDServer) + suite.env.RunTest(suite.checkConfigPDServer) } func (suite *configTestSuite) checkConfigPDServer(cluster *tests.TestCluster) { @@ -508,7 +508,7 @@ func createTTLUrl(url string, ttl int) string { } func (suite *configTestSuite) TestConfigTTL() { - suite.env.RunTestBasedOnMode(suite.checkConfigTTL) + suite.env.RunTest(suite.checkConfigTTL) } func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { @@ -571,7 +571,7 @@ func (suite *configTestSuite) checkConfigTTL(cluster *tests.TestCluster) { } func (suite *configTestSuite) TestTTLConflict() { - suite.env.RunTestBasedOnMode(suite.checkTTLConflict) + suite.env.RunTest(suite.checkTTLConflict) } func (suite *configTestSuite) checkTTLConflict(cluster *tests.TestCluster) { diff --git a/tests/testutil.go b/tests/testutil.go index 4bbfa8155b4..406e09345b9 100644 --- a/tests/testutil.go +++ b/tests/testutil.go @@ -271,25 +271,25 @@ func MustReportBuckets(re *require.Assertions, cluster *TestCluster, regionID ui return buckets } -// SchedulerMode is used for test purpose. -type SchedulerMode int +// Env is used for test purpose. +type Env int const ( - // Both represents both PD mode and API mode. - Both SchedulerMode = iota - // PDMode represents PD mode. - PDMode - // PDServiceMode represents API mode. - PDServiceMode + // Both represents both scheduler environments. + Both Env = iota + // NonMicroserviceEnv represents non-microservice env. + NonMicroserviceEnv + // MicroserviceEnv represents microservice env. + MicroserviceEnv ) // SchedulingTestEnvironment is used for test purpose. type SchedulingTestEnvironment struct { t *testing.T opts []ConfigOption - clusters map[SchedulerMode]*TestCluster + clusters map[Env]*TestCluster cancels []context.CancelFunc - RunMode SchedulerMode + Env Env } // NewSchedulingTestEnvironment is to create a new SchedulingTestEnvironment. @@ -297,38 +297,38 @@ func NewSchedulingTestEnvironment(t *testing.T, opts ...ConfigOption) *Schedulin return &SchedulingTestEnvironment{ t: t, opts: opts, - clusters: make(map[SchedulerMode]*TestCluster), + clusters: make(map[Env]*TestCluster), cancels: make([]context.CancelFunc, 0), } } -// RunTestBasedOnMode runs test based on mode. -// If mode not set, it will run test in both PD mode and API mode. -func (s *SchedulingTestEnvironment) RunTestBasedOnMode(test func(*TestCluster)) { - switch s.RunMode { - case PDMode: - s.RunTestInPDMode(test) - case PDServiceMode: - s.RunTestInPDServiceMode(test) +// RunTest is to run test based on the environment. +// If env not set, it will run test in both non-microservice env and microservice env. +func (s *SchedulingTestEnvironment) RunTest(test func(*TestCluster)) { + switch s.Env { + case NonMicroserviceEnv: + s.RunTestInNonMicroserviceEnv(test) + case MicroserviceEnv: + s.RunTestInMicroserviceEnv(test) default: - s.RunTestInPDMode(test) - s.RunTestInPDServiceMode(test) + s.RunTestInNonMicroserviceEnv(test) + s.RunTestInMicroserviceEnv(test) } } -// RunTestInPDMode is to run test in pd mode. -func (s *SchedulingTestEnvironment) RunTestInPDMode(test func(*TestCluster)) { - s.t.Logf("start test %s in pd mode", getTestName()) - if _, ok := s.clusters[PDMode]; !ok { - s.startCluster(PDMode) +// RunTestInNonMicroserviceMode is to run test in non-microservice environment. +func (s *SchedulingTestEnvironment) RunTestInNonMicroserviceEnv(test func(*TestCluster)) { + s.t.Logf("start test %s in non-microservice environment", getTestName()) + if _, ok := s.clusters[NonMicroserviceEnv]; !ok { + s.startCluster(NonMicroserviceEnv) } - test(s.clusters[PDMode]) + test(s.clusters[NonMicroserviceEnv]) } func getTestName() string { pc, _, _, _ := runtime.Caller(2) caller := runtime.FuncForPC(pc) - if caller == nil || strings.Contains(caller.Name(), "RunTestBasedOnMode") { + if caller == nil || strings.Contains(caller.Name(), "RunTest") { pc, _, _, _ = runtime.Caller(3) caller = runtime.FuncForPC(pc) } @@ -339,8 +339,8 @@ func getTestName() string { return "" } -// RunTestInPDServiceMode is to run test in pd service mode. -func (s *SchedulingTestEnvironment) RunTestInPDServiceMode(test func(*TestCluster)) { +// RunTestInMicroserviceMode is to run test in microservice environment. +func (s *SchedulingTestEnvironment) RunTestInMicroserviceEnv(test func(*TestCluster)) { re := require.New(s.t) re.NoError(failpoint.Enable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember", `return(true)`)) @@ -348,11 +348,11 @@ func (s *SchedulingTestEnvironment) RunTestInPDServiceMode(test func(*TestCluste re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/mcs/scheduling/server/fastUpdateMember")) re.NoError(failpoint.Disable("github.com/tikv/pd/server/cluster/highFrequencyClusterJobs")) }() - s.t.Logf("start test %s in pd service mode", getTestName()) - if _, ok := s.clusters[PDServiceMode]; !ok { - s.startCluster(PDServiceMode) + s.t.Logf("start test %s in microservice environment", getTestName()) + if _, ok := s.clusters[MicroserviceEnv]; !ok { + s.startCluster(MicroserviceEnv) } - test(s.clusters[PDServiceMode]) + test(s.clusters[MicroserviceEnv]) } // Cleanup is to cleanup the environment. @@ -365,12 +365,12 @@ func (s *SchedulingTestEnvironment) Cleanup() { } } -func (s *SchedulingTestEnvironment) startCluster(m SchedulerMode) { +func (s *SchedulingTestEnvironment) startCluster(m Env) { re := require.New(s.t) ctx, cancel := context.WithCancel(context.Background()) s.cancels = append(s.cancels, cancel) switch m { - case PDMode: + case NonMicroserviceEnv: cluster, err := NewTestCluster(ctx, 1, s.opts...) re.NoError(err) err = cluster.RunInitialServers() @@ -378,9 +378,9 @@ func (s *SchedulingTestEnvironment) startCluster(m SchedulerMode) { re.NotEmpty(cluster.WaitLeader()) leaderServer := cluster.GetServer(cluster.GetLeader()) re.NoError(leaderServer.BootstrapCluster()) - s.clusters[PDMode] = cluster - case PDServiceMode: - cluster, err := NewTestPDServiceCluster(ctx, 1, s.opts...) + s.clusters[NonMicroserviceEnv] = cluster + case MicroserviceEnv: + cluster, err := NewTestClusterWithKeyspaceGroup(ctx, 1, s.opts...) re.NoError(err) err = cluster.RunInitialServers() re.NoError(err) @@ -398,7 +398,7 @@ func (s *SchedulingTestEnvironment) startCluster(m SchedulerMode) { testutil.Eventually(re, func() bool { return cluster.GetLeaderServer().GetServer().IsServiceIndependent(constant.SchedulingServiceName) }) - s.clusters[PDServiceMode] = cluster + s.clusters[MicroserviceEnv] = cluster } } diff --git a/tools/pd-ctl/pdctl/command/config_command.go b/tools/pd-ctl/pdctl/command/config_command.go index c49b0e3b9e5..64da096a2f0 100644 --- a/tools/pd-ctl/pdctl/command/config_command.go +++ b/tools/pd-ctl/pdctl/command/config_command.go @@ -49,8 +49,8 @@ const ( ruleBundlePrefix = "pd/api/v1/config/placement-rule" pdServerPrefix = "pd/api/v1/config/pd-server" serviceMiddlewareConfigPrefix = "pd/api/v1/service-middleware/config" - // flagFromPDService has no influence for pd mode, but it is useful for us to debug in pd service mode. - flagFromPDService = "from_pd_service" + // flagFromPD has no influence for pd mode, but it is useful for us to debug in pd service mode. + flagFromPD = "from_pd" ) // NewConfigCommand return a config subcommand of rootCmd @@ -81,7 +81,7 @@ func NewShowConfigCommand() *cobra.Command { sc.AddCommand(newShowReplicationModeCommand()) sc.AddCommand(NewShowServerConfigCommand()) sc.AddCommand(NewShowServiceMiddlewareConfigCommand()) - sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") + sc.Flags().Bool(flagFromPD, false, "read data from PD rather than microservice") return sc } @@ -92,7 +92,7 @@ func NewShowAllConfigCommand() *cobra.Command { Short: "show all config of PD", Run: showAllConfigCommandFunc, } - sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") + sc.Flags().Bool(flagFromPD, false, "read data from PD rather than microservice") return sc } @@ -103,7 +103,7 @@ func NewShowScheduleConfigCommand() *cobra.Command { Short: "show schedule config of PD", Run: showScheduleConfigCommandFunc, } - sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") + sc.Flags().Bool(flagFromPD, false, "read data from PD rather than microservice") return sc } @@ -114,7 +114,7 @@ func NewShowReplicationConfigCommand() *cobra.Command { Short: "show replication config of PD", Run: showReplicationConfigCommandFunc, } - sc.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") + sc.Flags().Bool(flagFromPD, false, "read data from PD rather than microservice") return sc } @@ -528,7 +528,7 @@ func NewPlacementRulesCommand() *cobra.Command { show.Flags().String("id", "", "rule id") show.Flags().String("region", "", "region id") show.Flags().Bool("detail", false, "detailed match info for region") - show.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") + show.Flags().Bool(flagFromPD, false, "read data from PD rather than microservice") load := &cobra.Command{ Use: "load", Short: "load placement rules to a file", @@ -538,7 +538,7 @@ func NewPlacementRulesCommand() *cobra.Command { load.Flags().String("id", "", "rule id") load.Flags().String("region", "", "region id") load.Flags().String("out", "rules.json", "the filename contains rules") - load.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") + load.Flags().Bool(flagFromPD, false, "read data from PD rather than microservice") save := &cobra.Command{ Use: "save", Short: "save rules from file", @@ -554,7 +554,7 @@ func NewPlacementRulesCommand() *cobra.Command { Short: "show rule group configuration(s)", Run: showRuleGroupFunc, } - ruleGroupShow.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") + ruleGroupShow.Flags().Bool(flagFromPD, false, "read data from PD rather than microservice") ruleGroupSet := &cobra.Command{ Use: "set ", Short: "update rule group configuration", @@ -577,7 +577,7 @@ func NewPlacementRulesCommand() *cobra.Command { Run: getRuleBundle, } ruleBundleGet.Flags().String("out", "", "the output file") - ruleBundleGet.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") + ruleBundleGet.Flags().Bool(flagFromPD, false, "read data from PD rather than microservice") ruleBundleSet := &cobra.Command{ Use: "set", Short: "set rule group config and its rules from file", @@ -596,7 +596,7 @@ func NewPlacementRulesCommand() *cobra.Command { Run: loadRuleBundle, } ruleBundleLoad.Flags().String("out", "rules.json", "the output file") - ruleBundleLoad.Flags().Bool(flagFromPDService, false, "read data from PD service rather than microservice") + ruleBundleLoad.Flags().Bool(flagFromPD, false, "read data from PD rather than microservice") ruleBundleSave := &cobra.Command{ Use: "save", Short: "save all group configs and rules from file", @@ -895,7 +895,7 @@ func saveRuleBundle(cmd *cobra.Command, _ []string) { func buildHeader(cmd *cobra.Command) http.Header { header := http.Header{} - forbiddenRedirectToMicroservice, err := cmd.Flags().GetBool(flagFromPDService) + forbiddenRedirectToMicroservice, err := cmd.Flags().GetBool(flagFromPD) if err == nil && forbiddenRedirectToMicroservice { header.Add(apiutil.XForbiddenForwardToMicroserviceHeader, "true") } diff --git a/tools/pd-ctl/tests/config/config_test.go b/tools/pd-ctl/tests/config/config_test.go index aeb3352eccd..ed96c06489a 100644 --- a/tools/pd-ctl/tests/config/config_test.go +++ b/tools/pd-ctl/tests/config/config_test.go @@ -99,14 +99,14 @@ func (suite *configTestSuite) TearDownTest() { err = testutil.CheckPostJSON(testDialClient, urlPrefix+"/pd/api/v1/config/placement-rule", data, testutil.StatusOK(re)) re.NoError(err) } - suite.env.RunTestBasedOnMode(cleanFunc) + suite.env.RunTest(cleanFunc) suite.env.Cleanup() } func (suite *configTestSuite) TestConfig() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop", `return(true)`)) - suite.env.RunTestBasedOnMode(suite.checkConfig) + suite.env.RunTest(suite.checkConfig) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop")) } @@ -368,7 +368,7 @@ func (suite *configTestSuite) checkConfig(cluster *pdTests.TestCluster) { func (suite *configTestSuite) TestConfigForwardControl() { re := suite.Require() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop", `return(true)`)) - suite.env.RunTestBasedOnMode(suite.checkConfigForwardControl) + suite.env.RunTest(suite.checkConfigForwardControl) re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/dashboard/adapter/skipDashboardLoop")) } @@ -448,7 +448,7 @@ func (suite *configTestSuite) checkConfigForwardControl(cluster *pdTests.TestClu args := []string{"-u", pdAddr, "config", "show"} args = append(args, options...) if isFromPDService { - args = append(args, "--from_pd_service") + args = append(args, "--from_pd") } output, err := tests.ExecuteCommand(cmd, args...) re.NoError(err) @@ -477,7 +477,7 @@ func (suite *configTestSuite) checkConfigForwardControl(cluster *pdTests.TestClu args := []string{"-u", pdAddr, "config", "placement-rules"} args = append(args, options...) if isFromPDService { - args = append(args, "--from_pd_service") + args = append(args, "--from_pd") } output, err := tests.ExecuteCommand(cmd, args...) re.NoError(err) @@ -521,13 +521,13 @@ func (suite *configTestSuite) checkConfigForwardControl(cluster *pdTests.TestClu re.Equal(uint64(233), sche.GetPersistConfig().GetLeaderScheduleLimit()) re.Equal(7, sche.GetPersistConfig().GetMaxReplicas()) } - // show config from PD service rather than scheduling server + // show config from PD rather than scheduling server testConfig() - // show all config from PD service rather than scheduling server + // show all config from PD rather than scheduling server testConfig("all") - // show replication config from PD service rather than scheduling server + // show replication config from PD rather than scheduling server testConfig("replication") - // show schedule config from PD service rather than scheduling server + // show schedule config from PD rather than scheduling server testConfig("schedule") // Test Rule @@ -571,7 +571,7 @@ func (suite *configTestSuite) checkConfigForwardControl(cluster *pdTests.TestClu } func (suite *configTestSuite) TestPlacementRules() { - suite.env.RunTestBasedOnMode(suite.checkPlacementRules) + suite.env.RunTest(suite.checkPlacementRules) } func (suite *configTestSuite) checkPlacementRules(cluster *pdTests.TestCluster) { @@ -637,7 +637,7 @@ func (suite *configTestSuite) checkPlacementRules(cluster *pdTests.TestCluster) } func (suite *configTestSuite) TestPlacementRuleGroups() { - suite.env.RunTestBasedOnMode(suite.checkPlacementRuleGroups) + suite.env.RunTest(suite.checkPlacementRuleGroups) } func (suite *configTestSuite) checkPlacementRuleGroups(cluster *pdTests.TestCluster) { @@ -714,7 +714,7 @@ func (suite *configTestSuite) checkPlacementRuleGroups(cluster *pdTests.TestClus } func (suite *configTestSuite) TestPlacementRuleBundle() { - suite.env.RunTestBasedOnMode(suite.checkPlacementRuleBundle) + suite.env.RunTest(suite.checkPlacementRuleBundle) } func (suite *configTestSuite) checkPlacementRuleBundle(cluster *pdTests.TestCluster) { @@ -1051,7 +1051,7 @@ func TestServiceMiddlewareConfig(t *testing.T) { } func (suite *configTestSuite) TestUpdateDefaultReplicaConfig() { - suite.env.RunTestBasedOnMode(suite.checkUpdateDefaultReplicaConfig) + suite.env.RunTest(suite.checkUpdateDefaultReplicaConfig) } func (suite *configTestSuite) checkUpdateDefaultReplicaConfig(cluster *pdTests.TestCluster) { @@ -1200,7 +1200,7 @@ func (suite *configTestSuite) checkUpdateDefaultReplicaConfig(cluster *pdTests.T } func (suite *configTestSuite) TestPDServerConfig() { - suite.env.RunTestBasedOnMode(suite.checkPDServerConfig) + suite.env.RunTest(suite.checkPDServerConfig) } func (suite *configTestSuite) checkPDServerConfig(cluster *pdTests.TestCluster) { @@ -1233,7 +1233,7 @@ func (suite *configTestSuite) checkPDServerConfig(cluster *pdTests.TestCluster) } func (suite *configTestSuite) TestMicroserviceConfig() { - suite.env.RunTestBasedOnMode(suite.checkMicroserviceConfig) + suite.env.RunTest(suite.checkMicroserviceConfig) } func (suite *configTestSuite) checkMicroserviceConfig(cluster *pdTests.TestCluster) { @@ -1263,7 +1263,7 @@ func (suite *configTestSuite) checkMicroserviceConfig(cluster *pdTests.TestClust } func (suite *configTestSuite) TestRegionRules() { - suite.env.RunTestBasedOnMode(suite.checkRegionRules) + suite.env.RunTest(suite.checkRegionRules) } func (suite *configTestSuite) checkRegionRules(cluster *pdTests.TestCluster) { diff --git a/tools/pd-ctl/tests/hot/hot_test.go b/tools/pd-ctl/tests/hot/hot_test.go index d01f861d861..9254c31dfba 100644 --- a/tools/pd-ctl/tests/hot/hot_test.go +++ b/tools/pd-ctl/tests/hot/hot_test.go @@ -73,11 +73,11 @@ func (suite *hotTestSuite) TearDownTest() { } hotStat.HotCache.CleanCache() } - suite.env.RunTestBasedOnMode(cleanFunc) + suite.env.RunTest(cleanFunc) } func (suite *hotTestSuite) TestHot() { - suite.env.RunTestBasedOnMode(suite.checkHot) + suite.env.RunTest(suite.checkHot) } func (suite *hotTestSuite) checkHot(cluster *pdTests.TestCluster) { @@ -247,7 +247,7 @@ func (suite *hotTestSuite) checkHot(cluster *pdTests.TestCluster) { } func (suite *hotTestSuite) TestHotWithStoreID() { - suite.env.RunTestBasedOnMode(suite.checkHotWithStoreID) + suite.env.RunTest(suite.checkHotWithStoreID) } func (suite *hotTestSuite) checkHotWithStoreID(cluster *pdTests.TestCluster) { @@ -314,7 +314,7 @@ func (suite *hotTestSuite) checkHotWithStoreID(cluster *pdTests.TestCluster) { } func (suite *hotTestSuite) TestHotWithoutHotPeer() { - suite.env.RunTestBasedOnMode(suite.checkHotWithoutHotPeer) + suite.env.RunTest(suite.checkHotWithoutHotPeer) } func (suite *hotTestSuite) checkHotWithoutHotPeer(cluster *pdTests.TestCluster) { diff --git a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go index fff95856931..88ee782ab4b 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go @@ -41,7 +41,7 @@ func TestKeyspaceGroup(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() - tc, err := pdTests.NewTestPDServiceCluster(ctx, 1) + tc, err := pdTests.NewTestClusterWithKeyspaceGroup(ctx, 1) re.NoError(err) defer tc.Destroy() err = tc.RunInitialServers() @@ -102,7 +102,7 @@ func TestSplitKeyspaceGroup(t *testing.T) { for i := range 129 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestPDServiceCluster(ctx, 3, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestClusterWithKeyspaceGroup(ctx, 3, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -157,7 +157,7 @@ func TestExternalAllocNodeWhenStart(t *testing.T) { for i := range 10 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -197,7 +197,7 @@ func TestSetNodeAndPriorityKeyspaceGroup(t *testing.T) { for i := range 10 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestPDServiceCluster(ctx, 3, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestClusterWithKeyspaceGroup(ctx, 3, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -301,7 +301,7 @@ func TestMergeKeyspaceGroup(t *testing.T) { for i := range 129 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -420,7 +420,7 @@ func TestKeyspaceGroupState(t *testing.T) { for i := range 10 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -511,7 +511,7 @@ func TestShowKeyspaceGroupPrimary(t *testing.T) { for i := range 10 { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestPDServiceCluster(ctx, 1, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) diff --git a/tools/pd-ctl/tests/keyspace/keyspace_test.go b/tools/pd-ctl/tests/keyspace/keyspace_test.go index 6a523ced7b8..e6c28de599f 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_test.go @@ -49,7 +49,7 @@ func TestKeyspace(t *testing.T) { for i := 1; i < 10; i++ { keyspaces = append(keyspaces, fmt.Sprintf("keyspace_%d", i)) } - tc, err := pdTests.NewTestPDServiceCluster(ctx, 3, func(conf *config.Config, _ string) { + tc, err := pdTests.NewTestClusterWithKeyspaceGroup(ctx, 3, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = keyspaces }) re.NoError(err) @@ -155,7 +155,7 @@ func (suite *keyspaceTestSuite) SetupTest() { suite.ctx, suite.cancel = context.WithCancel(context.Background()) re.NoError(failpoint.Enable("github.com/tikv/pd/server/delayStartServerLoop", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion", "return(true)")) - tc, err := pdTests.NewTestPDServiceCluster(suite.ctx, 1) + tc, err := pdTests.NewTestClusterWithKeyspaceGroup(suite.ctx, 1) re.NoError(err) re.NoError(tc.RunInitialServers()) tc.WaitLeader() diff --git a/tools/pd-ctl/tests/operator/operator_test.go b/tools/pd-ctl/tests/operator/operator_test.go index 9e8c374ea49..39b8b04b758 100644 --- a/tools/pd-ctl/tests/operator/operator_test.go +++ b/tools/pd-ctl/tests/operator/operator_test.go @@ -59,7 +59,7 @@ func (suite *operatorTestSuite) TearDownSuite() { } func (suite *operatorTestSuite) TestOperator() { - suite.env.RunTestBasedOnMode(suite.checkOperator) + suite.env.RunTest(suite.checkOperator) } func (suite *operatorTestSuite) checkOperator(cluster *pdTests.TestCluster) { diff --git a/tools/pd-ctl/tests/scheduler/scheduler_test.go b/tools/pd-ctl/tests/scheduler/scheduler_test.go index f3a81845921..9de62f2b1db 100644 --- a/tools/pd-ctl/tests/scheduler/scheduler_test.go +++ b/tools/pd-ctl/tests/scheduler/scheduler_test.go @@ -96,7 +96,7 @@ func (suite *schedulerTestSuite) TearDownTest() { } } } - suite.env.RunTestBasedOnMode(cleanFunc) + suite.env.RunTest(cleanFunc) suite.env.Cleanup() } @@ -109,7 +109,7 @@ func (suite *schedulerTestSuite) checkDefaultSchedulers(re *require.Assertions, } func (suite *schedulerTestSuite) TestScheduler() { - suite.env.RunTestBasedOnMode(suite.checkScheduler) + suite.env.RunTest(suite.checkScheduler) } func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { @@ -408,7 +408,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *pdTests.TestCluster) { } func (suite *schedulerTestSuite) TestSchedulerConfig() { - suite.env.RunTestBasedOnMode(suite.checkSchedulerConfig) + suite.env.RunTest(suite.checkSchedulerConfig) } func (suite *schedulerTestSuite) checkSchedulerConfig(cluster *pdTests.TestCluster) { @@ -569,7 +569,7 @@ func (suite *schedulerTestSuite) checkSchedulerConfig(cluster *pdTests.TestClust } func (suite *schedulerTestSuite) TestGrantHotRegionScheduler() { - suite.env.RunTestBasedOnMode(suite.checkGrantHotRegionScheduler) + suite.env.RunTest(suite.checkGrantHotRegionScheduler) } func (suite *schedulerTestSuite) checkGrantHotRegionScheduler(cluster *pdTests.TestCluster) { @@ -685,7 +685,7 @@ func (suite *schedulerTestSuite) checkGrantHotRegionScheduler(cluster *pdTests.T } func (suite *schedulerTestSuite) TestHotRegionSchedulerConfig() { - suite.env.RunTestBasedOnMode(suite.checkHotRegionSchedulerConfig) + suite.env.RunTest(suite.checkHotRegionSchedulerConfig) } func (suite *schedulerTestSuite) checkHotRegionSchedulerConfig(cluster *pdTests.TestCluster) { @@ -849,7 +849,7 @@ func (suite *schedulerTestSuite) checkHotRegionSchedulerConfig(cluster *pdTests. } func (suite *schedulerTestSuite) TestSchedulerDiagnostic() { - suite.env.RunTestBasedOnMode(suite.checkSchedulerDiagnostic) + suite.env.RunTest(suite.checkSchedulerDiagnostic) } func (suite *schedulerTestSuite) checkSchedulerDiagnostic(cluster *pdTests.TestCluster) { @@ -913,7 +913,7 @@ func (suite *schedulerTestSuite) checkSchedulerDiagnostic(cluster *pdTests.TestC } func (suite *schedulerTestSuite) TestEvictLeaderScheduler() { - suite.env.RunTestBasedOnMode(suite.checkEvictLeaderScheduler) + suite.env.RunTest(suite.checkEvictLeaderScheduler) } func (suite *schedulerTestSuite) checkEvictLeaderScheduler(cluster *pdTests.TestCluster) { From ad172c7aa62df27e848241f15ea3e1f37fcf8d48 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Mon, 13 Jan 2025 18:09:10 +0800 Subject: [PATCH 13/33] pd-ctl: print warn information when set max-replica to less than current (#8983) close tikv/pd#8959 Signed-off-by: lhy1024 --- server/api/config.go | 5 +++ tools/pd-ctl/pdctl/command/config_command.go | 21 +++++++++ tools/pd-ctl/tests/config/config_test.go | 45 ++++++++++++++++++++ 3 files changed, 71 insertions(+) diff --git a/server/api/config.go b/server/api/config.go index ff4fe0add82..9e737fbe9bb 100644 --- a/server/api/config.go +++ b/server/api/config.go @@ -28,6 +28,7 @@ import ( "github.com/pingcap/errcode" "github.com/pingcap/errors" + "github.com/pingcap/failpoint" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" @@ -411,6 +412,10 @@ func (h *confHandler) SetScheduleConfig(w http.ResponseWriter, r *http.Request) // @Success 200 {object} sc.ReplicationConfig // @Router /config/replicate [get] func (h *confHandler) GetReplicationConfig(w http.ResponseWriter, r *http.Request) { + failpoint.Inject("getReplicationConfigFailed", func(v failpoint.Value) { + code := v.(int) + h.rd.JSON(w, code, "get config failed") + }) if h.svr.IsServiceIndependent(constant.SchedulingServiceName) && r.Header.Get(apiutil.XForbiddenForwardToMicroserviceHeader) != "true" { cfg, err := h.getSchedulingServerConfig() diff --git a/tools/pd-ctl/pdctl/command/config_command.go b/tools/pd-ctl/pdctl/command/config_command.go index 64da096a2f0..f0912c07c53 100644 --- a/tools/pd-ctl/pdctl/command/config_command.go +++ b/tools/pd-ctl/pdctl/command/config_command.go @@ -372,6 +372,8 @@ func postConfigDataWithPath(cmd *cobra.Command, key, value, path string) error { val, err := strconv.ParseFloat(value, 64) if err != nil { val = value + } else if key == "max-replicas" { + checkMaxReplicas(cmd, val.(float64)) } data[key] = val reqData, err := json.Marshal(data) @@ -386,6 +388,25 @@ func postConfigDataWithPath(cmd *cobra.Command, key, value, path string) error { return nil } +func checkMaxReplicas(cmd *cobra.Command, newReplica float64) { + header := buildHeader(cmd) + r, err := doRequest(cmd, replicatePrefix, http.MethodGet, header) + if err != nil { + cmd.Printf("Failed to get config when checking config: %s\n", err) + return + } + oldConfig := make(map[string]any) + err = json.Unmarshal([]byte(r), &oldConfig) + if err != nil { + cmd.Printf("Failed to unmarshal config when checking config: %s\n", err) + return + } + oldReplica, ok := oldConfig["max-replicas"].(float64) + if ok && newReplica < oldReplica { + cmd.Printf("Setting max-replica to %v which is less than the current replicas (%v). This may pose a risk. Please confirm the setting.\n", newReplica, oldReplica) + } +} + func setConfigCommandFunc(cmd *cobra.Command, args []string) { if len(args) != 2 { cmd.Println(cmd.UsageString()) diff --git a/tools/pd-ctl/tests/config/config_test.go b/tools/pd-ctl/tests/config/config_test.go index ed96c06489a..ca6b5e02c66 100644 --- a/tools/pd-ctl/tests/config/config_test.go +++ b/tools/pd-ctl/tests/config/config_test.go @@ -1199,6 +1199,51 @@ func (suite *configTestSuite) checkUpdateDefaultReplicaConfig(cluster *pdTests.T checkRuleIsolationLevel("host") } +func (suite *configTestSuite) TestMaxReplicaChanged() { + suite.env.RunTest(suite.checkMaxReplicaChanged) +} + +func (suite *configTestSuite) checkMaxReplicaChanged(cluster *pdTests.TestCluster) { + re := suite.Require() + leaderServer := cluster.GetLeaderServer() + pdAddr := leaderServer.GetAddr() + cmd := ctl.GetRootCmd() + + store := &metapb.Store{ + Id: 1, + State: metapb.StoreState_Up, + } + pdTests.MustPutStore(re, cluster, store) + + // test set max-replicas with invalid value + output, err := tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "set", "max-replicas", "z") + re.NoError(err) + re.NotContains(string(output), "Success!") + // test set max-replicas with less value + output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "set", "max-replicas", "2") + re.NoError(err) + re.Contains(string(output), "Success!") + re.Contains(string(output), "which is less than the current replicas") + // test set max-replicas with greater value + output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "set", "max-replicas", "3") + re.NoError(err) + re.Contains(string(output), "Success!") + re.NotContains(string(output), "which is less than the current replicas") + // test meet error when get config failed + failpoint.Enable("github.com/tikv/pd/server/api/getReplicationConfigFailed", `return(200)`) + output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "set", "max-replicas", "3") + re.NoError(err) + re.Contains(string(output), "Success!") + re.Contains(string(output), "Failed to unmarshal config when checking config") + failpoint.Disable("github.com/tikv/pd/server/api/getReplicationConfigFailed") + failpoint.Enable("github.com/tikv/pd/server/api/getReplicationConfigFailed", `return(500)`) + output, err = tests.ExecuteCommand(cmd, "-u", pdAddr, "config", "set", "max-replicas", "3") + re.NoError(err) + re.Contains(string(output), "Success!") + re.Contains(string(output), "Failed to get config when checking config") + failpoint.Disable("github.com/tikv/pd/server/api/getReplicationConfigFailed") +} + func (suite *configTestSuite) TestPDServerConfig() { suite.env.RunTest(suite.checkPDServerConfig) } From dd8df1ad147d64dc4ccb2618138ebbb2ad2a8b85 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 15 Jan 2025 12:01:34 +0800 Subject: [PATCH 14/33] server: use rateLimitCheck to reduce code (#8995) ref tikv/pd#4373 Signed-off-by: Ryan Leung --- server/grpc_service.go | 560 ++++++++++++++++++----------------------- 1 file changed, 248 insertions(+), 312 deletions(-) diff --git a/server/grpc_service.go b/server/grpc_service.go index 55639474ad0..649d02a37b4 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -44,6 +44,7 @@ import ( "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/errs" "github.com/tikv/pd/pkg/mcs/utils/constant" + "github.com/tikv/pd/pkg/ratelimit" "github.com/tikv/pd/pkg/storage/endpoint" "github.com/tikv/pd/pkg/storage/kv" "github.com/tikv/pd/pkg/utils/grpcutil" @@ -277,14 +278,12 @@ func (s *GrpcServer) GetClusterInfo(context.Context, *pdpb.GetClusterInfoRequest func (s *GrpcServer) GetMinTS( ctx context.Context, request *pdpb.GetMinTSRequest, ) (*pdpb.GetMinTSResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).GetMinTS(ctx, request) @@ -295,10 +294,7 @@ func (s *GrpcServer) GetMinTS( return rsp.(*pdpb.GetMinTSResponse), nil } - var ( - minTS *pdpb.Timestamp - err error - ) + var minTS *pdpb.Timestamp if s.IsServiceIndependent(constant.TSOServiceName) { minTS, err = s.GetMinTSFromTSOService() } else { @@ -434,14 +430,12 @@ func (s *GrpcServer) getMinTSFromSingleServer( // GetMembers implements gRPC PDServer. func (s *GrpcServer) GetMembers(context.Context, *pdpb.GetMembersRequest) (*pdpb.GetMembersResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } // Here we purposely do not check the cluster ID because the client does not know the correct cluster ID // at startup and needs to get the cluster ID with the first request (i.e. GetMembers). @@ -484,14 +478,12 @@ func (s *GrpcServer) GetMembers(context.Context, *pdpb.GetMembersRequest) (*pdpb // Tso implements gRPC PDServer. func (s *GrpcServer) Tso(stream pdpb.PD_TsoServer) error { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return err + } + if done != nil { + defer done() } if s.IsServiceIndependent(constant.TSOServiceName) { return s.forwardTSO(stream) @@ -602,14 +594,12 @@ func (s *GrpcServer) Tso(stream pdpb.PD_TsoServer) error { // Bootstrap implements gRPC PDServer. func (s *GrpcServer) Bootstrap(ctx context.Context, request *pdpb.BootstrapRequest) (*pdpb.BootstrapResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).Bootstrap(ctx, request) @@ -644,14 +634,12 @@ func (s *GrpcServer) Bootstrap(ctx context.Context, request *pdpb.BootstrapReque // IsBootstrapped implements gRPC PDServer. func (s *GrpcServer) IsBootstrapped(ctx context.Context, request *pdpb.IsBootstrappedRequest) (*pdpb.IsBootstrappedResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).IsBootstrapped(ctx, request) @@ -671,14 +659,12 @@ func (s *GrpcServer) IsBootstrapped(ctx context.Context, request *pdpb.IsBootstr // AllocID implements gRPC PDServer. func (s *GrpcServer) AllocID(ctx context.Context, request *pdpb.AllocIDRequest) (*pdpb.AllocIDResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).AllocID(ctx, request) @@ -705,15 +691,14 @@ func (s *GrpcServer) AllocID(ctx context.Context, request *pdpb.AllocIDRequest) // IsSnapshotRecovering implements gRPC PDServer. func (s *GrpcServer) IsSnapshotRecovering(ctx context.Context, _ *pdpb.IsSnapshotRecoveringRequest) (*pdpb.IsSnapshotRecoveringResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err } + if done != nil { + defer done() + } + if s.IsClosed() { return nil, errs.ErrNotStarted } @@ -732,14 +717,12 @@ func (s *GrpcServer) IsSnapshotRecovering(ctx context.Context, _ *pdpb.IsSnapsho // GetStore implements gRPC PDServer. func (s *GrpcServer) GetStore(ctx context.Context, request *pdpb.GetStoreRequest) (*pdpb.GetStoreResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).GetStore(ctx, request) @@ -786,14 +769,12 @@ func checkStore(rc *cluster.RaftCluster, storeID uint64) *pdpb.Error { // PutStore implements gRPC PDServer. func (s *GrpcServer) PutStore(ctx context.Context, request *pdpb.PutStoreRequest) (*pdpb.PutStoreResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).PutStore(ctx, request) @@ -841,14 +822,12 @@ func (s *GrpcServer) PutStore(ctx context.Context, request *pdpb.PutStoreRequest // GetAllStores implements gRPC PDServer. func (s *GrpcServer) GetAllStores(ctx context.Context, request *pdpb.GetAllStoresRequest) (*pdpb.GetAllStoresResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).GetAllStores(ctx, request) @@ -884,14 +863,12 @@ func (s *GrpcServer) GetAllStores(ctx context.Context, request *pdpb.GetAllStore // StoreHeartbeat implements gRPC PDServer. func (s *GrpcServer) StoreHeartbeat(ctx context.Context, request *pdpb.StoreHeartbeatRequest) (*pdpb.StoreHeartbeatResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).StoreHeartbeat(ctx, request) @@ -1063,14 +1040,12 @@ func (s *GrpcServer) ReportBuckets(stream pdpb.PD_ReportBucketsServer) error { cancel() } }() - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return err + } + if done != nil { + defer done() } for { request, err := server.recv() @@ -1179,14 +1154,12 @@ func (s *GrpcServer) RegionHeartbeat(stream pdpb.PD_RegionHeartbeatServer) error cancel() } }() - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return err + } + if done != nil { + defer done() } for { request, err := server.Recv() @@ -1387,14 +1360,12 @@ func (s *GrpcServer) GetRegion(ctx context.Context, request *pdpb.GetRegionReque failpoint.Inject("rateLimit", func() { failpoint.Return(nil, errs.ErrGRPCRateLimitExceeded(errs.ErrRateLimitExceeded)) }) - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).GetRegion(ctx, request) @@ -1449,14 +1420,12 @@ func (s *GrpcServer) GetRegion(ctx context.Context, request *pdpb.GetRegionReque // GetPrevRegion implements gRPC PDServer func (s *GrpcServer) GetPrevRegion(ctx context.Context, request *pdpb.GetRegionRequest) (*pdpb.GetRegionResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).GetPrevRegion(ctx, request) @@ -1506,14 +1475,12 @@ func (s *GrpcServer) GetPrevRegion(ctx context.Context, request *pdpb.GetRegionR // GetRegionByID implements gRPC PDServer. func (s *GrpcServer) GetRegionByID(ctx context.Context, request *pdpb.GetRegionByIDRequest) (*pdpb.GetRegionResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).GetRegionByID(ctx, request) @@ -1566,14 +1533,12 @@ func (s *GrpcServer) GetRegionByID(ctx context.Context, request *pdpb.GetRegionB // Deprecated: use BatchScanRegions instead. // ScanRegions implements gRPC PDServer. func (s *GrpcServer) ScanRegions(ctx context.Context, request *pdpb.ScanRegionsRequest) (*pdpb.ScanRegionsResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).ScanRegions(ctx, request) @@ -1622,14 +1587,12 @@ func (s *GrpcServer) ScanRegions(ctx context.Context, request *pdpb.ScanRegionsR // BatchScanRegions implements gRPC PDServer. func (s *GrpcServer) BatchScanRegions(ctx context.Context, request *pdpb.BatchScanRegionsRequest) (*pdpb.BatchScanRegionsResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).BatchScanRegions(ctx, request) @@ -1712,14 +1675,12 @@ func (s *GrpcServer) BatchScanRegions(ctx context.Context, request *pdpb.BatchSc // AskSplit implements gRPC PDServer. func (s *GrpcServer) AskSplit(ctx context.Context, request *pdpb.AskSplitRequest) (*pdpb.AskSplitResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).AskSplit(ctx, request) @@ -1756,14 +1717,12 @@ func (s *GrpcServer) AskSplit(ctx context.Context, request *pdpb.AskSplitRequest // AskBatchSplit implements gRPC PDServer. func (s *GrpcServer) AskBatchSplit(ctx context.Context, request *pdpb.AskBatchSplitRequest) (*pdpb.AskBatchSplitResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } rc := s.GetRaftCluster() @@ -1830,14 +1789,12 @@ func (s *GrpcServer) AskBatchSplit(ctx context.Context, request *pdpb.AskBatchSp // ReportSplit implements gRPC PDServer. func (s *GrpcServer) ReportSplit(ctx context.Context, request *pdpb.ReportSplitRequest) (*pdpb.ReportSplitResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).ReportSplit(ctx, request) @@ -1852,7 +1809,7 @@ func (s *GrpcServer) ReportSplit(ctx context.Context, request *pdpb.ReportSplitR if rc == nil { return &pdpb.ReportSplitResponse{Header: notBootstrappedHeader()}, nil } - _, err := rc.HandleReportSplit(request) + _, err = rc.HandleReportSplit(request) if err != nil { return &pdpb.ReportSplitResponse{ Header: wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, err.Error()), @@ -1866,14 +1823,12 @@ func (s *GrpcServer) ReportSplit(ctx context.Context, request *pdpb.ReportSplitR // ReportBatchSplit implements gRPC PDServer. func (s *GrpcServer) ReportBatchSplit(ctx context.Context, request *pdpb.ReportBatchSplitRequest) (*pdpb.ReportBatchSplitResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).ReportBatchSplit(ctx, request) @@ -1888,7 +1843,7 @@ func (s *GrpcServer) ReportBatchSplit(ctx context.Context, request *pdpb.ReportB if rc == nil { return &pdpb.ReportBatchSplitResponse{Header: notBootstrappedHeader()}, nil } - _, err := rc.HandleBatchReportSplit(request) + _, err = rc.HandleBatchReportSplit(request) if err != nil { return &pdpb.ReportBatchSplitResponse{ Header: wrapErrorToHeader(pdpb.ErrorType_UNKNOWN, @@ -1903,14 +1858,12 @@ func (s *GrpcServer) ReportBatchSplit(ctx context.Context, request *pdpb.ReportB // GetClusterConfig implements gRPC PDServer. func (s *GrpcServer) GetClusterConfig(ctx context.Context, request *pdpb.GetClusterConfigRequest) (*pdpb.GetClusterConfigResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).GetClusterConfig(ctx, request) @@ -1933,14 +1886,12 @@ func (s *GrpcServer) GetClusterConfig(ctx context.Context, request *pdpb.GetClus // PutClusterConfig implements gRPC PDServer. func (s *GrpcServer) PutClusterConfig(ctx context.Context, request *pdpb.PutClusterConfigRequest) (*pdpb.PutClusterConfigResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).PutClusterConfig(ctx, request) @@ -1972,14 +1923,12 @@ func (s *GrpcServer) PutClusterConfig(ctx context.Context, request *pdpb.PutClus // ScatterRegion implements gRPC PDServer. func (s *GrpcServer) ScatterRegion(ctx context.Context, request *pdpb.ScatterRegionRequest) (*pdpb.ScatterRegionResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } rc := s.GetRaftCluster() @@ -2084,14 +2033,12 @@ func (s *GrpcServer) ScatterRegion(ctx context.Context, request *pdpb.ScatterReg // GetGCSafePoint implements gRPC PDServer. func (s *GrpcServer) GetGCSafePoint(ctx context.Context, request *pdpb.GetGCSafePointRequest) (*pdpb.GetGCSafePointResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).GetGCSafePoint(ctx, request) @@ -2123,14 +2070,12 @@ func (s *GrpcServer) SyncRegions(stream pdpb.PD_SyncRegionsServer) error { if s.IsClosed() || s.cluster == nil { return errs.ErrNotStarted } - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return err + } + if done != nil { + defer done() } ctx := s.cluster.Context() if ctx == nil { @@ -2141,14 +2086,12 @@ func (s *GrpcServer) SyncRegions(stream pdpb.PD_SyncRegionsServer) error { // UpdateGCSafePoint implements gRPC PDServer. func (s *GrpcServer) UpdateGCSafePoint(ctx context.Context, request *pdpb.UpdateGCSafePointRequest) (*pdpb.UpdateGCSafePointResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).UpdateGCSafePoint(ctx, request) @@ -2188,14 +2131,12 @@ func (s *GrpcServer) UpdateGCSafePoint(ctx context.Context, request *pdpb.Update // UpdateServiceGCSafePoint update the safepoint for specific service func (s *GrpcServer) UpdateServiceGCSafePoint(ctx context.Context, request *pdpb.UpdateServiceGCSafePointRequest) (*pdpb.UpdateServiceGCSafePointResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).UpdateServiceGCSafePoint(ctx, request) @@ -2242,14 +2183,12 @@ func (s *GrpcServer) UpdateServiceGCSafePoint(ctx context.Context, request *pdpb // GetOperator gets information about the operator belonging to the specify region. func (s *GrpcServer) GetOperator(ctx context.Context, request *pdpb.GetOperatorRequest) (*pdpb.GetOperatorResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } rc := s.GetRaftCluster() @@ -2454,14 +2393,12 @@ func (*GrpcServer) SyncMaxTS(_ context.Context, _ *pdpb.SyncMaxTSRequest) (*pdpb // SplitRegions split regions by the given split keys func (s *GrpcServer) SplitRegions(ctx context.Context, request *pdpb.SplitRegionsRequest) (*pdpb.SplitRegionsResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } rc := s.GetRaftCluster() @@ -2518,14 +2455,12 @@ func (s *GrpcServer) SplitRegions(ctx context.Context, request *pdpb.SplitRegion // Only regions which split successfully will be scattered. // scatterFinishedPercentage indicates the percentage of successfully split regions that are scattered. func (s *GrpcServer) SplitAndScatterRegions(ctx context.Context, request *pdpb.SplitAndScatterRegionsRequest) (*pdpb.SplitAndScatterRegionsResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).SplitAndScatterRegions(ctx, request) @@ -2589,14 +2524,12 @@ func (s *GrpcServer) StoreGlobalConfig(_ context.Context, request *pdpb.StoreGlo if s.client == nil { return nil, errs.ErrEtcdNotStarted } - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } configPath := request.GetConfigPath() if configPath == "" { @@ -2635,14 +2568,12 @@ func (s *GrpcServer) LoadGlobalConfig(ctx context.Context, request *pdpb.LoadGlo if s.client == nil { return nil, errs.ErrEtcdNotStarted } - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } configPath := request.GetConfigPath() if configPath == "" { @@ -2683,14 +2614,12 @@ func (s *GrpcServer) WatchGlobalConfig(req *pdpb.WatchGlobalConfigRequest, serve if s.client == nil { return errs.ErrEtcdNotStarted } - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return err + } + if done != nil { + defer done() } ctx, cancel := context.WithCancel(server.Context()) defer cancel() @@ -2780,14 +2709,12 @@ func (s *GrpcServer) handleDamagedStore(stats *pdpb.StoreStats) { // ReportMinResolvedTS implements gRPC PDServer. func (s *GrpcServer) ReportMinResolvedTS(ctx context.Context, request *pdpb.ReportMinResolvedTsRequest) (*pdpb.ReportMinResolvedTsResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).ReportMinResolvedTS(ctx, request) @@ -2818,14 +2745,12 @@ func (s *GrpcServer) ReportMinResolvedTS(ctx context.Context, request *pdpb.Repo // SetExternalTimestamp implements gRPC PDServer. func (s *GrpcServer) SetExternalTimestamp(ctx context.Context, request *pdpb.SetExternalTimestampRequest) (*pdpb.SetExternalTimestampResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).SetExternalTimestamp(ctx, request) @@ -2854,14 +2779,12 @@ func (s *GrpcServer) SetExternalTimestamp(ctx context.Context, request *pdpb.Set // GetExternalTimestamp implements gRPC PDServer. func (s *GrpcServer) GetExternalTimestamp(ctx context.Context, request *pdpb.GetExternalTimestampRequest) (*pdpb.GetExternalTimestampResponse, error) { - if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { - fName := currentFunction() - limiter := s.GetGRPCRateLimiter() - if done, err := limiter.Allow(fName); err == nil { - defer done() - } else { - return nil, errs.ErrGRPCRateLimitExceeded(err) - } + done, err := s.rateLimitCheck() + if err != nil { + return nil, err + } + if done != nil { + defer done() } fn := func(ctx context.Context, client *grpc.ClientConn) (any, error) { return pdpb.NewPDClient(client).GetExternalTimestamp(ctx, request) @@ -2879,8 +2802,21 @@ func (s *GrpcServer) GetExternalTimestamp(ctx context.Context, request *pdpb.Get }, nil } -func currentFunction() string { - counter, _, _, _ := runtime.Caller(1) +func getCaller(skip int) string { + counter, _, _, _ := runtime.Caller(skip) s := strings.Split(runtime.FuncForPC(counter).Name(), ".") return s[len(s)-1] } + +func (s *GrpcServer) rateLimitCheck() (done ratelimit.DoneFunc, err error) { + if s.GetServiceMiddlewarePersistOptions().IsGRPCRateLimitEnabled() { + fName := getCaller(2) + limiter := s.GetGRPCRateLimiter() + if done, err = limiter.Allow(fName); err == nil { + return + } + err = errs.ErrGRPCRateLimitExceeded(err) + return + } + return +} From 2ae49cb546eee6c63ce093f4991363c3c4674377 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 16 Jan 2025 13:00:16 +0800 Subject: [PATCH 15/33] tests: make TestRemovingProgress stable (#8998) close tikv/pd#8992 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/server/api/api_test.go | 32 +++++++++++++++++++++++++++++--- 1 file changed, 29 insertions(+), 3 deletions(-) diff --git a/tests/server/api/api_test.go b/tests/server/api/api_test.go index 44a0ae69a46..859bf8cb56e 100644 --- a/tests/server/api/api_test.go +++ b/tests/server/api/api_test.go @@ -36,6 +36,7 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/response" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/pkg/utils/testutil" "github.com/tikv/pd/pkg/utils/typeutil" @@ -798,6 +799,24 @@ func TestRemovingProgress(t *testing.T) { output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores/progress?id=2", http.MethodGet, http.StatusNotFound) re.Contains(string(output), "no progress found for the given store ID") + // wait that stores are up + testutil.Eventually(re, func() bool { + output = sendRequest(re, leader.GetAddr()+"/pd/api/v1/stores", http.MethodGet, http.StatusOK) + var storesInfo response.StoresInfo + if err := json.Unmarshal(output, &storesInfo); err != nil { + return false + } + if len(storesInfo.Stores) != 3 { + return false + } + for _, store := range storesInfo.Stores { + if store.Store.GetNodeState() != metapb.NodeState_Serving { + return false + } + } + return true + }) + // remove store 1 and store 2 _ = sendRequest(re, leader.GetAddr()+"/pd/api/v1/store/1", http.MethodDelete, http.StatusOK) _ = sendRequest(re, leader.GetAddr()+"/pd/api/v1/store/2", http.MethodDelete, http.StatusOK) @@ -840,6 +859,9 @@ func TestRemovingProgress(t *testing.T) { testutil.Eventually(re, func() bool { // wait for cluster prepare + if leader.GetRaftCluster() == nil { + return false + } if !leader.GetRaftCluster().IsPrepared() { leader.GetRaftCluster().SetPrepared() return false @@ -866,7 +888,7 @@ func TestRemovingProgress(t *testing.T) { } // store 1: 40/10s = 4 // store 2: 20/10s = 2 - // average speed = (2+4)/2 = 33 + // average speed = (2+4)/2 = 3.0 if p.CurrentSpeed != 3.0 { return false } @@ -1097,7 +1119,9 @@ func sendRequest(re *require.Assertions, url string, method string, statusCode i testutil.Eventually(re, func() bool { resp, err := tests.TestDialClient.Do(req) - re.NoError(err) + if err != nil { + return false + } defer resp.Body.Close() // Due to service unavailability caused by environmental issues, @@ -1105,7 +1129,9 @@ func sendRequest(re *require.Assertions, url string, method string, statusCode i if resp.StatusCode == http.StatusServiceUnavailable { return false } - re.Equal(statusCode, resp.StatusCode) + if resp.StatusCode != statusCode { + return false + } output, err = io.ReadAll(resp.Body) re.NoError(err) return true From 79882a7a24726e040d7dfb98511b2517fa544d25 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Thu, 16 Jan 2025 14:33:49 +0800 Subject: [PATCH 16/33] *: clean up the testing environment (#9000) ref tikv/pd#7969 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- tests/integrations/client/client_test.go | 4 +--- tests/integrations/mcs/tso/api_test.go | 12 +++++----- .../mcs/tso/keyspace_group_manager_test.go | 4 ++-- tests/integrations/mcs/tso/proxy_test.go | 22 +++++++++---------- tests/integrations/tso/client_test.go | 10 ++++----- tests/server/cluster/cluster_test.go | 2 ++ .../region_syncer/region_syncer_test.go | 6 ++--- .../server/storage/hot_region_storage_test.go | 8 ++++--- tools/pd-backup/tests/backup_test.go | 2 +- tools/pd-ctl/tests/keyspace/keyspace_test.go | 7 ++---- tools/pd-ctl/tests/store/store_test.go | 2 +- .../tests/unsafe/unsafe_operation_test.go | 1 - 12 files changed, 38 insertions(+), 42 deletions(-) diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 6bd25567f81..0e0bf25d74d 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -229,10 +229,10 @@ func TestGetTSAfterTransferLeader(t *testing.T) { defer cancel() cluster, err := tests.NewTestCluster(ctx, 2) re.NoError(err) + defer cluster.Destroy() endpoints := runServer(re, cluster) leader := cluster.WaitLeader() re.NotEmpty(leader) - defer cluster.Destroy() cli := setupCli(ctx, re, endpoints, opt.WithCustomTimeoutOption(10*time.Second)) defer cli.Close() @@ -517,8 +517,6 @@ func (suite *followerForwardAndHandleTestSuite) SetupSuite() { }) } -func (*followerForwardAndHandleTestSuite) TearDownTest() {} - func (suite *followerForwardAndHandleTestSuite) TearDownSuite() { suite.cluster.Destroy() suite.clean() diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index aa153a7eb9f..7f874a52eb2 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -137,12 +137,12 @@ func TestTSOServerStartFirst(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - apiCluster, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { + cluster, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 1, func(conf *config.Config, _ string) { conf.Keyspace.PreAlloc = []string{"k1", "k2"} }) - defer apiCluster.Destroy() + defer cluster.Destroy() re.NoError(err) - addr := apiCluster.GetConfig().GetClientURL() + addr := cluster.GetConfig().GetClientURL() ch := make(chan struct{}) defer close(ch) clusterCh := make(chan *tests.TestTSOCluster) @@ -155,11 +155,11 @@ func TestTSOServerStartFirst(t *testing.T) { clusterCh <- tsoCluster ch <- struct{}{} }() - err = apiCluster.RunInitialServers() + err = cluster.RunInitialServers() re.NoError(err) - leaderName := apiCluster.WaitLeader() + leaderName := cluster.WaitLeader() re.NotEmpty(leaderName) - pdLeaderServer := apiCluster.GetServer(leaderName) + pdLeaderServer := cluster.GetServer(leaderName) re.NoError(pdLeaderServer.BootstrapCluster()) re.NoError(err) tsoCluster := <-clusterCh diff --git a/tests/integrations/mcs/tso/keyspace_group_manager_test.go b/tests/integrations/mcs/tso/keyspace_group_manager_test.go index 51ace0f08c4..9bd5073b5a5 100644 --- a/tests/integrations/mcs/tso/keyspace_group_manager_test.go +++ b/tests/integrations/mcs/tso/keyspace_group_manager_test.go @@ -544,12 +544,12 @@ func TestTwiceSplitKeyspaceGroup(t *testing.T) { } }) re.NoError(err) + defer tc.Destroy() pdAddr := tc.GetConfig().GetClientURL() // Start PD and tso server. err = tc.RunInitialServers() re.NoError(err) - defer tc.Destroy() tc.WaitLeader() leaderServer := tc.GetLeaderServer() re.NoError(leaderServer.BootstrapCluster()) @@ -741,12 +741,12 @@ func TestGetTSOImmediately(t *testing.T) { } }) re.NoError(err) + defer tc.Destroy() pdAddr := tc.GetConfig().GetClientURL() // Start PD and tso server. err = tc.RunInitialServers() re.NoError(err) - defer tc.Destroy() tc.WaitLeader() leaderServer := tc.GetLeaderServer() re.NoError(leaderServer.BootstrapCluster()) diff --git a/tests/integrations/mcs/tso/proxy_test.go b/tests/integrations/mcs/tso/proxy_test.go index 99213742e6c..a0c161c5a70 100644 --- a/tests/integrations/mcs/tso/proxy_test.go +++ b/tests/integrations/mcs/tso/proxy_test.go @@ -43,8 +43,8 @@ type tsoProxyTestSuite struct { suite.Suite ctx context.Context cancel context.CancelFunc - apiCluster *tests.TestCluster - apiLeader *tests.TestServer + cluster *tests.TestCluster + leader *tests.TestServer backendEndpoints string tsoCluster *tests.TestTSOCluster defaultReq *pdpb.TsoRequest @@ -62,15 +62,15 @@ func (s *tsoProxyTestSuite) SetupSuite() { var err error s.ctx, s.cancel = context.WithCancel(context.Background()) // Create an API cluster with 1 server - s.apiCluster, err = tests.NewTestClusterWithKeyspaceGroup(s.ctx, 1) + s.cluster, err = tests.NewTestClusterWithKeyspaceGroup(s.ctx, 1) re.NoError(err) - err = s.apiCluster.RunInitialServers() + err = s.cluster.RunInitialServers() re.NoError(err) - leaderName := s.apiCluster.WaitLeader() + leaderName := s.cluster.WaitLeader() re.NotEmpty(leaderName) - s.apiLeader = s.apiCluster.GetServer(leaderName) - s.backendEndpoints = s.apiLeader.GetAddr() - re.NoError(s.apiLeader.BootstrapCluster()) + s.leader = s.cluster.GetServer(leaderName) + s.backendEndpoints = s.leader.GetAddr() + re.NoError(s.leader.BootstrapCluster()) // Create a TSO cluster with 2 servers s.tsoCluster, err = tests.NewTestTSOCluster(s.ctx, 2, s.backendEndpoints) @@ -78,7 +78,7 @@ func (s *tsoProxyTestSuite) SetupSuite() { s.tsoCluster.WaitForDefaultPrimaryServing(re) s.defaultReq = &pdpb.TsoRequest{ - Header: &pdpb.RequestHeader{ClusterId: s.apiLeader.GetClusterID()}, + Header: &pdpb.RequestHeader{ClusterId: s.leader.GetClusterID()}, Count: 1, } @@ -89,7 +89,7 @@ func (s *tsoProxyTestSuite) SetupSuite() { func (s *tsoProxyTestSuite) TearDownSuite() { cleanupGRPCStreams(s.cleanupFuncs) s.tsoCluster.Destroy() - s.apiCluster.Destroy() + s.cluster.Destroy() s.cancel() } @@ -362,7 +362,7 @@ func (s *tsoProxyTestSuite) generateRequests(requestsPerClient int) []*pdpb.TsoR reqs := make([]*pdpb.TsoRequest, requestsPerClient) for i := range requestsPerClient { reqs[i] = &pdpb.TsoRequest{ - Header: &pdpb.RequestHeader{ClusterId: s.apiLeader.GetClusterID()}, + Header: &pdpb.RequestHeader{ClusterId: s.leader.GetClusterID()}, Count: uint32(i) + 1, // Make sure the count is positive. } } diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index 4822d6439ca..5cd74c705fc 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -498,9 +498,9 @@ func TestMixedTSODeployment(t *testing.T) { }() ctx, cancel := context.WithCancel(context.Background()) + defer cancel() cluster, err := tests.NewTestCluster(ctx, 1) re.NoError(err) - defer cancel() defer cluster.Destroy() err = cluster.RunInitialServers() @@ -542,10 +542,12 @@ func TestMixedTSODeployment(t *testing.T) { func TestUpgradingPDAndTSOClusters(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) + defer cancel() // Create an PD cluster which has 3 servers pdCluster, err := tests.NewTestClusterWithKeyspaceGroup(ctx, 3) re.NoError(err) + defer pdCluster.Destroy() err = pdCluster.RunInitialServers() re.NoError(err) leaderName := pdCluster.WaitLeader() @@ -569,7 +571,7 @@ func TestUpgradingPDAndTSOClusters(t *testing.T) { mcs.WaitForTSOServiceAvailable(ctx, re, pdClient) // Restart the API cluster - pdCluster, err = tests.RestartTestPDCluster(ctx, pdCluster) + _, err = tests.RestartTestPDCluster(ctx, pdCluster) re.NoError(err) // The TSO service should be eventually healthy mcs.WaitForTSOServiceAvailable(ctx, re, pdClient) @@ -577,12 +579,10 @@ func TestUpgradingPDAndTSOClusters(t *testing.T) { // Restart the TSO cluster tsoCluster, err = tests.RestartTestTSOCluster(ctx, tsoCluster) re.NoError(err) + defer tsoCluster.Destroy() // The TSO service should be eventually healthy mcs.WaitForTSOServiceAvailable(ctx, re, pdClient) - tsoCluster.Destroy() - pdCluster.Destroy() - cancel() re.NoError(failpoint.Disable("github.com/tikv/pd/client/servicediscovery/usePDServiceMode")) } diff --git a/tests/server/cluster/cluster_test.go b/tests/server/cluster/cluster_test.go index 357a76ace21..4ee4a9c8cf4 100644 --- a/tests/server/cluster/cluster_test.go +++ b/tests/server/cluster/cluster_test.go @@ -640,9 +640,11 @@ func TestRaftClusterStartTSOJob(t *testing.T) { name := "pd1" // case 1: normal start ctx, cancel := context.WithCancel(context.Background()) + defer cancel() tc, err := tests.NewTestCluster(ctx, 1, func(conf *config.Config, _ string) { conf.LeaderLease = 300 }) + defer tc.Destroy() re.NoError(err) re.NoError(tc.RunInitialServers()) re.NotEmpty(tc.WaitLeader()) diff --git a/tests/server/region_syncer/region_syncer_test.go b/tests/server/region_syncer/region_syncer_test.go index 7edcaa5ad9b..3a00d33fbaf 100644 --- a/tests/server/region_syncer/region_syncer_test.go +++ b/tests/server/region_syncer/region_syncer_test.go @@ -38,15 +38,13 @@ func TestMain(m *testing.M) { func TestRegionSyncer(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) + defer cancel() re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/storage/levelDBStorageFastFlush", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/syncer/noFastExitSync", `return(true)`)) re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/syncer/disableClientStreaming", `return(true)`)) cluster, err := tests.NewTestCluster(ctx, 3, func(conf *config.Config, _ string) { conf.PDServerCfg.UseRegionStorage = true }) - defer func() { - cluster.Destroy() - cancel() - }() + defer cluster.Destroy() re.NoError(err) re.NoError(cluster.RunInitialServers()) diff --git a/tests/server/storage/hot_region_storage_test.go b/tests/server/storage/hot_region_storage_test.go index 0643dda08d5..e34d6021f1d 100644 --- a/tests/server/storage/hot_region_storage_test.go +++ b/tests/server/storage/hot_region_storage_test.go @@ -46,6 +46,7 @@ func TestHotRegionStorage(t *testing.T) { }, ) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) re.NotEmpty(cluster.WaitLeader()) @@ -67,7 +68,7 @@ func TestHotRegionStorage(t *testing.T) { for _, store := range stores { tests.MustPutStore(re, cluster, store) } - defer cluster.Destroy() + startTime := time.Now().Unix() tests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b"), core.SetWrittenBytes(3000000000), core.SetReportInterval(uint64(startTime-utils.RegionHeartBeatReportInterval), uint64(startTime))) @@ -154,6 +155,7 @@ func TestHotRegionStorageReservedDayConfigChange(t *testing.T) { }, ) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) re.NotEmpty(cluster.WaitLeader()) @@ -175,7 +177,7 @@ func TestHotRegionStorageReservedDayConfigChange(t *testing.T) { for _, store := range stores { tests.MustPutStore(re, cluster, store) } - defer cluster.Destroy() + startTime := time.Now().Unix() tests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b"), core.SetWrittenBytes(3000000000), core.SetReportInterval(uint64(startTime-utils.RegionHeartBeatReportInterval), uint64(startTime))) @@ -246,6 +248,7 @@ func TestHotRegionStorageWriteIntervalConfigChange(t *testing.T) { }, ) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) re.NotEmpty(cluster.WaitLeader()) @@ -267,7 +270,6 @@ func TestHotRegionStorageWriteIntervalConfigChange(t *testing.T) { for _, store := range stores { tests.MustPutStore(re, cluster, store) } - defer cluster.Destroy() startTime := time.Now().Unix() tests.MustPutRegion(re, cluster, 1, 1, []byte("a"), []byte("b"), core.SetWrittenBytes(3000000000), diff --git a/tools/pd-backup/tests/backup_test.go b/tools/pd-backup/tests/backup_test.go index 7c8c03d96e0..2e1297fe034 100644 --- a/tools/pd-backup/tests/backup_test.go +++ b/tools/pd-backup/tests/backup_test.go @@ -35,6 +35,7 @@ func TestBackup(t *testing.T) { defer cancel() cluster, err := tests.NewTestCluster(ctx, 1) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) re.NotEmpty(cluster.WaitLeader()) @@ -42,7 +43,6 @@ func TestBackup(t *testing.T) { leaderServer.BootstrapCluster() pdAddr := cluster.GetConfig().GetClientURL() urls := strings.Split(pdAddr, ",") - defer cluster.Destroy() client, err := clientv3.New(clientv3.Config{ Endpoints: urls, DialTimeout: 3 * time.Second, diff --git a/tools/pd-ctl/tests/keyspace/keyspace_test.go b/tools/pd-ctl/tests/keyspace/keyspace_test.go index e6c28de599f..c17d496c6c5 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_test.go @@ -167,13 +167,10 @@ func (suite *keyspaceTestSuite) SetupTest() { func (suite *keyspaceTestSuite) TearDownTest() { re := suite.Require() - re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) - re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) -} - -func (suite *keyspaceTestSuite) TearDownSuite() { suite.cancel() suite.cluster.Destroy() + re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/keyspace/skipSplitRegion")) } func (suite *keyspaceTestSuite) TestShowKeyspace() { diff --git a/tools/pd-ctl/tests/store/store_test.go b/tools/pd-ctl/tests/store/store_test.go index 8ff36b79abe..414991c97ec 100644 --- a/tools/pd-ctl/tests/store/store_test.go +++ b/tools/pd-ctl/tests/store/store_test.go @@ -46,6 +46,7 @@ func TestStoreLimitV2(t *testing.T) { defer cancel() cluster, err := pdTests.NewTestCluster(ctx, 1) re.NoError(err) + defer cluster.Destroy() err = cluster.RunInitialServers() re.NoError(err) re.NotEmpty(cluster.WaitLeader()) @@ -54,7 +55,6 @@ func TestStoreLimitV2(t *testing.T) { leaderServer := cluster.GetLeaderServer() re.NoError(leaderServer.BootstrapCluster()) - defer cluster.Destroy() // store command args := []string{"-u", pdAddr, "config", "set", "store-limit-version", "v2"} diff --git a/tools/pd-ctl/tests/unsafe/unsafe_operation_test.go b/tools/pd-ctl/tests/unsafe/unsafe_operation_test.go index 652645e1570..86f11f43647 100644 --- a/tools/pd-ctl/tests/unsafe/unsafe_operation_test.go +++ b/tools/pd-ctl/tests/unsafe/unsafe_operation_test.go @@ -39,7 +39,6 @@ func TestRemoveFailedStores(t *testing.T) { re.NoError(err) pdAddr := cluster.GetConfig().GetClientURL() cmd := ctl.GetRootCmd() - defer cluster.Destroy() args := []string{"-u", pdAddr, "unsafe", "remove-failed-stores", "1,2,3"} _, err = tests.ExecuteCommand(cmd, args...) From 9805a9541aebbb6e1054b7505d101f861650d33a Mon Sep 17 00:00:00 2001 From: lucasliang Date: Fri, 17 Jan 2025 12:07:16 +0800 Subject: [PATCH 17/33] cluster: return NodeState in the response of StoreHeartbeat. (#8991) close tikv/pd#9001, ref tikv/tikv#18042, ref tikv/tikv#18107 In this PR, it returns the `NodeState` in `StoreHeartbeatResponse` to store nodes, to notify the corresponsive store to handle it if necessary. Signed-off-by: lucasliang Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- server/cluster/cluster.go | 4 ++++ server/cluster/cluster_test.go | 1 + tests/integrations/go.mod | 2 +- tests/integrations/go.sum | 4 ++-- tools/go.mod | 2 +- tools/go.sum | 4 ++-- 8 files changed, 14 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index c1107b6ffc5..d3e42f3f381 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 + github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 github.com/pingcap/tidb-dashboard v0.0.0-20241104061623-bce95733dad7 diff --git a/go.sum b/go.sum index e6156e9ecf7..97095710805 100644 --- a/go.sum +++ b/go.sum @@ -392,8 +392,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 h1:xYNSJjYNur4Dr5bV+9BXK9n5E0T1zlcAN25XX68+mOg= -github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 h1:RW/oeRwHxB9pGTtSCgf4wrsHw9RwrUg7+wAQRsW8KfE= +github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 8923efef9a6..f11c16032d8 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1043,6 +1043,10 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest if store := c.GetStore(storeID); store != nil { statistics.UpdateStoreHeartbeatMetrics(store) } + // Supply NodeState in the response to help the store handle special cases + // more conveniently, such as avoiding calling `remove_peer` redundantly under + // NodeState_Removing. + resp.State = store.GetNodeState() c.PutStore(newStore) var ( regions map[uint64]*core.RegionInfo diff --git a/server/cluster/cluster_test.go b/server/cluster/cluster_test.go index 2f6d04bbf52..d01939b9ef9 100644 --- a/server/cluster/cluster_test.go +++ b/server/cluster/cluster_test.go @@ -110,6 +110,7 @@ func TestStoreHeartbeat(t *testing.T) { re.NotEqual(int64(0), s.GetLastHeartbeatTS().UnixNano()) re.Equal(req.GetStats(), s.GetStoreStats()) re.Equal("v2", cluster.GetStore(1).GetStoreLimit().Version()) + re.Equal(s.GetMeta().GetNodeState(), resp.GetState()) storeMetasAfterHeartbeat = append(storeMetasAfterHeartbeat, s.GetMeta()) } diff --git a/tests/integrations/go.mod b/tests/integrations/go.mod index ec1d74923d6..432826c52bd 100644 --- a/tests/integrations/go.mod +++ b/tests/integrations/go.mod @@ -14,7 +14,7 @@ require ( github.com/go-sql-driver/mysql v1.7.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 + github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 diff --git a/tests/integrations/go.sum b/tests/integrations/go.sum index 5f78324c3c2..9900c796278 100644 --- a/tests/integrations/go.sum +++ b/tests/integrations/go.sum @@ -385,8 +385,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 h1:xYNSJjYNur4Dr5bV+9BXK9n5E0T1zlcAN25XX68+mOg= -github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 h1:RW/oeRwHxB9pGTtSCgf4wrsHw9RwrUg7+wAQRsW8KfE= +github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= diff --git a/tools/go.mod b/tools/go.mod index 31309986d92..2ccfa4abd60 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -22,7 +22,7 @@ require ( github.com/mattn/go-shellwords v1.0.12 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 + github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/common v0.55.0 diff --git a/tools/go.sum b/tools/go.sum index b9c49d466ef..93d685c193b 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -386,8 +386,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 h1:xYNSJjYNur4Dr5bV+9BXK9n5E0T1zlcAN25XX68+mOg= -github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 h1:RW/oeRwHxB9pGTtSCgf4wrsHw9RwrUg7+wAQRsW8KfE= +github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= From a99abe3cf2326c2a8e0617594b9f4f4615b608d4 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 17 Jan 2025 15:29:22 +0800 Subject: [PATCH 18/33] cluster: remove unnecessary get store (#9002) ref tikv/pd#9003 Signed-off-by: Ryan Leung --- server/cluster/cluster.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index f11c16032d8..6a0416acd67 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1040,9 +1040,7 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest newStore = newStore.Clone(core.SetLastPersistTime(nowTime)) } } - if store := c.GetStore(storeID); store != nil { - statistics.UpdateStoreHeartbeatMetrics(store) - } + statistics.UpdateStoreHeartbeatMetrics(store) // Supply NodeState in the response to help the store handle special cases // more conveniently, such as avoiding calling `remove_peer` redundantly under // NodeState_Removing. From 2902597746a0817e868b7049d5924b8a6440ca16 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 17 Jan 2025 16:26:55 +0800 Subject: [PATCH 19/33] metrics: remove the Heartbeat distribution (#9010) close tikv/pd#9003 Signed-off-by: lhy1024 --- metrics/grafana/pd.json | 472 ------------------ pkg/mcs/scheduling/server/cluster.go | 3 - .../buckets/bucket_stat_informer.go | 7 - pkg/statistics/buckets/hot_bucket_cache.go | 1 - pkg/statistics/buckets/metric.go | 10 - pkg/statistics/hot_peer_cache.go | 25 - pkg/statistics/metrics.go | 70 --- pkg/statistics/store.go | 5 - server/cluster/cluster.go | 1 - 9 files changed, 594 deletions(-) diff --git a/metrics/grafana/pd.json b/metrics/grafana/pd.json index 62b2e7234ef..7a83b5d5448 100644 --- a/metrics/grafana/pd.json +++ b/metrics/grafana/pd.json @@ -12940,478 +12940,6 @@ "title": "Heartbeat", "type": "row" }, - { - "collapsed": true, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 31 - }, - "id": 1400, - "panels": [ - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 24 - }, - "id": 1401, - "options": { - "displayMode": "lcd", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.5", - "targets": [ - { - "expr": "sum(rate(pd_scheduler_read_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Read Region Byte", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "custom": {}, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 24 - }, - "id": 1402, - "options": { - "displayMode": "lcd", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.5", - "targets": [ - { - "expr": "sum(rate(pd_scheduler_write_byte_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "B" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Write Region Byte", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "custom": { - "align": null - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 32 - }, - "id": 1403, - "options": { - "displayMode": "lcd", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.5", - "targets": [ - { - "expr": "sum(rate(pd_scheduler_read_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "C" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Read Region Key", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "custom": { - "align": null - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 32 - }, - "id": 1404, - "options": { - "displayMode": "lcd", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.5", - "targets": [ - { - "expr": "sum(rate(pd_scheduler_write_key_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "C" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Write Region Key", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "custom": { - "align": null - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 40 - }, - "id": 1405, - "interval": "", - "options": { - "displayMode": "lcd", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.5", - "repeatDirection": "h", - "targets": [ - { - "expr": "sum(rate(pd_scheduler_store_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Store Heartbeat Interval", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "custom": { - "align": null - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - } - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 40 - }, - "id": 1406, - "interval": "", - "options": { - "displayMode": "lcd", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.5", - "repeatDirection": "h", - "targets": [ - { - "expr": "sum(rate(pd_scheduler_region_heartbeat_interval_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=\"$tidb_cluster\"}[1m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Region Heartbeat Interval", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "custom": { - "align": null - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 48 - }, - "id": 1407, - "interval": "", - "options": { - "displayMode": "lcd", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.5", - "repeatDirection": "h", - "targets": [ - { - "expr": "sum(rate(pd_server_bucket_report_interval_seconds_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\"}[1m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Bucket Report Interval", - "transparent": true, - "type": "bargauge" - }, - { - "datasource": "${DS_TEST-CLUSTER}", - "fieldConfig": { - "defaults": { - "custom": { - "align": null - }, - "mappings": [] - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 48 - }, - "id": 1408, - "interval": "", - "options": { - "displayMode": "lcd", - "orientation": "horizontal", - "reduceOptions": { - "calcs": [ - "mean" - ], - "fields": "", - "values": false - }, - "showUnfilled": true - }, - "pluginVersion": "7.1.5", - "repeatDirection": "h", - "targets": [ - { - "expr": "sum(rate(pd_scheduler_buckets_hot_degree_hist_bucket{k8s_cluster=\"$k8s_cluster\", tidb_cluster=~\"$tidb_cluster.*\"}[1m])) by (le)", - "format": "heatmap", - "hide": false, - "interval": "", - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "timeFrom": null, - "timeShift": null, - "title": "Hot Degree of Bucket", - "transparent": true, - "type": "bargauge" - } - ], - "title": "Heartbeat distribution ", - "type": "row" - }, { "collapsed": true, "gridPos": { diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index 43e0179412e..c07174762ff 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -414,9 +414,6 @@ func (c *Cluster) HandleStoreHeartbeat(heartbeat *schedulingpb.StoreHeartbeatReq nowTime := time.Now() newStore := store.Clone(core.SetStoreStats(stats), core.SetLastHeartbeatTS(nowTime)) - if store := c.GetStore(storeID); store != nil { - statistics.UpdateStoreHeartbeatMetrics(store) - } c.PutStore(newStore) c.hotStat.Observe(storeID, newStore.GetStoreStats()) c.hotStat.FilterUnhealthyStore(c) diff --git a/pkg/statistics/buckets/bucket_stat_informer.go b/pkg/statistics/buckets/bucket_stat_informer.go index 888027f1f5f..032c85b78b0 100644 --- a/pkg/statistics/buckets/bucket_stat_informer.go +++ b/pkg/statistics/buckets/bucket_stat_informer.go @@ -208,10 +208,3 @@ func (b *BucketTreeItem) calculateHotDegree() { } } } - -// collectBucketsMetrics collects the metrics of the hot stats. -func (b *BucketTreeItem) collectBucketsMetrics() { - for _, bucket := range b.stats { - bucketsHotDegreeHist.Observe(float64(bucket.HotDegree)) - } -} diff --git a/pkg/statistics/buckets/hot_bucket_cache.go b/pkg/statistics/buckets/hot_bucket_cache.go index f29ef1563e8..88c7c064727 100644 --- a/pkg/statistics/buckets/hot_bucket_cache.go +++ b/pkg/statistics/buckets/hot_bucket_cache.go @@ -201,7 +201,6 @@ func (h *HotBucketCache) checkBucketsFlow(buckets *metapb.Buckets) (newItem *Buc } newItem.inherit(overlaps) newItem.calculateHotDegree() - newItem.collectBucketsMetrics() return newItem, overlaps } diff --git a/pkg/statistics/buckets/metric.go b/pkg/statistics/buckets/metric.go index ebc911b1410..cf8b492346f 100644 --- a/pkg/statistics/buckets/metric.go +++ b/pkg/statistics/buckets/metric.go @@ -19,15 +19,6 @@ import ( ) var ( - bucketsHotDegreeHist = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "buckets_hot_degree_hist", - Help: "Bucketed histogram of bucket hot degree", - Buckets: prometheus.LinearBuckets(-20, 2, 20), // [-20 20] - }) - bucketsTaskDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "pd", @@ -39,6 +30,5 @@ var ( ) func init() { - prometheus.MustRegister(bucketsHotDegreeHist) prometheus.MustRegister(bucketsTaskDuration) } diff --git a/pkg/statistics/hot_peer_cache.go b/pkg/statistics/hot_peer_cache.go index 77b5d567ca4..962290a8ba6 100644 --- a/pkg/statistics/hot_peer_cache.go +++ b/pkg/statistics/hot_peer_cache.go @@ -135,30 +135,6 @@ func (f *HotPeerCache) incMetrics(action utils.ActionType, storeID uint64) { f.metrics[storeID][action].Inc() } -func (f *HotPeerCache) collectPeerMetrics(loads []float64, interval uint64) { - regionHeartbeatIntervalHist.Observe(float64(interval)) - if interval == 0 { - return - } - // TODO: use unified metrics. (keep backward compatibility at the same time) - for _, k := range f.kind.RegionStats() { - switch k { - case utils.RegionReadBytes: - readByteHist.Observe(loads[int(k)]) - case utils.RegionReadKeys: - readKeyHist.Observe(loads[int(k)]) - case utils.RegionWriteBytes: - writeByteHist.Observe(loads[int(k)]) - case utils.RegionWriteKeys: - writeKeyHist.Observe(loads[int(k)]) - case utils.RegionWriteQueryNum: - writeQueryHist.Observe(loads[int(k)]) - case utils.RegionReadQueryNum: - readQueryHist.Observe(loads[int(k)]) - } - } -} - // CollectExpiredItems collects expired items, mark them as needDelete and puts them into inherit items func (f *HotPeerCache) CollectExpiredItems(region *core.RegionInfo) []*HotPeerStat { regionID := region.GetID() @@ -185,7 +161,6 @@ func (f *HotPeerCache) CheckPeerFlow(region *core.RegionInfo, peers []*metapb.Pe return nil } - f.collectPeerMetrics(deltaLoads, interval) // update metrics regionID := region.GetID() regionPeers := region.GetPeers() diff --git a/pkg/statistics/metrics.go b/pkg/statistics/metrics.go index 68cbf142479..64e040b8d66 100644 --- a/pkg/statistics/metrics.go +++ b/pkg/statistics/metrics.go @@ -81,70 +81,6 @@ var ( Name: "label_level", Help: "Number of regions in the different label level.", }, []string{"type"}) - readByteHist = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "read_byte_hist", - Help: "The distribution of region read bytes", - Buckets: prometheus.ExponentialBuckets(1, 8, 12), - }) - writeByteHist = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "write_byte_hist", - Help: "The distribution of region write bytes", - Buckets: prometheus.ExponentialBuckets(1, 8, 12), - }) - readKeyHist = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "read_key_hist", - Help: "The distribution of region read keys", - Buckets: prometheus.ExponentialBuckets(1, 2, 18), - }) - writeKeyHist = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "write_key_hist", - Help: "The distribution of region write keys", - Buckets: prometheus.ExponentialBuckets(1, 2, 18), - }) - readQueryHist = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "read_query_hist", - Help: "The distribution of region read query", - Buckets: prometheus.ExponentialBuckets(1, 2, 12), - }) - writeQueryHist = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "write_query_hist", - Help: "The distribution of region write query", - Buckets: prometheus.ExponentialBuckets(1, 2, 12), - }) - regionHeartbeatIntervalHist = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "region_heartbeat_interval_hist", - Help: "Bucketed histogram of the batch size of handled requests.", - Buckets: prometheus.LinearBuckets(0, 30, 20), - }) - storeHeartbeatIntervalHist = prometheus.NewHistogram( - prometheus.HistogramOpts{ - Namespace: "pd", - Subsystem: "scheduler", - Name: "store_heartbeat_interval_hist", - Help: "Bucketed histogram of the batch size of handled requests.", - Buckets: prometheus.LinearBuckets(0, 5, 12), - }) regionAbnormalPeerDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ @@ -187,12 +123,6 @@ func init() { prometheus.MustRegister(configStatusGauge) prometheus.MustRegister(StoreLimitGauge) prometheus.MustRegister(regionLabelLevelGauge) - prometheus.MustRegister(readByteHist) - prometheus.MustRegister(readKeyHist) - prometheus.MustRegister(writeKeyHist) - prometheus.MustRegister(writeByteHist) - prometheus.MustRegister(regionHeartbeatIntervalHist) - prometheus.MustRegister(storeHeartbeatIntervalHist) prometheus.MustRegister(regionAbnormalPeerDuration) prometheus.MustRegister(hotCacheFlowQueueStatusGauge) prometheus.MustRegister(hotPeerSummary) diff --git a/pkg/statistics/store.go b/pkg/statistics/store.go index fbbfea72700..39c638caaff 100644 --- a/pkg/statistics/store.go +++ b/pkg/statistics/store.go @@ -129,11 +129,6 @@ func (s *StoresStats) FilterUnhealthyStore(cluster core.StoreSetInformer) { } } -// UpdateStoreHeartbeatMetrics is used to update store heartbeat interval metrics -func UpdateStoreHeartbeatMetrics(store *core.StoreInfo) { - storeHeartbeatIntervalHist.Observe(time.Since(store.GetLastHeartbeatTS()).Seconds()) -} - // RollingStoreStats are multiple sets of recent historical records with specified windows size. type RollingStoreStats struct { syncutil.RWMutex diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 6a0416acd67..4a712e13698 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1040,7 +1040,6 @@ func (c *RaftCluster) HandleStoreHeartbeat(heartbeat *pdpb.StoreHeartbeatRequest newStore = newStore.Clone(core.SetLastPersistTime(nowTime)) } } - statistics.UpdateStoreHeartbeatMetrics(store) // Supply NodeState in the response to help the store handle special cases // more conveniently, such as avoiding calling `remove_peer` redundantly under // NodeState_Removing. From 604b0d6d4537d06f58a7f46aa9d722963354ce4c Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 17 Jan 2025 16:47:52 +0800 Subject: [PATCH 20/33] pkg: avoid make channel every time (#9009) close tikv/pd#9004 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/ratelimit/option.go | 36 +++++++++++++++++++---------- pkg/utils/tsoutil/tso_dispatcher.go | 7 +++++- 2 files changed, 30 insertions(+), 13 deletions(-) diff --git a/pkg/ratelimit/option.go b/pkg/ratelimit/option.go index e6d4a4f8ff0..65bb86a4d56 100644 --- a/pkg/ratelimit/option.go +++ b/pkg/ratelimit/option.go @@ -37,30 +37,36 @@ type Option func(string, *Controller) UpdateStatus // AddLabelAllowList adds a label into allow list. // It means the given label will not be limited func AddLabelAllowList() Option { - return func(label string, l *Controller) UpdateStatus { - l.labelAllowList[label] = struct{}{} + return func(label string, c *Controller) UpdateStatus { + c.labelAllowList[label] = struct{}{} return InAllowList } } // UpdateConcurrencyLimiter creates a concurrency limiter for a given label if it doesn't exist. func UpdateConcurrencyLimiter(limit uint64) Option { - return func(label string, l *Controller) UpdateStatus { - if _, allow := l.labelAllowList[label]; allow { + return func(label string, c *Controller) UpdateStatus { + if _, allow := c.labelAllowList[label]; allow { return InAllowList } - lim, _ := l.limiters.LoadOrStore(label, newLimiter()) + lim, loaded := c.limiters.Load(label) + if !loaded { + lim, _ = c.limiters.LoadOrStore(label, newLimiter()) + } return lim.(*limiter).updateConcurrencyConfig(limit) } } // UpdateQPSLimiter creates a QPS limiter for a given label if it doesn't exist. func UpdateQPSLimiter(limit float64, burst int) Option { - return func(label string, l *Controller) UpdateStatus { - if _, allow := l.labelAllowList[label]; allow { + return func(label string, c *Controller) UpdateStatus { + if _, allow := c.labelAllowList[label]; allow { return InAllowList } - lim, _ := l.limiters.LoadOrStore(label, newLimiter()) + lim, loaded := c.limiters.Load(label) + if !loaded { + lim, _ = c.limiters.LoadOrStore(label, newLimiter()) + } return lim.(*limiter).updateQPSConfig(limit, burst) } } @@ -71,18 +77,24 @@ func UpdateDimensionConfig(cfg *DimensionConfig) Option { if _, allow := c.labelAllowList[label]; allow { return InAllowList } - lim, _ := c.limiters.LoadOrStore(label, newLimiter()) + lim, loaded := c.limiters.Load(label) + if !loaded { + lim, _ = c.limiters.LoadOrStore(label, newLimiter()) + } return lim.(*limiter).updateDimensionConfig(cfg) } } // InitLimiter creates empty concurrency limiter for a given label by config if it doesn't exist. func InitLimiter() Option { - return func(label string, l *Controller) UpdateStatus { - if _, allow := l.labelAllowList[label]; allow { + return func(label string, c *Controller) UpdateStatus { + if _, allow := c.labelAllowList[label]; allow { return InAllowList } - l.limiters.LoadOrStore(label, newLimiter()) + _, loaded := c.limiters.Load(label) + if !loaded { + c.limiters.LoadOrStore(label, newLimiter()) + } return LimiterNotChanged } } diff --git a/pkg/utils/tsoutil/tso_dispatcher.go b/pkg/utils/tsoutil/tso_dispatcher.go index be7d4fa6d83..c563358adfe 100644 --- a/pkg/utils/tsoutil/tso_dispatcher.go +++ b/pkg/utils/tsoutil/tso_dispatcher.go @@ -69,7 +69,12 @@ func (s *TSODispatcher) DispatchRequest( doneCh <-chan struct{}, errCh chan<- error, tsoPrimaryWatchers ...*etcdutil.LoopWatcher) { - val, loaded := s.dispatchChs.LoadOrStore(req.getForwardedHost(), make(chan Request, maxMergeRequests)) + key := req.getForwardedHost() + val, loaded := s.dispatchChs.Load(key) + if !loaded { + val = make(chan Request, maxMergeRequests) + val, loaded = s.dispatchChs.LoadOrStore(key, val) + } reqCh := val.(chan Request) if !loaded { tsDeadlineCh := make(chan *TSDeadline, 1) From a69ee01287378a2072e56e2920727f596dd6850c Mon Sep 17 00:00:00 2001 From: JmPotato Date: Fri, 17 Jan 2025 22:10:01 +0800 Subject: [PATCH 21/33] server, core: implement the query region gRPC server (#8979) ref tikv/pd#8690 Implement the query region gRPC server interface. Signed-off-by: JmPotato --- go.mod | 2 +- go.sum | 4 +- pkg/core/region.go | 117 ++++++++++++++++++++++++++++++++++++++ pkg/core/region_test.go | 88 ++++++++++++++++++++++++++++ pkg/core/region_tree.go | 20 +++++++ server/grpc_service.go | 57 +++++++++++++++++++ server/metrics.go | 10 ++++ tests/integrations/go.mod | 2 +- tests/integrations/go.sum | 4 +- tools/go.mod | 2 +- tools/go.sum | 4 +- 11 files changed, 301 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index d3e42f3f381..8d0ad90fb11 100644 --- a/go.mod +++ b/go.mod @@ -34,7 +34,7 @@ require ( github.com/pingcap/errcode v0.3.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 + github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 github.com/pingcap/tidb-dashboard v0.0.0-20241104061623-bce95733dad7 diff --git a/go.sum b/go.sum index 97095710805..8850c1a6509 100644 --- a/go.sum +++ b/go.sum @@ -392,8 +392,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 h1:RW/oeRwHxB9pGTtSCgf4wrsHw9RwrUg7+wAQRsW8KfE= -github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1 h1:rTAyiswGyWSGHJVa4Mkhdi8YfGqfA4LrUVKsH9nrJ8E= +github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= diff --git a/pkg/core/region.go b/pkg/core/region.go index 706e6bbd712..94fc525f11b 100644 --- a/pkg/core/region.go +++ b/pkg/core/region.go @@ -47,6 +47,7 @@ import ( const ( randomRegionMaxRetry = 10 scanRegionLimit = 1000 + batchSearchSize = 16 // CollectFactor is the factor to collect the count of region. CollectFactor = 0.9 ) @@ -1464,6 +1465,122 @@ func (r *RegionsInfo) GetStoreRegions(storeID uint64) []*RegionInfo { return regions } +// TODO: benchmark the performance of `QueryRegions`. +// QueryRegions searches RegionInfo from regionTree by keys and IDs in batch. +func (r *RegionsInfo) QueryRegions( + keys, prevKeys [][]byte, ids []uint64, needBuckets bool, +) ([]uint64, []uint64, map[uint64]*pdpb.RegionResponse) { + // Iterate the region keys to find the regions. + regions := r.getRegionsByKeys(keys) + // Assert the returned regions count matches the input keys. + if len(regions) != len(keys) { + panic("returned regions count mismatch with the input keys") + } + // Iterate the prevKeys to find the regions. + prevRegions := r.getRegionsByPrevKeys(prevKeys) + // Assert the returned regions count matches the input keys. + if len(prevRegions) != len(prevKeys) { + panic("returned prev regions count mismatch with the input keys") + } + // Build the key -> ID map for the final results. + regionsByID := make(map[uint64]*pdpb.RegionResponse, len(regions)) + keyIDMap := sortOutKeyIDMap(regionsByID, regions, needBuckets) + prevKeyIDMap := sortOutKeyIDMap(regionsByID, prevRegions, needBuckets) + // Iterate the region IDs to find the regions. + for _, id := range ids { + // Check if the region has been found. + if regionFound, ok := regionsByID[id]; (ok && regionFound != nil) || id == 0 { + continue + } + // If the given region ID is not found in the region tree, set the region to nil. + if region := r.GetRegion(id); region == nil { + regionsByID[id] = nil + } else { + regionResp := &pdpb.RegionResponse{ + Region: region.GetMeta(), + Leader: region.GetLeader(), + DownPeers: region.GetDownPeers(), + PendingPeers: region.GetPendingPeers(), + } + if needBuckets { + regionResp.Buckets = region.GetBuckets() + } + regionsByID[id] = regionResp + } + } + return keyIDMap, prevKeyIDMap, regionsByID +} + +// getRegionsByKeys searches RegionInfo from regionTree by keys. +func (r *RegionsInfo) getRegionsByKeys(keys [][]byte) []*RegionInfo { + regions := make([]*RegionInfo, 0, len(keys)) + // Split the keys into multiple batches, and search each batch separately. + // This is to avoid the lock contention on the `regionTree`. + for _, batch := range splitKeysIntoBatches(keys) { + r.t.RLock() + results := r.tree.searchByKeys(batch) + r.t.RUnlock() + regions = append(regions, results...) + } + return regions +} + +func splitKeysIntoBatches(keys [][]byte) [][][]byte { + keysLen := len(keys) + batches := make([][][]byte, 0, (keysLen+batchSearchSize-1)/batchSearchSize) + for i := 0; i < keysLen; i += batchSearchSize { + end := i + batchSearchSize + if end > keysLen { + end = keysLen + } + batches = append(batches, keys[i:end]) + } + return batches +} + +func (r *RegionsInfo) getRegionsByPrevKeys(prevKeys [][]byte) []*RegionInfo { + regions := make([]*RegionInfo, 0, len(prevKeys)) + for _, batch := range splitKeysIntoBatches(prevKeys) { + r.t.RLock() + results := r.tree.searchByPrevKeys(batch) + r.t.RUnlock() + regions = append(regions, results...) + } + return regions +} + +// sortOutKeyIDMap will iterate the regions, convert it to a slice of regionID that corresponds to the input regions. +// It will also update `regionsByID` with the regionID and regionResponse. +func sortOutKeyIDMap( + regionsByID map[uint64]*pdpb.RegionResponse, regions []*RegionInfo, needBuckets bool, +) []uint64 { + keyIDMap := make([]uint64, len(regions)) + for idx, region := range regions { + regionID := region.GetMeta().GetId() + keyIDMap[idx] = regionID + // Check if the region has been found. + if regionFound, ok := regionsByID[regionID]; (ok && regionFound != nil) || regionID == 0 { + continue + } + // If the given key is not found in the region tree, set the region to nil. + if region == nil { + regionsByID[regionID] = nil + } else { + regionResp := &pdpb.RegionResponse{ + Region: region.GetMeta(), + Leader: region.GetLeader(), + DownPeers: region.GetDownPeers(), + PendingPeers: region.GetPendingPeers(), + } + if needBuckets { + regionResp.Buckets = region.GetBuckets() + } + regionsByID[regionID] = regionResp + } + } + return keyIDMap +} + // SubTreeRegionType is the type of sub tree region. type SubTreeRegionType string diff --git a/pkg/core/region_test.go b/pkg/core/region_test.go index 473421b0e52..cfd05b776f2 100644 --- a/pkg/core/region_test.go +++ b/pkg/core/region_test.go @@ -1203,3 +1203,91 @@ func TestScanRegion(t *testing.T) { re.Len(scanNoError([]byte("a"), []byte("e"), 0), 3) re.Len(scanNoError([]byte("c"), []byte("e"), 0), 1) } + +func TestQueryRegions(t *testing.T) { + re := require.New(t) + regions := NewRegionsInfo() + regions.CheckAndPutRegion(NewTestRegionInfo(1, 1, []byte("a"), []byte("b"))) + regions.CheckAndPutRegion(NewTestRegionInfo(2, 1, []byte("b"), []byte("c"))) + regions.CheckAndPutRegion(NewTestRegionInfo(3, 1, []byte("d"), []byte("e"))) + // Query regions by keys. + keyIDMap, prevKeyIDMap, regionsByID := regions.QueryRegions( + [][]byte{[]byte("a"), []byte("b"), []byte("c")}, + nil, + nil, + false, + ) + re.Len(keyIDMap, 3) + re.Empty(prevKeyIDMap) + re.Equal(uint64(1), keyIDMap[0]) + re.Equal(uint64(2), keyIDMap[1]) + re.Zero(keyIDMap[2]) // The key is not in the region tree, so its ID should be 0. + re.Len(regionsByID, 2) + re.Equal(uint64(1), regionsByID[1].GetRegion().GetId()) + re.Equal(uint64(2), regionsByID[2].GetRegion().GetId()) + // Query regions by IDs. + keyIDMap, prevKeyIDMap, regionsByID = regions.QueryRegions( + nil, + nil, + []uint64{1, 2, 3}, + false, + ) + re.Empty(keyIDMap) + re.Empty(prevKeyIDMap) + re.Len(regionsByID, 3) + re.Equal(uint64(1), regionsByID[1].GetRegion().GetId()) + re.Equal(uint64(2), regionsByID[2].GetRegion().GetId()) + re.Equal(uint64(3), regionsByID[3].GetRegion().GetId()) + // Query the region that does not exist. + keyIDMap, prevKeyIDMap, regionsByID = regions.QueryRegions( + nil, + nil, + []uint64{4}, + false, + ) + re.Empty(keyIDMap) + re.Empty(prevKeyIDMap) + re.Len(regionsByID, 1) + re.Nil(regionsByID[4]) + keyIDMap, prevKeyIDMap, regionsByID = regions.QueryRegions( + [][]byte{[]byte("c")}, + nil, + nil, + false, + ) + re.Len(keyIDMap, 1) + re.Empty(prevKeyIDMap) + re.Zero(keyIDMap[0]) + re.Empty(regionsByID) + keyIDMap, prevKeyIDMap, regionsByID = regions.QueryRegions( + [][]byte{[]byte("c")}, + nil, + []uint64{4}, + false, + ) + re.Len(keyIDMap, 1) + re.Empty(prevKeyIDMap) + re.Zero(keyIDMap[0]) + re.Nil(regionsByID[4]) + // Query regions by keys, previous keys and IDs. + keyIDMap, prevKeyIDMap, regionsByID = regions.QueryRegions( + [][]byte{[]byte("b"), []byte("c")}, + [][]byte{[]byte("a"), []byte("b"), []byte("c"), []byte("d"), []byte("e"), []byte("f")}, + []uint64{1, 3}, + false, + ) + re.Len(keyIDMap, 2) + re.Len(prevKeyIDMap, 6) + re.Equal(uint64(2), keyIDMap[0]) + re.Zero(keyIDMap[1]) + re.Zero(prevKeyIDMap[0]) + re.Equal(uint64(1), prevKeyIDMap[1]) + re.Zero(prevKeyIDMap[2]) + re.Zero(prevKeyIDMap[3]) + re.Zero(prevKeyIDMap[4]) + re.Zero(prevKeyIDMap[5]) + re.Len(regionsByID, 3) + re.Equal(uint64(1), regionsByID[1].GetRegion().GetId()) + re.Equal(uint64(2), regionsByID[2].GetRegion().GetId()) + re.Equal(uint64(3), regionsByID[3].GetRegion().GetId()) +} diff --git a/pkg/core/region_tree.go b/pkg/core/region_tree.go index 6efafd133cf..1e0a0d5565d 100644 --- a/pkg/core/region_tree.go +++ b/pkg/core/region_tree.go @@ -255,6 +255,26 @@ func (t *regionTree) searchPrev(regionKey []byte) *RegionInfo { return prevRegionItem.RegionInfo } +// searchByKeys searches the regions by keys and return a slice of `*RegionInfo` whose order is the same as the input keys. +func (t *regionTree) searchByKeys(keys [][]byte) []*RegionInfo { + regions := make([]*RegionInfo, len(keys)) + // TODO: do we need to deduplicate the input keys? + for idx, key := range keys { + regions[idx] = t.search(key) + } + return regions +} + +// searchByPrevKeys searches the regions by prevKeys and return a slice of `*RegionInfo` whose order is the same as the input keys. +func (t *regionTree) searchByPrevKeys(prevKeys [][]byte) []*RegionInfo { + regions := make([]*RegionInfo, len(prevKeys)) + // TODO: do we need to deduplicate the input keys? + for idx, key := range prevKeys { + regions[idx] = t.searchPrev(key) + } + return regions +} + // find returns the range item contains the start key. func (t *regionTree) find(item *regionItem) *regionItem { var result *regionItem diff --git a/server/grpc_service.go b/server/grpc_service.go index 649d02a37b4..b985e870a03 100644 --- a/server/grpc_service.go +++ b/server/grpc_service.go @@ -1530,6 +1530,63 @@ func (s *GrpcServer) GetRegionByID(ctx context.Context, request *pdpb.GetRegionB }, nil } +// QueryRegion provides a stream processing of the region query. +func (s *GrpcServer) QueryRegion(stream pdpb.PD_QueryRegionServer) error { + done, err := s.rateLimitCheck() + if err != nil { + return err + } + if done != nil { + defer done() + } + + for { + request, err := stream.Recv() + if err == io.EOF { + return nil + } + if err != nil { + return errors.WithStack(err) + } + + // TODO: add forwarding logic. + + if clusterID := keypath.ClusterID(); request.GetHeader().GetClusterId() != clusterID { + return errs.ErrMismatchClusterID(clusterID, request.GetHeader().GetClusterId()) + } + rc := s.GetRaftCluster() + if rc == nil { + resp := &pdpb.QueryRegionResponse{ + Header: notBootstrappedHeader(), + } + if err = stream.Send(resp); err != nil { + return errors.WithStack(err) + } + continue + } + needBuckets := rc.GetStoreConfig().IsEnableRegionBucket() && request.GetNeedBuckets() + + start := time.Now() + keyIDMap, prevKeyIDMap, regionsByID := rc.QueryRegions( + request.GetKeys(), + request.GetPrevKeys(), + request.GetIds(), + needBuckets, + ) + regionQueryDuration.Observe(time.Since(start).Seconds()) + // Build the response and send it to the client. + response := &pdpb.QueryRegionResponse{ + Header: wrapHeader(), + KeyIdMap: keyIDMap, + PrevKeyIdMap: prevKeyIDMap, + RegionsById: regionsByID, + } + if err := stream.Send(response); err != nil { + return errors.WithStack(err) + } + } +} + // Deprecated: use BatchScanRegions instead. // ScanRegions implements gRPC PDServer. func (s *GrpcServer) ScanRegions(ctx context.Context, request *pdpb.ScanRegionsRequest) (*pdpb.ScanRegionsResponse, error) { diff --git a/server/metrics.go b/server/metrics.go index fdcc5b4be22..dd07447140b 100644 --- a/server/metrics.go +++ b/server/metrics.go @@ -99,6 +99,15 @@ var ( Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13), }) + regionQueryDuration = prometheus.NewHistogram( + prometheus.HistogramOpts{ + Namespace: "pd", + Subsystem: "server", + Name: "region_query_duration_seconds", + Help: "Bucketed histogram of processing time (s) of region query requests.", + Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13), + }) + bucketReportLatency = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "pd", @@ -172,6 +181,7 @@ func init() { prometheus.MustRegister(tsoProxyBatchSize) prometheus.MustRegister(tsoProxyForwardTimeoutCounter) prometheus.MustRegister(tsoHandleDuration) + prometheus.MustRegister(regionQueryDuration) prometheus.MustRegister(regionHeartbeatHandleDuration) prometheus.MustRegister(storeHeartbeatHandleDuration) prometheus.MustRegister(bucketReportCounter) diff --git a/tests/integrations/go.mod b/tests/integrations/go.mod index 432826c52bd..fca5b54bb07 100644 --- a/tests/integrations/go.mod +++ b/tests/integrations/go.mod @@ -14,7 +14,7 @@ require ( github.com/go-sql-driver/mysql v1.7.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 + github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/client_model v0.6.1 diff --git a/tests/integrations/go.sum b/tests/integrations/go.sum index 9900c796278..c23ee2733a6 100644 --- a/tests/integrations/go.sum +++ b/tests/integrations/go.sum @@ -385,8 +385,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 h1:RW/oeRwHxB9pGTtSCgf4wrsHw9RwrUg7+wAQRsW8KfE= -github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1 h1:rTAyiswGyWSGHJVa4Mkhdi8YfGqfA4LrUVKsH9nrJ8E= +github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= diff --git a/tools/go.mod b/tools/go.mod index 2ccfa4abd60..eb4c32afc13 100644 --- a/tools/go.mod +++ b/tools/go.mod @@ -22,7 +22,7 @@ require ( github.com/mattn/go-shellwords v1.0.12 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 + github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 github.com/prometheus/common v0.55.0 diff --git a/tools/go.sum b/tools/go.sum index 93d685c193b..93cbc6736a9 100644 --- a/tools/go.sum +++ b/tools/go.sum @@ -386,8 +386,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ue github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w= -github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412 h1:RW/oeRwHxB9pGTtSCgf4wrsHw9RwrUg7+wAQRsW8KfE= -github.com/pingcap/kvproto v0.0.0-20250117013947-1fdf41372412/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1 h1:rTAyiswGyWSGHJVa4Mkhdi8YfGqfA4LrUVKsH9nrJ8E= +github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= From 093b84112e19a2ad54ca3282726147bc7198b8f2 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 21 Jan 2025 11:35:17 +0800 Subject: [PATCH 22/33] *: check file header (#9015) ref tikv/pd#4322 Signed-off-by: Ryan Leung --- .golangci.yml | 19 +++++++++++++ client/gc_client.go | 2 +- client/inner_client.go | 14 ++++++++++ client/pkg/caller/caller.go | 2 +- client/pkg/circuitbreaker/circuit_breaker.go | 3 +- .../circuitbreaker/circuit_breaker_test.go | 3 +- client/pkg/utils/testutil/check_env_dummy.go | 1 + client/pkg/utils/testutil/check_env_linux.go | 1 + client/pkg/utils/timerutil/pool.go | 14 ++++++++++ client/pkg/utils/timerutil/pool_test.go | 14 ++++++++++ .../controller/controller_test.go | 8 +++--- client/resource_group/controller/limiter.go | 8 +++--- .../resource_group/controller/limiter_test.go | 8 +++--- client/resource_group/controller/testutil.go | 8 +++--- client/resource_group/controller/util.go | 10 +++---- docs/swagger/placeholder.go | 14 ++++++++++ pkg/btree/btree_generic.go | 28 +++++++++---------- pkg/btree/btree_generic_test.go | 4 +-- pkg/errs/errno.go | 2 +- pkg/gc/safepoint_v2.go | 2 +- pkg/gctuner/tuner_test.go | 2 +- .../resourcemanager/server/resource_group.go | 2 +- .../server/resource_group_test.go | 14 ++++++++++ pkg/mcs/scheduling/server/cluster.go | 14 ++++++++++ pkg/mcs/server/server.go | 2 +- pkg/mcs/utils/constant/constant.go | 2 +- pkg/mcs/utils/expected_primary.go | 2 +- pkg/mcs/utils/util.go | 2 +- pkg/schedule/filter/filters_test.go | 3 +- .../schedulers/balance_benchmark_test.go | 3 +- pkg/storage/hot_region_storage.go | 2 +- pkg/storage/hot_region_storage_test.go | 1 + pkg/storage/leveldb_backend_test.go | 2 +- pkg/storage/region_storage.go | 2 +- pkg/storage/region_storage_test.go | 2 +- .../unsafe_recovery_controller.go | 2 +- .../unsafe_recovery_controller_test.go | 2 +- pkg/utils/grpcutil/grpcutil_test.go | 14 ++++++++++ pkg/utils/keypath/cluster_id.go | 2 +- pkg/utils/keypath/key_path_test.go | 2 +- pkg/utils/netutil/address_test.go | 3 +- pkg/utils/tempurl/check_env_dummy.go | 1 + pkg/utils/tempurl/check_env_linux.go | 1 + pkg/utils/timerutil/pool.go | 14 ++++++++++ pkg/utils/timerutil/pool_test.go | 14 ++++++++++ pkg/window/counter.go | 6 ++-- pkg/window/counter_test.go | 6 ++-- pkg/window/policy.go | 6 ++-- pkg/window/policy_test.go | 6 ++-- pkg/window/reduce.go | 6 ++-- pkg/window/window.go | 6 ++-- pkg/window/window_test.go | 6 ++-- server/api/pprof_test.go | 3 +- .../mcs/discovery/register_test.go | 2 +- tests/integrations/mcs/scheduling/api_test.go | 14 ++++++++++ .../mcs/scheduling/server_test.go | 2 +- tests/integrations/mcs/tso/server_test.go | 2 +- tests/integrations/realcluster/cluster.go | 2 +- .../realcluster/cluster_id_test.go | 2 +- .../integrations/realcluster/etcd_key_test.go | 2 +- tests/integrations/realcluster/mock_db.go | 2 +- .../realcluster/reboot_pd_test.go | 2 +- .../realcluster/scheduler_test.go | 2 +- tests/integrations/realcluster/ts_test.go | 2 +- tests/integrations/realcluster/util.go | 2 +- tools/pd-backup/pdbackup/backup_test.go | 14 ++++++++++ tools/pd-ctl/pdctl/command/global_test.go | 3 +- tools/pd-heartbeat-bench/config/config.go | 14 ++++++++++ tools/pd-simulator/simulator/metrics.go | 14 ++++++++++ tools/pd-ut/alloc/check_env_dummy.go | 1 + tools/pd-ut/alloc/check_env_linux.go | 1 + 71 files changed, 309 insertions(+), 94 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index a44052b22e8..561c785b078 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -29,6 +29,7 @@ linters: - reassign - intrange - gci + - goheader linters-settings: gocritic: # Which checks should be disabled; can't be combined with 'enabled-checks'; default is empty @@ -241,6 +242,24 @@ linters-settings: - prefix(github.com/pingcap) - prefix(github.com/tikv/pd) - blank + goheader: + values: + regexp: + COPYRIGHT-HEADER: Copyright \d{4} TiKV Project Authors. + template: |- + {{ COPYRIGHT-HEADER }} + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. issues: exclude-rules: - path: (_test\.go|pkg/mock/.*\.go|tests/.*\.go) diff --git a/client/gc_client.go b/client/gc_client.go index 45fc8b40b91..117013ab2fa 100644 --- a/client/gc_client.go +++ b/client/gc_client.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/client/inner_client.go b/client/inner_client.go index 8379b6a51a9..cff5b95ed54 100644 --- a/client/inner_client.go +++ b/client/inner_client.go @@ -1,3 +1,17 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pd import ( diff --git a/client/pkg/caller/caller.go b/client/pkg/caller/caller.go index 3df2297b0ac..04bd221216b 100644 --- a/client/pkg/caller/caller.go +++ b/client/pkg/caller/caller.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/client/pkg/circuitbreaker/circuit_breaker.go b/client/pkg/circuitbreaker/circuit_breaker.go index 0acee5d5c8d..7466124f2bc 100644 --- a/client/pkg/circuitbreaker/circuit_breaker.go +++ b/client/pkg/circuitbreaker/circuit_breaker.go @@ -4,13 +4,14 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package circuitbreaker import ( diff --git a/client/pkg/circuitbreaker/circuit_breaker_test.go b/client/pkg/circuitbreaker/circuit_breaker_test.go index e62e55c1ab8..ff7da9508c2 100644 --- a/client/pkg/circuitbreaker/circuit_breaker_test.go +++ b/client/pkg/circuitbreaker/circuit_breaker_test.go @@ -4,13 +4,14 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package circuitbreaker import ( diff --git a/client/pkg/utils/testutil/check_env_dummy.go b/client/pkg/utils/testutil/check_env_dummy.go index c8f4d268c9d..fed0636a112 100644 --- a/client/pkg/utils/testutil/check_env_dummy.go +++ b/client/pkg/utils/testutil/check_env_dummy.go @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + //go:build !linux // +build !linux diff --git a/client/pkg/utils/testutil/check_env_linux.go b/client/pkg/utils/testutil/check_env_linux.go index ebe6a8e0663..0c566f0dcd6 100644 --- a/client/pkg/utils/testutil/check_env_linux.go +++ b/client/pkg/utils/testutil/check_env_linux.go @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + //go:build linux // +build linux diff --git a/client/pkg/utils/timerutil/pool.go b/client/pkg/utils/timerutil/pool.go index cee061201c2..32f76b39572 100644 --- a/client/pkg/utils/timerutil/pool.go +++ b/client/pkg/utils/timerutil/pool.go @@ -1,3 +1,17 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Copyright 2020 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/client/pkg/utils/timerutil/pool_test.go b/client/pkg/utils/timerutil/pool_test.go index 3e0575daea3..ad550c7b56a 100644 --- a/client/pkg/utils/timerutil/pool_test.go +++ b/client/pkg/utils/timerutil/pool_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Copyright 2020 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/client/resource_group/controller/controller_test.go b/client/resource_group/controller/controller_test.go index f0bdc62d6d3..02fd9e3d750 100644 --- a/client/resource_group/controller/controller_test.go +++ b/client/resource_group/controller/controller_test.go @@ -1,7 +1,3 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package controller import ( diff --git a/client/resource_group/controller/limiter.go b/client/resource_group/controller/limiter.go index c8843da00bd..1e1d2c4c475 100644 --- a/client/resource_group/controller/limiter.go +++ b/client/resource_group/controller/limiter.go @@ -1,7 +1,3 @@ -// Copyright 2015 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Copyright 2015 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package controller import ( diff --git a/client/resource_group/controller/limiter_test.go b/client/resource_group/controller/limiter_test.go index 24cdee0bbc3..8c524defb23 100644 --- a/client/resource_group/controller/limiter_test.go +++ b/client/resource_group/controller/limiter_test.go @@ -1,7 +1,3 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package controller import ( diff --git a/client/resource_group/controller/testutil.go b/client/resource_group/controller/testutil.go index de71cff4d0b..3e769545709 100644 --- a/client/resource_group/controller/testutil.go +++ b/client/resource_group/controller/testutil.go @@ -1,7 +1,3 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package controller import "time" diff --git a/client/resource_group/controller/util.go b/client/resource_group/controller/util.go index 3b491e02c8f..09328746fea 100644 --- a/client/resource_group/controller/util.go +++ b/client/resource_group/controller/util.go @@ -1,14 +1,10 @@ -// Copyright 2023 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -16,6 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Copyright 2023 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + package controller import ( diff --git a/docs/swagger/placeholder.go b/docs/swagger/placeholder.go index c0a891a256f..ca1d43cbe0b 100644 --- a/docs/swagger/placeholder.go +++ b/docs/swagger/placeholder.go @@ -1 +1,15 @@ +// Copyright 2020 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package swagger diff --git a/pkg/btree/btree_generic.go b/pkg/btree/btree_generic.go index 599614678eb..1318ea3eb4e 100644 --- a/pkg/btree/btree_generic.go +++ b/pkg/btree/btree_generic.go @@ -1,3 +1,17 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Copyright 2014-2022 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -59,20 +73,6 @@ // Those without this prefix are specific to the 'Item' interface, and use // its 'Less' function for ordering. -// Copyright 2022 TiKV Project Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - // nolint package btree diff --git a/pkg/btree/btree_generic_test.go b/pkg/btree/btree_generic_test.go index 8b432229470..1cf0851cb44 100644 --- a/pkg/btree/btree_generic_test.go +++ b/pkg/btree/btree_generic_test.go @@ -1,4 +1,4 @@ -// Copyright 2014-2022 Google Inc. +// Copyright 2022 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Copyright 2022 TiKV Project Authors. +// Copyright 2014-2022 Google Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/errs/errno.go b/pkg/errs/errno.go index 25f69af327f..571d90b8651 100644 --- a/pkg/errs/errno.go +++ b/pkg/errs/errno.go @@ -1,4 +1,4 @@ -// Copyright 2020 PingCAP, Inc. +// Copyright 2020 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/gc/safepoint_v2.go b/pkg/gc/safepoint_v2.go index acdd4e6eef8..3ecae974785 100644 --- a/pkg/gc/safepoint_v2.go +++ b/pkg/gc/safepoint_v2.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/gctuner/tuner_test.go b/pkg/gctuner/tuner_test.go index d46bb8edf94..ef02010bcb7 100644 --- a/pkg/gctuner/tuner_test.go +++ b/pkg/gctuner/tuner_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/mcs/resourcemanager/server/resource_group.go b/pkg/mcs/resourcemanager/server/resource_group.go index 65d2959e870..d8ac16c7c5b 100644 --- a/pkg/mcs/resourcemanager/server/resource_group.go +++ b/pkg/mcs/resourcemanager/server/resource_group.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/mcs/resourcemanager/server/resource_group_test.go b/pkg/mcs/resourcemanager/server/resource_group_test.go index 8f058a212bb..a72e09172c6 100644 --- a/pkg/mcs/resourcemanager/server/resource_group_test.go +++ b/pkg/mcs/resourcemanager/server/resource_group_test.go @@ -1,3 +1,17 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package server import ( diff --git a/pkg/mcs/scheduling/server/cluster.go b/pkg/mcs/scheduling/server/cluster.go index c07174762ff..b55655e9747 100644 --- a/pkg/mcs/scheduling/server/cluster.go +++ b/pkg/mcs/scheduling/server/cluster.go @@ -1,3 +1,17 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package server import ( diff --git a/pkg/mcs/server/server.go b/pkg/mcs/server/server.go index 9692b52beb3..2f06829c9f0 100644 --- a/pkg/mcs/server/server.go +++ b/pkg/mcs/server/server.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/mcs/utils/constant/constant.go b/pkg/mcs/utils/constant/constant.go index 6f684bdb977..297c4798257 100644 --- a/pkg/mcs/utils/constant/constant.go +++ b/pkg/mcs/utils/constant/constant.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/mcs/utils/expected_primary.go b/pkg/mcs/utils/expected_primary.go index b8a317ed251..a31abc9e88f 100644 --- a/pkg/mcs/utils/expected_primary.go +++ b/pkg/mcs/utils/expected_primary.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/mcs/utils/util.go b/pkg/mcs/utils/util.go index 9e187b3a361..92f8a1d1e8b 100644 --- a/pkg/mcs/utils/util.go +++ b/pkg/mcs/utils/util.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/schedule/filter/filters_test.go b/pkg/schedule/filter/filters_test.go index 35ba2c33589..c093f249083 100644 --- a/pkg/schedule/filter/filters_test.go +++ b/pkg/schedule/filter/filters_test.go @@ -4,13 +4,14 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package filter import ( diff --git a/pkg/schedule/schedulers/balance_benchmark_test.go b/pkg/schedule/schedulers/balance_benchmark_test.go index abf4c0b3def..ae807b2c363 100644 --- a/pkg/schedule/schedulers/balance_benchmark_test.go +++ b/pkg/schedule/schedulers/balance_benchmark_test.go @@ -4,13 +4,14 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package schedulers import ( diff --git a/pkg/storage/hot_region_storage.go b/pkg/storage/hot_region_storage.go index b5ec8dda6ba..bbf5fef456d 100644 --- a/pkg/storage/hot_region_storage.go +++ b/pkg/storage/hot_region_storage.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/storage/hot_region_storage_test.go b/pkg/storage/hot_region_storage_test.go index cd713685eb0..92d6a1c96cd 100644 --- a/pkg/storage/hot_region_storage_test.go +++ b/pkg/storage/hot_region_storage_test.go @@ -8,6 +8,7 @@ // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. diff --git a/pkg/storage/leveldb_backend_test.go b/pkg/storage/leveldb_backend_test.go index 0bf626834cd..d4b73ee9059 100644 --- a/pkg/storage/leveldb_backend_test.go +++ b/pkg/storage/leveldb_backend_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/storage/region_storage.go b/pkg/storage/region_storage.go index 8e6a4d87429..e7a744b6f9d 100644 --- a/pkg/storage/region_storage.go +++ b/pkg/storage/region_storage.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/storage/region_storage_test.go b/pkg/storage/region_storage_test.go index c0ae04c8ded..a611420315a 100644 --- a/pkg/storage/region_storage_test.go +++ b/pkg/storage/region_storage_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/unsaferecovery/unsafe_recovery_controller.go b/pkg/unsaferecovery/unsafe_recovery_controller.go index dce2069e316..5ece375f5de 100644 --- a/pkg/unsaferecovery/unsafe_recovery_controller.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/unsaferecovery/unsafe_recovery_controller_test.go b/pkg/unsaferecovery/unsafe_recovery_controller_test.go index 0dab6aeca1d..669ca16de34 100644 --- a/pkg/unsaferecovery/unsafe_recovery_controller_test.go +++ b/pkg/unsaferecovery/unsafe_recovery_controller_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/utils/grpcutil/grpcutil_test.go b/pkg/utils/grpcutil/grpcutil_test.go index db77a2e4002..d11777a3cfc 100644 --- a/pkg/utils/grpcutil/grpcutil_test.go +++ b/pkg/utils/grpcutil/grpcutil_test.go @@ -1,3 +1,17 @@ +// Copyright 2019 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package grpcutil import ( diff --git a/pkg/utils/keypath/cluster_id.go b/pkg/utils/keypath/cluster_id.go index e1117f15738..f12e6205916 100644 --- a/pkg/utils/keypath/cluster_id.go +++ b/pkg/utils/keypath/cluster_id.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/utils/keypath/key_path_test.go b/pkg/utils/keypath/key_path_test.go index 18096ca47bb..e6c2a4c3f6b 100644 --- a/pkg/utils/keypath/key_path_test.go +++ b/pkg/utils/keypath/key_path_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/pkg/utils/netutil/address_test.go b/pkg/utils/netutil/address_test.go index 127c9a6d0f7..70302450a7d 100644 --- a/pkg/utils/netutil/address_test.go +++ b/pkg/utils/netutil/address_test.go @@ -4,13 +4,14 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package netutil import ( diff --git a/pkg/utils/tempurl/check_env_dummy.go b/pkg/utils/tempurl/check_env_dummy.go index 58d889bbfd6..81d1acdbe7e 100644 --- a/pkg/utils/tempurl/check_env_dummy.go +++ b/pkg/utils/tempurl/check_env_dummy.go @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + //go:build !linux // +build !linux diff --git a/pkg/utils/tempurl/check_env_linux.go b/pkg/utils/tempurl/check_env_linux.go index 7e3dffc105b..15c17f55753 100644 --- a/pkg/utils/tempurl/check_env_linux.go +++ b/pkg/utils/tempurl/check_env_linux.go @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + //go:build linux // +build linux diff --git a/pkg/utils/timerutil/pool.go b/pkg/utils/timerutil/pool.go index cee061201c2..32f76b39572 100644 --- a/pkg/utils/timerutil/pool.go +++ b/pkg/utils/timerutil/pool.go @@ -1,3 +1,17 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Copyright 2020 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/utils/timerutil/pool_test.go b/pkg/utils/timerutil/pool_test.go index 3e0575daea3..ad550c7b56a 100644 --- a/pkg/utils/timerutil/pool_test.go +++ b/pkg/utils/timerutil/pool_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + // Copyright 2020 The Go Authors. All rights reserved. // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. diff --git a/pkg/window/counter.go b/pkg/window/counter.go index c56c202a414..faf8d0dddc6 100644 --- a/pkg/window/counter.go +++ b/pkg/window/counter.go @@ -1,6 +1,3 @@ -// The MIT License (MIT) -// Copyright (c) 2022 go-kratos Project Authors. -// // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +// The MIT License (MIT) +// Copyright (c) 2022 go-kratos Project Authors. + package window import ( diff --git a/pkg/window/counter_test.go b/pkg/window/counter_test.go index 2cb25babdba..86a32f53537 100644 --- a/pkg/window/counter_test.go +++ b/pkg/window/counter_test.go @@ -1,6 +1,3 @@ -// The MIT License (MIT) -// Copyright (c) 2022 go-kratos Project Authors. -// // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +// The MIT License (MIT) +// Copyright (c) 2022 go-kratos Project Authors. + package window import ( diff --git a/pkg/window/policy.go b/pkg/window/policy.go index 14e33e3ee74..1852ba21ac5 100644 --- a/pkg/window/policy.go +++ b/pkg/window/policy.go @@ -1,6 +1,3 @@ -// The MIT License (MIT) -// Copyright (c) 2022 go-kratos Project Authors. -// // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +// The MIT License (MIT) +// Copyright (c) 2022 go-kratos Project Authors. + package window import ( diff --git a/pkg/window/policy_test.go b/pkg/window/policy_test.go index b5a04c03e4b..7289b2c85c0 100644 --- a/pkg/window/policy_test.go +++ b/pkg/window/policy_test.go @@ -1,6 +1,3 @@ -// The MIT License (MIT) -// Copyright (c) 2022 go-kratos Project Authors. -// // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +// The MIT License (MIT) +// Copyright (c) 2022 go-kratos Project Authors. + package window import ( diff --git a/pkg/window/reduce.go b/pkg/window/reduce.go index 0df21ff4c4f..f6968d0b72f 100644 --- a/pkg/window/reduce.go +++ b/pkg/window/reduce.go @@ -1,6 +1,3 @@ -// The MIT License (MIT) -// Copyright (c) 2022 go-kratos Project Authors. -// // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +// The MIT License (MIT) +// Copyright (c) 2022 go-kratos Project Authors. + package window // Sum the values within the window. diff --git a/pkg/window/window.go b/pkg/window/window.go index 6d7a54131ce..c48e4b96b42 100644 --- a/pkg/window/window.go +++ b/pkg/window/window.go @@ -1,6 +1,3 @@ -// The MIT License (MIT) -// Copyright (c) 2022 go-kratos Project Authors. -// // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +// The MIT License (MIT) +// Copyright (c) 2022 go-kratos Project Authors. + package window import "fmt" diff --git a/pkg/window/window_test.go b/pkg/window/window_test.go index 4cedd12ae01..ffc0e5af4bb 100644 --- a/pkg/window/window_test.go +++ b/pkg/window/window_test.go @@ -1,6 +1,3 @@ -// The MIT License (MIT) -// Copyright (c) 2022 go-kratos Project Authors. -// // Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); @@ -15,6 +12,9 @@ // See the License for the specific language governing permissions and // limitations under the License. +// The MIT License (MIT) +// Copyright (c) 2022 go-kratos Project Authors. + package window import ( diff --git a/server/api/pprof_test.go b/server/api/pprof_test.go index 1bb1668e374..fc3e37adf83 100644 --- a/server/api/pprof_test.go +++ b/server/api/pprof_test.go @@ -4,13 +4,14 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package api import ( diff --git a/tests/integrations/mcs/discovery/register_test.go b/tests/integrations/mcs/discovery/register_test.go index 217d8137e5b..92fe11dbcb0 100644 --- a/tests/integrations/mcs/discovery/register_test.go +++ b/tests/integrations/mcs/discovery/register_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/tests/integrations/mcs/scheduling/api_test.go b/tests/integrations/mcs/scheduling/api_test.go index 55407831df2..75d784dcb6e 100644 --- a/tests/integrations/mcs/scheduling/api_test.go +++ b/tests/integrations/mcs/scheduling/api_test.go @@ -1,3 +1,17 @@ +// Copyright 2023 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package scheduling_test import ( diff --git a/tests/integrations/mcs/scheduling/server_test.go b/tests/integrations/mcs/scheduling/server_test.go index d79bcb47228..9b01e6e48a6 100644 --- a/tests/integrations/mcs/scheduling/server_test.go +++ b/tests/integrations/mcs/scheduling/server_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/tests/integrations/mcs/tso/server_test.go b/tests/integrations/mcs/tso/server_test.go index 168741204de..84044867409 100644 --- a/tests/integrations/mcs/tso/server_test.go +++ b/tests/integrations/mcs/tso/server_test.go @@ -4,7 +4,7 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, diff --git a/tests/integrations/realcluster/cluster.go b/tests/integrations/realcluster/cluster.go index fc5d1bc4441..9485a2aff79 100644 --- a/tests/integrations/realcluster/cluster.go +++ b/tests/integrations/realcluster/cluster.go @@ -1,4 +1,4 @@ -// Copyright 2024 TiKV Authors +// Copyright 2024 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/integrations/realcluster/cluster_id_test.go b/tests/integrations/realcluster/cluster_id_test.go index 82233cd5b8c..32e038e74d4 100644 --- a/tests/integrations/realcluster/cluster_id_test.go +++ b/tests/integrations/realcluster/cluster_id_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 TiKV Authors +// Copyright 2024 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/integrations/realcluster/etcd_key_test.go b/tests/integrations/realcluster/etcd_key_test.go index 10c392834af..c96455e4667 100644 --- a/tests/integrations/realcluster/etcd_key_test.go +++ b/tests/integrations/realcluster/etcd_key_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 TiKV Authors +// Copyright 2024 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/integrations/realcluster/mock_db.go b/tests/integrations/realcluster/mock_db.go index 8a21bc1bb7d..eda12e941df 100644 --- a/tests/integrations/realcluster/mock_db.go +++ b/tests/integrations/realcluster/mock_db.go @@ -1,4 +1,4 @@ -// Copyright 2023 TiKV Authors +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/integrations/realcluster/reboot_pd_test.go b/tests/integrations/realcluster/reboot_pd_test.go index 1a042872b93..ef85ea0bf99 100644 --- a/tests/integrations/realcluster/reboot_pd_test.go +++ b/tests/integrations/realcluster/reboot_pd_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 TiKV Authors +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/integrations/realcluster/scheduler_test.go b/tests/integrations/realcluster/scheduler_test.go index 0e4d10acd80..4daafdeca27 100644 --- a/tests/integrations/realcluster/scheduler_test.go +++ b/tests/integrations/realcluster/scheduler_test.go @@ -1,4 +1,4 @@ -// Copyright 2024 TiKV Authors +// Copyright 2024 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/integrations/realcluster/ts_test.go b/tests/integrations/realcluster/ts_test.go index 58f50abdf3e..4df0b9181c4 100644 --- a/tests/integrations/realcluster/ts_test.go +++ b/tests/integrations/realcluster/ts_test.go @@ -1,4 +1,4 @@ -// Copyright 2023 TiKV Authors +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tests/integrations/realcluster/util.go b/tests/integrations/realcluster/util.go index a33f9c071c6..fc350832eea 100644 --- a/tests/integrations/realcluster/util.go +++ b/tests/integrations/realcluster/util.go @@ -1,4 +1,4 @@ -// Copyright 2023 TiKV Authors +// Copyright 2023 TiKV Project Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/tools/pd-backup/pdbackup/backup_test.go b/tools/pd-backup/pdbackup/backup_test.go index 3734fb6782b..e6dc1e6345f 100644 --- a/tools/pd-backup/pdbackup/backup_test.go +++ b/tools/pd-backup/pdbackup/backup_test.go @@ -1,3 +1,17 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package pdbackup import ( diff --git a/tools/pd-ctl/pdctl/command/global_test.go b/tools/pd-ctl/pdctl/command/global_test.go index 0d1cf74ac74..74efec76d5c 100644 --- a/tools/pd-ctl/pdctl/command/global_test.go +++ b/tools/pd-ctl/pdctl/command/global_test.go @@ -4,13 +4,14 @@ // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + package command import ( diff --git a/tools/pd-heartbeat-bench/config/config.go b/tools/pd-heartbeat-bench/config/config.go index 41c14845074..42a506b7d50 100644 --- a/tools/pd-heartbeat-bench/config/config.go +++ b/tools/pd-heartbeat-bench/config/config.go @@ -1,3 +1,17 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package config import ( diff --git a/tools/pd-simulator/simulator/metrics.go b/tools/pd-simulator/simulator/metrics.go index 19e675c56a9..fabd626591e 100644 --- a/tools/pd-simulator/simulator/metrics.go +++ b/tools/pd-simulator/simulator/metrics.go @@ -1,3 +1,17 @@ +// Copyright 2022 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package simulator import "github.com/prometheus/client_golang/prometheus" diff --git a/tools/pd-ut/alloc/check_env_dummy.go b/tools/pd-ut/alloc/check_env_dummy.go index b9b8eb4827a..1375623c1d8 100644 --- a/tools/pd-ut/alloc/check_env_dummy.go +++ b/tools/pd-ut/alloc/check_env_dummy.go @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + //go:build !linux // +build !linux diff --git a/tools/pd-ut/alloc/check_env_linux.go b/tools/pd-ut/alloc/check_env_linux.go index c4d20bf6a65..4e1c058bfb6 100644 --- a/tools/pd-ut/alloc/check_env_linux.go +++ b/tools/pd-ut/alloc/check_env_linux.go @@ -11,6 +11,7 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. + //go:build linux // +build linux From b66703c26d2bff9e9d2e28416233d39839fb4b22 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Tue, 21 Jan 2025 16:24:26 +0800 Subject: [PATCH 23/33] scheduler: support changing batch for slow score scheduler (#8888) ref tikv/pd#7156 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/schedule/schedulers/evict_slow_store.go | 36 +++++++++-- .../schedulers/evict_slow_store_test.go | 61 +++++++++++++++++++ .../pd-ctl/tests/scheduler/scheduler_test.go | 16 +++++ 3 files changed, 109 insertions(+), 4 deletions(-) diff --git a/pkg/schedule/schedulers/evict_slow_store.go b/pkg/schedule/schedulers/evict_slow_store.go index 8d8c014b110..e3e09430359 100644 --- a/pkg/schedule/schedulers/evict_slow_store.go +++ b/pkg/schedule/schedulers/evict_slow_store.go @@ -49,6 +49,9 @@ type evictSlowStoreSchedulerConfig struct { // Duration gap for recovering the candidate, unit: s. RecoveryDurationGap uint64 `json:"recovery-duration"` EvictedStores []uint64 `json:"evict-stores"` + // TODO: We only add batch for evict-slow-store-scheduler now. + // If necessary, we also need to support evict-slow-trend-scheduler. + Batch int `json:"batch"` } func initEvictSlowStoreSchedulerConfig() *evictSlowStoreSchedulerConfig { @@ -57,6 +60,7 @@ func initEvictSlowStoreSchedulerConfig() *evictSlowStoreSchedulerConfig { lastSlowStoreCaptureTS: time.Time{}, RecoveryDurationGap: defaultRecoveryDurationGap, EvictedStores: make([]uint64, 0), + Batch: EvictLeaderBatchSize, } } @@ -65,6 +69,7 @@ func (conf *evictSlowStoreSchedulerConfig) clone() *evictSlowStoreSchedulerConfi defer conf.RUnlock() return &evictSlowStoreSchedulerConfig{ RecoveryDurationGap: conf.RecoveryDurationGap, + Batch: conf.Batch, } } @@ -81,8 +86,10 @@ func (conf *evictSlowStoreSchedulerConfig) getKeyRangesByID(id uint64) []core.Ke return []core.KeyRange{core.NewKeyRange("", "")} } -func (*evictSlowStoreSchedulerConfig) getBatch() int { - return EvictLeaderBatchSize +func (conf *evictSlowStoreSchedulerConfig) getBatch() int { + conf.RLock() + defer conf.RUnlock() + return conf.Batch } func (conf *evictSlowStoreSchedulerConfig) evictStore() uint64 { @@ -145,22 +152,39 @@ func (handler *evictSlowStoreHandler) updateConfig(w http.ResponseWriter, r *htt return } recoveryDurationGapFloat, ok := input["recovery-duration"].(float64) - if !ok { + if input["recovery-duration"] != nil && !ok { handler.rd.JSON(w, http.StatusInternalServerError, errors.New("invalid argument for 'recovery-duration'").Error()) return } + batch := handler.config.getBatch() + batchFloat, ok := input["batch"].(float64) + if input["batch"] != nil && !ok { + handler.rd.JSON(w, http.StatusInternalServerError, errors.New("invalid argument for 'batch'").Error()) + return + } + if ok { + if batchFloat < 1 || batchFloat > 10 { + handler.rd.JSON(w, http.StatusBadRequest, "batch is invalid, it should be in [1, 10]") + return + } + batch = (int)(batchFloat) + } + handler.config.Lock() defer handler.config.Unlock() prevRecoveryDurationGap := handler.config.RecoveryDurationGap + prevBatch := handler.config.Batch recoveryDurationGap := uint64(recoveryDurationGapFloat) handler.config.RecoveryDurationGap = recoveryDurationGap + handler.config.Batch = batch if err := handler.config.save(); err != nil { handler.rd.JSON(w, http.StatusInternalServerError, err.Error()) handler.config.RecoveryDurationGap = prevRecoveryDurationGap + handler.config.Batch = prevBatch return } - log.Info("evict-slow-store-scheduler update 'recovery-duration' - unit: s", zap.Uint64("prev", prevRecoveryDurationGap), zap.Uint64("cur", recoveryDurationGap)) + log.Info("evict-slow-store-scheduler update config", zap.Uint64("prev-recovery-duration", prevRecoveryDurationGap), zap.Uint64("cur-recovery-duration", recoveryDurationGap), zap.Int("prev-batch", prevBatch), zap.Int("cur-batch", batch)) handler.rd.JSON(w, http.StatusOK, "Config updated.") } @@ -194,6 +218,9 @@ func (s *evictSlowStoreScheduler) ReloadConfig() error { if err := s.conf.load(newCfg); err != nil { return err } + if newCfg.Batch == 0 { + newCfg.Batch = EvictLeaderBatchSize + } old := make(map[uint64]struct{}) for _, id := range s.conf.EvictedStores { old[id] = struct{}{} @@ -205,6 +232,7 @@ func (s *evictSlowStoreScheduler) ReloadConfig() error { pauseAndResumeLeaderTransfer(s.conf.cluster, constant.In, old, new) s.conf.RecoveryDurationGap = newCfg.RecoveryDurationGap s.conf.EvictedStores = newCfg.EvictedStores + s.conf.Batch = newCfg.Batch return nil } diff --git a/pkg/schedule/schedulers/evict_slow_store_test.go b/pkg/schedule/schedulers/evict_slow_store_test.go index 636d856fc14..29cf1c4f2fa 100644 --- a/pkg/schedule/schedulers/evict_slow_store_test.go +++ b/pkg/schedule/schedulers/evict_slow_store_test.go @@ -18,6 +18,7 @@ import ( "context" "testing" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/pingcap/failpoint" @@ -146,3 +147,63 @@ func (suite *evictSlowStoreTestSuite) TestEvictSlowStorePersistFail() { ops, _ = suite.es.Schedule(suite.tc, false) re.NotEmpty(ops) } + +func TestEvictSlowStoreBatch(t *testing.T) { + re := require.New(t) + cancel, _, tc, oc := prepareSchedulersTest() + defer cancel() + + // Add stores + tc.AddLeaderStore(1, 0) + tc.AddLeaderStore(2, 0) + tc.AddLeaderStore(3, 0) + // Add regions with leader in store 1 + for i := range 10000 { + tc.AddLeaderRegion(uint64(i), 1, 2) + } + + storage := storage.NewStorageWithMemoryBackend() + es, err := CreateScheduler(types.EvictSlowStoreScheduler, oc, storage, ConfigSliceDecoder(types.EvictSlowStoreScheduler, []string{}), nil) + re.NoError(err) + re.NoError(failpoint.Enable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap", "return(true)")) + storeInfo := tc.GetStore(1) + newStoreInfo := storeInfo.Clone(func(store *core.StoreInfo) { + store.GetStoreStats().SlowScore = 100 + }) + tc.PutStore(newStoreInfo) + re.True(es.IsScheduleAllowed(tc)) + // Add evict leader scheduler to store 1 + ops, _ := es.Schedule(tc, false) + re.Len(ops, 3) + operatorutil.CheckMultiTargetTransferLeader(re, ops[0], operator.OpLeader, 1, []uint64{2}) + re.Equal(types.EvictSlowStoreScheduler.String(), ops[0].Desc()) + + es.(*evictSlowStoreScheduler).conf.Batch = 5 + re.NoError(es.(*evictSlowStoreScheduler).conf.save()) + ops, _ = es.Schedule(tc, false) + re.Len(ops, 5) + + newStoreInfo = storeInfo.Clone(func(store *core.StoreInfo) { + store.GetStoreStats().SlowScore = 0 + }) + + tc.PutStore(newStoreInfo) + // no slow store need to evict. + ops, _ = es.Schedule(tc, false) + re.Empty(ops) + + es2, ok := es.(*evictSlowStoreScheduler) + re.True(ok) + re.Zero(es2.conf.evictStore()) + + // check the value from storage. + var persistValue evictSlowStoreSchedulerConfig + err = es2.conf.load(&persistValue) + re.NoError(err) + + re.Equal(es2.conf.EvictedStores, persistValue.EvictedStores) + re.Zero(persistValue.evictStore()) + re.True(persistValue.readyForRecovery()) + re.Equal(5, persistValue.Batch) + re.NoError(failpoint.Disable("github.com/tikv/pd/pkg/schedule/schedulers/transientRecoveryGap")) +} diff --git a/tools/pd-ctl/tests/scheduler/scheduler_test.go b/tools/pd-ctl/tests/scheduler/scheduler_test.go index 9de62f2b1db..2b48e4abf21 100644 --- a/tools/pd-ctl/tests/scheduler/scheduler_test.go +++ b/tools/pd-ctl/tests/scheduler/scheduler_test.go @@ -566,6 +566,22 @@ func (suite *schedulerTestSuite) checkSchedulerConfig(cluster *pdTests.TestClust }) echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-leader-scheduler"}, nil) re.Contains(echo, "Success!") + + // test evict slow store scheduler + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-slow-store-scheduler"}, nil) + re.Contains(echo, "Success!") + conf = make(map[string]any) + conf1 = make(map[string]any) + testutil.Eventually(re, func() bool { + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "evict-slow-store-scheduler", "show"}, &conf) + return conf["batch"] == 3. + }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "evict-slow-store-scheduler", "set", "batch", "10"}, nil) + re.Contains(echo, "Success!") + testutil.Eventually(re, func() bool { + mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "evict-slow-store-scheduler"}, &conf1) + return conf1["batch"] == 10. + }) } func (suite *schedulerTestSuite) TestGrantHotRegionScheduler() { From ae6df14874d4deeba5713bbb6e390c82c1ab7ef3 Mon Sep 17 00:00:00 2001 From: JmPotato Date: Wed, 22 Jan 2025 11:41:23 +0800 Subject: [PATCH 24/33] client/sd: unify the service discovery callbacks within a struct (#9014) ref tikv/pd#8690 Unify the service discovery callbacks within a struct. Signed-off-by: JmPotato --- client/clients/tso/client.go | 5 +- client/inner_client.go | 5 +- client/servicediscovery/callbacks.go | 104 ++++++++++++++++++ .../mock_service_discovery.go | 11 +- client/servicediscovery/service_discovery.go | 92 ++++++---------- .../service_discovery_test.go | 3 +- .../servicediscovery/tso_service_discovery.go | 27 ++--- tests/integrations/client/client_test.go | 3 +- 8 files changed, 166 insertions(+), 84 deletions(-) create mode 100644 client/servicediscovery/callbacks.go diff --git a/client/clients/tso/client.go b/client/clients/tso/client.go index 7bc768ee21b..8af2890750e 100644 --- a/client/clients/tso/client.go +++ b/client/clients/tso/client.go @@ -116,9 +116,8 @@ func NewClient( }, } - eventSrc := svcDiscovery.(sd.TSOEventSource) - eventSrc.SetTSOLeaderURLUpdatedCallback(c.updateTSOLeaderURL) - c.svcDiscovery.AddServiceURLsSwitchedCallback(c.scheduleUpdateTSOConnectionCtxs) + c.svcDiscovery.ExecAndAddLeaderSwitchedCallback(c.updateTSOLeaderURL) + c.svcDiscovery.AddMembersChangedCallback(c.scheduleUpdateTSOConnectionCtxs) return c } diff --git a/client/inner_client.go b/client/inner_client.go index cff5b95ed54..de13bb324f5 100644 --- a/client/inner_client.go +++ b/client/inner_client.go @@ -143,11 +143,12 @@ func (c *innerClient) resetTSOClientLocked(mode pdpb.ServiceMode) { } } -func (c *innerClient) scheduleUpdateTokenConnection() { +func (c *innerClient) scheduleUpdateTokenConnection(string) error { select { case c.updateTokenConnectionCh <- struct{}{}: default: } + return nil } func (c *innerClient) getServiceMode() pdpb.ServiceMode { @@ -188,7 +189,7 @@ func (c *innerClient) setup() error { } // Register callbacks - c.serviceDiscovery.AddServingURLSwitchedCallback(c.scheduleUpdateTokenConnection) + c.serviceDiscovery.AddLeaderSwitchedCallback(c.scheduleUpdateTokenConnection) // Create dispatchers c.createTokenDispatcher() diff --git a/client/servicediscovery/callbacks.go b/client/servicediscovery/callbacks.go new file mode 100644 index 00000000000..c21217f7e44 --- /dev/null +++ b/client/servicediscovery/callbacks.go @@ -0,0 +1,104 @@ +// Copyright 2025 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package servicediscovery + +import ( + "sync" + + "github.com/pingcap/kvproto/pkg/pdpb" +) + +type leaderSwitchedCallbackFunc func(string) error + +// serviceCallbacks contains all the callback functions for service discovery events +type serviceCallbacks struct { + sync.RWMutex + // serviceModeUpdateCb will be called when the service mode gets updated + serviceModeUpdateCb func(pdpb.ServiceMode) + // leaderSwitchedCbs will be called after the leader switched + leaderSwitchedCbs []leaderSwitchedCallbackFunc + // membersChangedCbs will be called after there is any membership change in the + // leader and followers + membersChangedCbs []func() +} + +func newServiceCallbacks() *serviceCallbacks { + return &serviceCallbacks{ + leaderSwitchedCbs: make([]leaderSwitchedCallbackFunc, 0), + membersChangedCbs: make([]func(), 0), + } +} + +func (c *serviceCallbacks) setServiceModeUpdateCallback(cb func(pdpb.ServiceMode)) { + c.Lock() + defer c.Unlock() + c.serviceModeUpdateCb = cb +} + +func (c *serviceCallbacks) addLeaderSwitchedCallback(cb leaderSwitchedCallbackFunc) { + c.Lock() + defer c.Unlock() + c.leaderSwitchedCbs = append(c.leaderSwitchedCbs, cb) +} + +func (c *serviceCallbacks) addMembersChangedCallback(cb func()) { + c.Lock() + defer c.Unlock() + c.membersChangedCbs = append(c.membersChangedCbs, cb) +} + +func (c *serviceCallbacks) onServiceModeUpdate(mode pdpb.ServiceMode) { + c.RLock() + cb := c.serviceModeUpdateCb + c.RUnlock() + + if cb == nil { + return + } + cb(mode) +} + +func (c *serviceCallbacks) onLeaderSwitched(leader string) error { + c.RLock() + cbs := make([]leaderSwitchedCallbackFunc, len(c.leaderSwitchedCbs)) + copy(cbs, c.leaderSwitchedCbs) + c.RUnlock() + + var err error + for _, cb := range cbs { + if cb == nil { + continue + } + err = cb(leader) + if err != nil { + return err + } + } + return nil +} + +func (c *serviceCallbacks) onMembersChanged() { + c.RLock() + cbs := make([]func(), len(c.membersChangedCbs)) + copy(cbs, c.membersChangedCbs) + c.RUnlock() + + for _, cb := range cbs { + if cb == nil { + continue + } + cb() + } +} diff --git a/client/servicediscovery/mock_service_discovery.go b/client/servicediscovery/mock_service_discovery.go index 6ca649f4575..362626243f7 100644 --- a/client/servicediscovery/mock_service_discovery.go +++ b/client/servicediscovery/mock_service_discovery.go @@ -100,8 +100,11 @@ func (*mockServiceDiscovery) ScheduleCheckMemberChanged() {} // CheckMemberChanged implements the ServiceDiscovery interface. func (*mockServiceDiscovery) CheckMemberChanged() error { return nil } -// AddServingURLSwitchedCallback implements the ServiceDiscovery interface. -func (*mockServiceDiscovery) AddServingURLSwitchedCallback(...func()) {} +// ExecAndAddLeaderSwitchedCallback implements the ServiceDiscovery interface. +func (*mockServiceDiscovery) ExecAndAddLeaderSwitchedCallback(leaderSwitchedCallbackFunc) {} -// AddServiceURLsSwitchedCallback implements the ServiceDiscovery interface. -func (*mockServiceDiscovery) AddServiceURLsSwitchedCallback(...func()) {} +// AddLeaderSwitchedCallback implements the ServiceDiscovery interface. +func (*mockServiceDiscovery) AddLeaderSwitchedCallback(leaderSwitchedCallbackFunc) {} + +// AddMembersChangedCallback implements the ServiceDiscovery interface. +func (*mockServiceDiscovery) AddMembersChangedCallback(func()) {} diff --git a/client/servicediscovery/service_discovery.go b/client/servicediscovery/service_discovery.go index f5ac665b7cd..1a6b8a7043f 100644 --- a/client/servicediscovery/service_discovery.go +++ b/client/servicediscovery/service_discovery.go @@ -126,14 +126,16 @@ type ServiceDiscovery interface { // CheckMemberChanged immediately check if there is any membership change among the leader/followers // in a quorum-based cluster or among the primary/secondaries in a primary/secondary configured cluster. CheckMemberChanged() error - // AddServingURLSwitchedCallback adds callbacks which will be called when the leader + // ExecAndAddLeaderSwitchedCallback executes the callback once and adds it to the callback list then. + ExecAndAddLeaderSwitchedCallback(cb leaderSwitchedCallbackFunc) + // AddLeaderSwitchedCallback adds callbacks which will be called when the leader // in a quorum-based cluster or the primary in a primary/secondary configured cluster // is switched. - AddServingURLSwitchedCallback(callbacks ...func()) - // AddServiceURLsSwitchedCallback adds callbacks which will be called when any leader/follower + AddLeaderSwitchedCallback(cb leaderSwitchedCallbackFunc) + // AddMembersChangedCallback adds callbacks which will be called when any leader/follower // in a quorum-based cluster or any primary/secondary in a primary/secondary configured cluster // is changed. - AddServiceURLsSwitchedCallback(callbacks ...func()) + AddMembersChangedCallback(cb func()) } // ServiceClient is an interface that defines a set of operations for a raw PD gRPC client to specific PD server. @@ -394,18 +396,8 @@ func (c *serviceBalancer) get() (ret ServiceClient) { // UpdateKeyspaceIDFunc is the function type for updating the keyspace ID. type UpdateKeyspaceIDFunc func() error -type tsoLeaderURLUpdatedFunc func(string) error -// TSOEventSource subscribes to events related to changes in the TSO leader/primary from the service discovery. -type TSOEventSource interface { - // SetTSOLeaderURLUpdatedCallback adds a callback which will be called when the TSO leader/primary is updated. - SetTSOLeaderURLUpdatedCallback(callback tsoLeaderURLUpdatedFunc) -} - -var ( - _ ServiceDiscovery = (*serviceDiscovery)(nil) - _ TSOEventSource = (*serviceDiscovery)(nil) -) +var _ ServiceDiscovery = (*serviceDiscovery)(nil) // serviceDiscovery is the service discovery client of PD/PD service which is quorum based type serviceDiscovery struct { @@ -426,15 +418,7 @@ type serviceDiscovery struct { // url -> a gRPC connection clientConns sync.Map // Store as map[string]*grpc.ClientConn - // serviceModeUpdateCb will be called when the service mode gets updated - serviceModeUpdateCb func(pdpb.ServiceMode) - // leaderSwitchedCbs will be called after the leader switched - leaderSwitchedCbs []func() - // membersChangedCbs will be called after there is any membership change in the - // leader and followers - membersChangedCbs []func() - // tsoLeaderUpdatedCb will be called when the TSO leader is updated. - tsoLeaderUpdatedCb tsoLeaderURLUpdatedFunc + callbacks *serviceCallbacks checkMembershipCh chan struct{} @@ -474,12 +458,13 @@ func NewServiceDiscovery( cancel: cancel, wg: wg, apiCandidateNodes: [apiKindCount]*serviceBalancer{newServiceBalancer(emptyErrorFn), newServiceBalancer(regionAPIErrorFn)}, - serviceModeUpdateCb: serviceModeUpdateCb, + callbacks: newServiceCallbacks(), updateKeyspaceIDFunc: updateKeyspaceIDFunc, keyspaceID: keyspaceID, tlsCfg: tlsCfg, option: option, } + pdsd.callbacks.setServiceModeUpdateCallback(serviceModeUpdateCb) urls = tlsutil.AddrsToURLs(urls, tlsCfg) pdsd.urls.Store(urls) return pdsd @@ -570,7 +555,7 @@ func (c *serviceDiscovery) updateServiceModeLoop() { failpoint.Return() }) failpoint.Inject("usePDServiceMode", func() { - c.serviceModeUpdateCb(pdpb.ServiceMode_PD_SVC_MODE) + c.callbacks.onServiceModeUpdate(pdpb.ServiceMode_PD_SVC_MODE) failpoint.Return() }) @@ -791,27 +776,29 @@ func (c *serviceDiscovery) CheckMemberChanged() error { return c.updateMember() } -// AddServingURLSwitchedCallback adds callbacks which will be called -// when the leader is switched. -func (c *serviceDiscovery) AddServingURLSwitchedCallback(callbacks ...func()) { - c.leaderSwitchedCbs = append(c.leaderSwitchedCbs, callbacks...) -} - -// AddServiceURLsSwitchedCallback adds callbacks which will be called when -// any leader/follower is changed. -func (c *serviceDiscovery) AddServiceURLsSwitchedCallback(callbacks ...func()) { - c.membersChangedCbs = append(c.membersChangedCbs, callbacks...) -} - -// SetTSOLeaderURLUpdatedCallback adds a callback which will be called when the TSO leader is updated. -func (c *serviceDiscovery) SetTSOLeaderURLUpdatedCallback(callback tsoLeaderURLUpdatedFunc) { +// ExecAndAddLeaderSwitchedCallback executes the callback once and adds it to the callback list then. +func (c *serviceDiscovery) ExecAndAddLeaderSwitchedCallback(callback leaderSwitchedCallbackFunc) { url := c.getLeaderURL() if len(url) > 0 { if err := callback(url); err != nil { - log.Error("[tso] failed to call back when tso leader url update", zap.String("url", url), errs.ZapError(err)) + log.Error("[pd] failed to run a callback with the current leader url", + zap.String("url", url), errs.ZapError(err)) } } - c.tsoLeaderUpdatedCb = callback + c.AddLeaderSwitchedCallback(callback) +} + +// AddLeaderSwitchedCallback adds callbacks which will be called when the leader +// in a quorum-based cluster or the primary in a primary/secondary configured cluster +// is switched. +func (c *serviceDiscovery) AddLeaderSwitchedCallback(callback leaderSwitchedCallbackFunc) { + c.callbacks.addLeaderSwitchedCallback(callback) +} + +// AddMembersChangedCallback adds callbacks which will be called when any primary/secondary +// in a primary/secondary configured cluster is changed. +func (c *serviceDiscovery) AddMembersChangedCallback(callback func()) { + c.callbacks.addMembersChangedCallback(callback) } // getLeaderURL returns the leader URL. @@ -867,9 +854,7 @@ func (c *serviceDiscovery) checkServiceModeChanged() error { // If the method is not supported, we set it to pd mode. // TODO: it's a hack way to solve the compatibility issue. // we need to remove this after all maintained version supports the method. - if c.serviceModeUpdateCb != nil { - c.serviceModeUpdateCb(pdpb.ServiceMode_PD_SVC_MODE) - } + c.callbacks.onServiceModeUpdate(pdpb.ServiceMode_PD_SVC_MODE) return nil } return err @@ -877,9 +862,7 @@ func (c *serviceDiscovery) checkServiceModeChanged() error { if clusterInfo == nil || len(clusterInfo.ServiceModes) == 0 { return errors.WithStack(errs.ErrNoServiceModeReturned) } - if c.serviceModeUpdateCb != nil { - c.serviceModeUpdateCb(clusterInfo.ServiceModes[0]) - } + c.callbacks.onServiceModeUpdate(clusterInfo.ServiceModes[0]) return nil } @@ -968,9 +951,7 @@ func (c *serviceDiscovery) updateURLs(members []*pdpb.Member) { } c.urls.Store(urls) // Run callbacks to reflect the membership changes in the leader and followers. - for _, cb := range c.membersChangedCbs { - cb() - } + c.callbacks.onMembersChanged() log.Info("[pd] update member urls", zap.Strings("old-urls", oldURLs), zap.Strings("new-urls", urls)) } @@ -987,13 +968,8 @@ func (c *serviceDiscovery) switchLeader(url string) (bool, error) { c.leader.Store(leaderClient) } // Run callbacks - if c.tsoLeaderUpdatedCb != nil { - if err := c.tsoLeaderUpdatedCb(url); err != nil { - return true, err - } - } - for _, cb := range c.leaderSwitchedCbs { - cb() + if err := c.callbacks.onLeaderSwitched(url); err != nil { + return true, err } log.Info("[pd] switch leader", zap.String("new-leader", url), zap.String("old-leader", oldLeader.GetURL())) return true, err diff --git a/client/servicediscovery/service_discovery_test.go b/client/servicediscovery/service_discovery_test.go index 0a678718fdc..0c7d87d774b 100644 --- a/client/servicediscovery/service_discovery_test.go +++ b/client/servicediscovery/service_discovery_test.go @@ -400,7 +400,7 @@ func TestUpdateURLs(t *testing.T) { } return } - cli := &serviceDiscovery{option: opt.NewOption()} + cli := &serviceDiscovery{callbacks: newServiceCallbacks(), option: opt.NewOption()} cli.urls.Store([]string{}) cli.updateURLs(members[1:]) re.Equal(getURLs([]*pdpb.Member{members[1], members[3], members[2]}), cli.GetServiceURLs()) @@ -422,6 +422,7 @@ func TestGRPCDialOption(t *testing.T) { ctx, cancel := context.WithTimeout(context.TODO(), 500*time.Millisecond) defer cancel() cli := &serviceDiscovery{ + callbacks: newServiceCallbacks(), checkMembershipCh: make(chan struct{}, 1), ctx: ctx, cancel: cancel, diff --git a/client/servicediscovery/tso_service_discovery.go b/client/servicediscovery/tso_service_discovery.go index 7734fd23107..06401427522 100644 --- a/client/servicediscovery/tso_service_discovery.go +++ b/client/servicediscovery/tso_service_discovery.go @@ -55,10 +55,7 @@ const ( tsoQueryRetryInterval = 500 * time.Millisecond ) -var ( - _ ServiceDiscovery = (*tsoServiceDiscovery)(nil) - _ TSOEventSource = (*tsoServiceDiscovery)(nil) -) +var _ ServiceDiscovery = (*tsoServiceDiscovery)(nil) // keyspaceGroupSvcDiscovery is used for discovering the serving endpoints of the keyspace // group to which the keyspace belongs @@ -144,7 +141,7 @@ type tsoServiceDiscovery struct { clientConns sync.Map // Store as map[string]*grpc.ClientConn // tsoLeaderUpdatedCb will be called when the TSO leader is updated. - tsoLeaderUpdatedCb tsoLeaderURLUpdatedFunc + tsoLeaderUpdatedCb leaderSwitchedCallbackFunc checkMembershipCh chan struct{} @@ -361,16 +358,8 @@ func (c *tsoServiceDiscovery) CheckMemberChanged() error { return nil } -// AddServingURLSwitchedCallback adds callbacks which will be called when the primary in -// a primary/secondary configured cluster is switched. -func (*tsoServiceDiscovery) AddServingURLSwitchedCallback(...func()) {} - -// AddServiceURLsSwitchedCallback adds callbacks which will be called when any primary/secondary -// in a primary/secondary configured cluster is changed. -func (*tsoServiceDiscovery) AddServiceURLsSwitchedCallback(...func()) {} - -// SetTSOLeaderURLUpdatedCallback adds a callback which will be called when the TSO leader is updated. -func (c *tsoServiceDiscovery) SetTSOLeaderURLUpdatedCallback(callback tsoLeaderURLUpdatedFunc) { +// ExecAndAddLeaderSwitchedCallback executes the callback once and adds it to the callback list then. +func (c *tsoServiceDiscovery) ExecAndAddLeaderSwitchedCallback(callback leaderSwitchedCallbackFunc) { url := c.getPrimaryURL() if len(url) > 0 { if err := callback(url); err != nil { @@ -380,6 +369,14 @@ func (c *tsoServiceDiscovery) SetTSOLeaderURLUpdatedCallback(callback tsoLeaderU c.tsoLeaderUpdatedCb = callback } +// AddLeaderSwitchedCallback adds callbacks which will be called when the primary in +// a primary/secondary configured cluster is switched. +func (*tsoServiceDiscovery) AddLeaderSwitchedCallback(leaderSwitchedCallbackFunc) {} + +// AddMembersChangedCallback adds callbacks which will be called when any primary/secondary +// in a primary/secondary configured cluster is changed. +func (*tsoServiceDiscovery) AddMembersChangedCallback(func()) {} + // GetServiceClient implements ServiceDiscovery func (c *tsoServiceDiscovery) GetServiceClient() ServiceClient { return c.serviceDiscovery.GetServiceClient() diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index 0e0bf25d74d..af8cdc00a7e 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -238,8 +238,9 @@ func TestGetTSAfterTransferLeader(t *testing.T) { defer cli.Close() var leaderSwitched atomic.Bool - cli.GetServiceDiscovery().AddServingURLSwitchedCallback(func() { + cli.GetServiceDiscovery().AddLeaderSwitchedCallback(func(string) error { leaderSwitched.Store(true) + return nil }) err = cluster.GetServer(leader).ResignLeader() re.NoError(err) From 6cadbe7d59a6f89752182fead0ebb1ed076d97c7 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 22 Jan 2025 16:24:02 +0800 Subject: [PATCH 25/33] *: rename file and clean up (#8996) ref tikv/pd#5766 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 4 +-- client/http/client.go | 2 +- client/inner_client.go | 33 ++++++++++++------- client/servicediscovery/service_discovery.go | 6 ++-- pkg/keyspace/tso_keyspace_group.go | 3 -- pkg/tso/global_allocator.go | 2 +- pkg/utils/keypath/key_path.go | 2 +- .../{micro_service.go => microservice.go} | 0 server/cluster/cluster.go | 4 +-- server/config/service_middleware_config.go | 2 +- .../service_middleware_persist_options.go | 2 +- tests/integrations/mcs/tso/api_test.go | 2 +- tests/integrations/tso/client_test.go | 2 +- tools/pd-ctl/pdctl/command/config_command.go | 2 +- .../pd-ctl/pdctl/command/keyspace_command.go | 2 +- .../tests/keyspace/keyspace_group_test.go | 2 +- 16 files changed, 39 insertions(+), 31 deletions(-) rename server/apiv2/handlers/{micro_service.go => microservice.go} (100%) diff --git a/client/client.go b/client/client.go index e3d3f4e5b14..fa0a1473ba7 100644 --- a/client/client.go +++ b/client/client.go @@ -438,7 +438,7 @@ func (c *client) UpdateOption(option opt.DynamicOption, value any) error { } case opt.EnableTSOFollowerProxy: if c.inner.getServiceMode() != pdpb.ServiceMode_PD_SVC_MODE { - return errors.New("[pd] tso follower proxy is only supported in PD mode") + return errors.New("[pd] tso follower proxy is only supported when PD provides TSO") } enable, ok := value.(bool) if !ok { @@ -526,7 +526,7 @@ func (c *client) GetLocalTS(ctx context.Context, _ string) (physical int64, logi // GetMinTS implements the TSOClient interface. func (c *client) GetMinTS(ctx context.Context) (physical int64, logical int64, err error) { - // Handle compatibility issue in case of PD/PD service doesn't support GetMinTS API. + // Handle compatibility issue in case of PD doesn't support GetMinTS API. serviceMode := c.inner.getServiceMode() switch serviceMode { case pdpb.ServiceMode_UNKNOWN_SVC_MODE: diff --git a/client/http/client.go b/client/http/client.go index b7109166a30..7f49f706d8f 100644 --- a/client/http/client.go +++ b/client/http/client.go @@ -245,7 +245,7 @@ func (ci *clientInner) doRequest( if readErr != nil { logFields = append(logFields, zap.NamedError("read-body-error", err)) } else { - // PD service will return a JSON body containing the detailed error message + // PD will return a JSON body containing the detailed error message // when the status code is not `http.StatusOK` 200. bs = bytes.TrimSpace(bs) logFields = append(logFields, zap.ByteString("body", bs)) diff --git a/client/inner_client.go b/client/inner_client.go index de13bb324f5..7ce8f42386e 100644 --- a/client/inner_client.go +++ b/client/inner_client.go @@ -81,19 +81,31 @@ func (c *innerClient) setServiceMode(newMode pdpb.ServiceMode) { // If we are using TSO server proxy, we always use PD_SVC_MODE. newMode = pdpb.ServiceMode_PD_SVC_MODE } - if newMode == c.serviceMode { return } - log.Info("[pd] changing service mode", - zap.String("old-mode", c.serviceMode.String()), - zap.String("new-mode", newMode.String())) + log.Info("[pd] changing TSO provider", + zap.String("old", convertToString(c.serviceMode)), + zap.String("new", convertToString(newMode))) c.resetTSOClientLocked(newMode) oldMode := c.serviceMode c.serviceMode = newMode - log.Info("[pd] service mode changed", - zap.String("old-mode", oldMode.String()), - zap.String("new-mode", newMode.String())) + log.Info("[pd] TSO provider changed", + zap.String("old", convertToString(oldMode)), + zap.String("new", convertToString(newMode))) +} + +func convertToString(mode pdpb.ServiceMode) string { + switch mode { + case pdpb.ServiceMode_PD_SVC_MODE: + return "pd" + case pdpb.ServiceMode_API_SVC_MODE: + return "tso server" + case pdpb.ServiceMode_UNKNOWN_SVC_MODE: + return "unknown" + default: + return "invalid" + } } // Reset a new TSO client. @@ -116,9 +128,8 @@ func (c *innerClient) resetTSOClientLocked(mode pdpb.ServiceMode) { newTSOCli = tso.NewClient(c.ctx, c.option, newTSOSvcDiscovery, &tso.MSStreamBuilderFactory{}) if err := newTSOSvcDiscovery.Init(); err != nil { - log.Error("[pd] failed to initialize tso service discovery. keep the current service mode", + log.Error("[pd] failed to initialize tso service discovery", zap.Strings("svr-urls", c.svrUrls), - zap.String("current-mode", c.serviceMode.String()), zap.Error(err)) return } @@ -133,12 +144,12 @@ func (c *innerClient) resetTSOClientLocked(mode pdpb.ServiceMode) { oldTSOClient.Close() // Replace the old TSO service discovery if needed. oldTSOSvcDiscovery := c.tsoSvcDiscovery - // If newTSOSvcDiscovery is nil, that's expected, as it means we are switching to PD mode and + // If newTSOSvcDiscovery is nil, that's expected, as it means we are switching to non-microservice env and // no tso microservice discovery is needed. c.tsoSvcDiscovery = newTSOSvcDiscovery // Close the old TSO service discovery safely after both the old client and service discovery are replaced. if oldTSOSvcDiscovery != nil { - // We are switching from PD service mode to PD mode, so delete the old tso microservice discovery. + // We are switching from microservice env to non-microservice env, so delete the old tso microservice discovery. oldTSOSvcDiscovery.Close() } } diff --git a/client/servicediscovery/service_discovery.go b/client/servicediscovery/service_discovery.go index 1a6b8a7043f..cf1b3372591 100644 --- a/client/servicediscovery/service_discovery.go +++ b/client/servicediscovery/service_discovery.go @@ -399,7 +399,7 @@ type UpdateKeyspaceIDFunc func() error var _ ServiceDiscovery = (*serviceDiscovery)(nil) -// serviceDiscovery is the service discovery client of PD/PD service which is quorum based +// serviceDiscovery is the service discovery client of PD which is quorum based type serviceDiscovery struct { isInitialized bool @@ -487,7 +487,7 @@ func (c *serviceDiscovery) Init() error { log.Info("[pd] init cluster id", zap.Uint64("cluster-id", c.clusterID)) // We need to update the keyspace ID before we discover and update the service mode - // so that TSO in API mode can be initialized with the correct keyspace ID. + // so that TSO in PD can be initialized with the correct keyspace ID. if c.keyspaceID == constants.NullKeyspaceID && c.updateKeyspaceIDFunc != nil { if err := c.initRetry(c.updateKeyspaceIDFunc); err != nil { return err @@ -851,7 +851,7 @@ func (c *serviceDiscovery) checkServiceModeChanged() error { clusterInfo, err := c.getClusterInfo(c.ctx, leaderURL, c.option.Timeout) if err != nil { if strings.Contains(err.Error(), "Unimplemented") { - // If the method is not supported, we set it to pd mode. + // If the method is not supported, we fallback to non-microservice env. // TODO: it's a hack way to solve the compatibility issue. // we need to remove this after all maintained version supports the method. c.callbacks.onServiceModeUpdate(pdpb.ServiceMode_PD_SVC_MODE) diff --git a/pkg/keyspace/tso_keyspace_group.go b/pkg/keyspace/tso_keyspace_group.go index a6068ffc2bb..86e0c9e9cf1 100644 --- a/pkg/keyspace/tso_keyspace_group.go +++ b/pkg/keyspace/tso_keyspace_group.go @@ -374,7 +374,6 @@ func (m *GroupManager) saveKeyspaceGroups(keyspaceGroups []*endpoint.KeyspaceGro // GetKeyspaceConfigByKind returns the keyspace config for the given user kind. func (m *GroupManager) GetKeyspaceConfigByKind(userKind endpoint.UserKind) (map[string]string, error) { - // when server is not in API mode, we don't need to return the keyspace config if m == nil { return map[string]string{}, nil } @@ -418,7 +417,6 @@ var failpointOnce sync.Once // UpdateKeyspaceForGroup updates the keyspace field for the keyspace group. func (m *GroupManager) UpdateKeyspaceForGroup(userKind endpoint.UserKind, groupID string, keyspaceID uint32, mutation int) error { - // when server is not in API mode, we don't need to update the keyspace for keyspace group if m == nil { return nil } @@ -477,7 +475,6 @@ func (m *GroupManager) updateKeyspaceForGroupLocked(userKind endpoint.UserKind, // UpdateKeyspaceGroup updates the keyspace group. func (m *GroupManager) UpdateKeyspaceGroup(oldGroupID, newGroupID string, oldUserKind, newUserKind endpoint.UserKind, keyspaceID uint32) error { - // when server is not in API mode, we don't need to update the keyspace group if m == nil { return nil } diff --git a/pkg/tso/global_allocator.go b/pkg/tso/global_allocator.go index 553e0b0effd..ea91bcdfadc 100644 --- a/pkg/tso/global_allocator.go +++ b/pkg/tso/global_allocator.go @@ -171,7 +171,7 @@ func (gta *GlobalTSOAllocator) Reset() { } // primaryElectionLoop is used to maintain the TSO primary election and TSO's -// running allocator. It is only used in API mode. +// running allocator. It is only used in microservice env. func (gta *GlobalTSOAllocator) primaryElectionLoop() { defer logutil.LogPanic() defer gta.wg.Done() diff --git a/pkg/utils/keypath/key_path.go b/pkg/utils/keypath/key_path.go index 1a56ad3330a..13dfe16d907 100644 --- a/pkg/utils/keypath/key_path.go +++ b/pkg/utils/keypath/key_path.go @@ -321,7 +321,7 @@ func svcRootPath(svcName string) string { return path.Join(constant.MicroserviceRootPath, c, svcName) } -// LegacyRootPath returns the root path of legacy pd service. +// LegacyRootPath returns the root path of legacy PD. // Path: /pd/{cluster_id} func LegacyRootPath() string { return path.Join(pdRootPath, strconv.FormatUint(ClusterID(), 10)) diff --git a/server/apiv2/handlers/micro_service.go b/server/apiv2/handlers/microservice.go similarity index 100% rename from server/apiv2/handlers/micro_service.go rename to server/apiv2/handlers/microservice.go diff --git a/server/cluster/cluster.go b/server/cluster/cluster.go index 4a712e13698..35d3d4ad4f8 100644 --- a/server/cluster/cluster.go +++ b/server/cluster/cluster.go @@ -1180,7 +1180,7 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio // Due to some config changes need to update the region stats as well, // so we do some extra checks here. // TODO: Due to the accuracy requirements of the API "/regions/check/xxx", - // region stats needs to be collected in API mode. + // region stats needs to be collected in microservice env. // We need to think of a better way to reduce this part of the cost in the future. if hasRegionStats && c.regionStats.RegionStatsNeedUpdate(region) { ctx.MiscRunner.RunTask( @@ -1251,7 +1251,7 @@ func (c *RaftCluster) processRegionHeartbeat(ctx *core.MetaProcessContext, regio ratelimit.CollectRegionStatsAsync, func(ctx context.Context) { // TODO: Due to the accuracy requirements of the API "/regions/check/xxx", - // region stats needs to be collected in API mode. + // region stats needs to be collected in microservice env. // We need to think of a better way to reduce this part of the cost in the future. cluster.Collect(ctx, c, region) }, diff --git a/server/config/service_middleware_config.go b/server/config/service_middleware_config.go index 223e19dba12..c2fc59c2a82 100644 --- a/server/config/service_middleware_config.go +++ b/server/config/service_middleware_config.go @@ -22,7 +22,7 @@ const ( defaultEnableGRPCRateLimitMiddleware = true ) -// ServiceMiddlewareConfig is the configuration for PD Service middleware. +// ServiceMiddlewareConfig is the configuration for PD middleware. type ServiceMiddlewareConfig struct { AuditConfig `json:"audit"` RateLimitConfig `json:"rate-limit"` diff --git a/server/config/service_middleware_persist_options.go b/server/config/service_middleware_persist_options.go index d59d23146f7..dfb266581bc 100644 --- a/server/config/service_middleware_persist_options.go +++ b/server/config/service_middleware_persist_options.go @@ -40,7 +40,7 @@ func NewServiceMiddlewarePersistOptions(cfg *ServiceMiddlewareConfig) *ServiceMi return o } -// GetAuditConfig returns pd service middleware configurations. +// GetAuditConfig returns PD middleware configurations. func (o *ServiceMiddlewarePersistOptions) GetAuditConfig() *AuditConfig { return o.audit.Load().(*AuditConfig) } diff --git a/tests/integrations/mcs/tso/api_test.go b/tests/integrations/mcs/tso/api_test.go index 7f874a52eb2..7faf0996ca7 100644 --- a/tests/integrations/mcs/tso/api_test.go +++ b/tests/integrations/mcs/tso/api_test.go @@ -227,7 +227,7 @@ func TestForwardOnlyTSONoScheduling(t *testing.T) { testutil.StatusOK(re), testutil.StringContain(re, "Reset ts successfully"), testutil.WithHeader(re, apiutil.XForwardedToMicroserviceHeader, "true")) re.NoError(err) - // If close tso server, it should try forward to tso server, but return error in pd service mode. + // If close tso server, it should try forward to tso server, but return error in non-serverless env. ttc.Destroy() err = testutil.CheckPostJSON(tests.TestDialClient, fmt.Sprintf("%s/%s", urlPrefix, "admin/reset-ts"), input, testutil.Status(re, http.StatusInternalServerError), testutil.StringContain(re, "[PD:apiutil:ErrRedirect]redirect failed")) diff --git a/tests/integrations/tso/client_test.go b/tests/integrations/tso/client_test.go index 5cd74c705fc..77ed84ab5c9 100644 --- a/tests/integrations/tso/client_test.go +++ b/tests/integrations/tso/client_test.go @@ -555,7 +555,7 @@ func TestUpgradingPDAndTSOClusters(t *testing.T) { pdLeader := pdCluster.GetServer(leaderName) backendEndpoints := pdLeader.GetAddr() - // Create a pd client in PD mode to let the API leader to forward requests to the TSO cluster. + // Create a PD client in microservice env to let the PD leader to forward requests to the TSO cluster. re.NoError(failpoint.Enable("github.com/tikv/pd/client/servicediscovery/usePDServiceMode", "return(true)")) pdClient, err := pd.NewClientWithContext(context.Background(), caller.TestComponent, diff --git a/tools/pd-ctl/pdctl/command/config_command.go b/tools/pd-ctl/pdctl/command/config_command.go index f0912c07c53..b9ab43fddff 100644 --- a/tools/pd-ctl/pdctl/command/config_command.go +++ b/tools/pd-ctl/pdctl/command/config_command.go @@ -49,7 +49,7 @@ const ( ruleBundlePrefix = "pd/api/v1/config/placement-rule" pdServerPrefix = "pd/api/v1/config/pd-server" serviceMiddlewareConfigPrefix = "pd/api/v1/service-middleware/config" - // flagFromPD has no influence for pd mode, but it is useful for us to debug in pd service mode. + // flagFromPD is useful for us to debug. flagFromPD = "from_pd" ) diff --git a/tools/pd-ctl/pdctl/command/keyspace_command.go b/tools/pd-ctl/pdctl/command/keyspace_command.go index 2ecee481df4..4fd131a8ffe 100644 --- a/tools/pd-ctl/pdctl/command/keyspace_command.go +++ b/tools/pd-ctl/pdctl/command/keyspace_command.go @@ -101,7 +101,7 @@ func showKeyspaceNameCommandFunc(cmd *cobra.Command, args []string) { } resp, err := doRequest(cmd, url, http.MethodGet, http.Header{}) // Retry without the force_refresh_group_id if the keyspace group manager is not initialized. - // This can happen when PD is not running in API mode. + // This can happen when PD is not running in microservice env. if err != nil && refreshGroupID && strings.Contains(err.Error(), handlers.GroupManagerUninitializedErr) { resp, err = doRequest(cmd, fmt.Sprintf("%s/%s", keyspacePrefix, args[0]), http.MethodGet, http.Header{}) } diff --git a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go index 88ee782ab4b..58c6f5a5762 100644 --- a/tools/pd-ctl/tests/keyspace/keyspace_group_test.go +++ b/tools/pd-ctl/tests/keyspace/keyspace_group_test.go @@ -597,7 +597,7 @@ func TestShowKeyspaceGroupPrimary(t *testing.T) { re.NoError(failpoint.Disable("github.com/tikv/pd/server/delayStartServerLoop")) } -func TestInPDMode(t *testing.T) { +func TestCmdWithoutKeyspaceGroupInitialized(t *testing.T) { re := require.New(t) ctx, cancel := context.WithCancel(context.Background()) defer cancel() From 5a5c07efaa5af3f4c324897106338ffd397356b8 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 22 Jan 2025 16:38:23 +0800 Subject: [PATCH 26/33] client: fix backoffer initialization (#9012) ref tikv/pd#8835, close tikv/pd#9013 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/clients/tso/dispatcher.go | 2 +- client/servicediscovery/service_discovery.go | 9 ++++++--- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/client/clients/tso/dispatcher.go b/client/clients/tso/dispatcher.go index 1cc2b2aa940..6bdbbf653c5 100644 --- a/client/clients/tso/dispatcher.go +++ b/client/clients/tso/dispatcher.go @@ -175,7 +175,7 @@ func (td *tsoDispatcher) handleDispatcher(wg *sync.WaitGroup) { <-batchingTimer.C defer batchingTimer.Stop() - bo := retry.InitialBackoffer(sd.UpdateMemberBackOffBaseTime, sd.UpdateMemberTimeout, sd.UpdateMemberBackOffBaseTime) + bo := retry.InitialBackoffer(sd.UpdateMemberBackOffBaseTime, sd.UpdateMemberMaxBackoffTime, sd.UpdateMemberTimeout) tsoBatchLoop: for { select { diff --git a/client/servicediscovery/service_discovery.go b/client/servicediscovery/service_discovery.go index cf1b3372591..3657a5229c2 100644 --- a/client/servicediscovery/service_discovery.go +++ b/client/servicediscovery/service_discovery.go @@ -47,11 +47,14 @@ import ( const ( // MemberUpdateInterval is the interval to update the member list. MemberUpdateInterval = time.Minute + // UpdateMemberMaxBackoffTime is the max time to back off when updating the member list. + UpdateMemberMaxBackoffTime = 100 * time.Millisecond + // UpdateMemberBackOffBaseTime is the base time to back off when updating the member list. + // Here we use 20ms is because getting timestamp will print a warning log if the time exceeds 30ms. + UpdateMemberBackOffBaseTime = 20 * time.Millisecond // UpdateMemberTimeout is the timeout to update the member list. // Use a shorter timeout to recover faster from network isolation. UpdateMemberTimeout = time.Second - // UpdateMemberBackOffBaseTime is the base time to back off when updating the member list. - UpdateMemberBackOffBaseTime = 100 * time.Millisecond serviceModeUpdateInterval = 3 * time.Second ) @@ -533,7 +536,7 @@ func (c *serviceDiscovery) updateMemberLoop() { ticker := time.NewTicker(MemberUpdateInterval) defer ticker.Stop() - bo := retry.InitialBackoffer(UpdateMemberBackOffBaseTime, UpdateMemberTimeout, UpdateMemberBackOffBaseTime) + bo := retry.InitialBackoffer(UpdateMemberBackOffBaseTime, UpdateMemberMaxBackoffTime, UpdateMemberTimeout) for { select { case <-ctx.Done(): From 2a1f2dc98f514debea9f6712b9ff54e35c08cd2d Mon Sep 17 00:00:00 2001 From: Artem Danilov Date: Mon, 27 Jan 2025 00:21:59 -0800 Subject: [PATCH 27/33] client: fix pd client metrics registration (#8994) ref tikv/pd#8678 Signed-off-by: artem_danilov Co-authored-by: artem_danilov Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/metrics/metrics.go | 24 ++++++++++++++++++++ client/pkg/circuitbreaker/circuit_breaker.go | 10 ++++++-- 2 files changed, 32 insertions(+), 2 deletions(-) diff --git a/client/metrics/metrics.go b/client/metrics/metrics.go index 67268c826f5..da7637b19be 100644 --- a/client/metrics/metrics.go +++ b/client/metrics/metrics.go @@ -15,6 +15,7 @@ package metrics import ( + "sync" "sync/atomic" "github.com/prometheus/client_golang/prometheus" @@ -26,6 +27,28 @@ var initialized int32 func init() { initMetrics(prometheus.Labels{}) initCmdDurations() + initRegisteredConsumers() +} + +var consumersInitializers = struct { + sync.Mutex + value []func() +}{} + +// RegisterConsumer registers a consumer to be initialized when the metrics are (re)initialized +func RegisterConsumer(initConsumer func()) { + consumersInitializers.Lock() + defer consumersInitializers.Unlock() + consumersInitializers.value = append(consumersInitializers.value, initConsumer) + initConsumer() +} + +func initRegisteredConsumers() { + consumersInitializers.Lock() + defer consumersInitializers.Unlock() + for _, initConsumer := range consumersInitializers.value { + initConsumer() + } } // InitAndRegisterMetrics initializes and registers the metrics manually. @@ -34,6 +57,7 @@ func InitAndRegisterMetrics(constLabels prometheus.Labels) { // init metrics with constLabels initMetrics(constLabels) initCmdDurations() + initRegisteredConsumers() // register metrics registerMetrics() } diff --git a/client/pkg/circuitbreaker/circuit_breaker.go b/client/pkg/circuitbreaker/circuit_breaker.go index 7466124f2bc..4621dfe4bc0 100644 --- a/client/pkg/circuitbreaker/circuit_breaker.go +++ b/client/pkg/circuitbreaker/circuit_breaker.go @@ -110,12 +110,18 @@ func NewCircuitBreaker(name string, st Settings) *CircuitBreaker { cb.config = &st cb.state = cb.newState(time.Now(), StateClosed) - metricName := replacer.Replace(name) + m.RegisterConsumer(func() { + registerMetrics(cb) + }) + return cb +} + +func registerMetrics(cb *CircuitBreaker) { + metricName := replacer.Replace(cb.name) cb.successCounter = m.CircuitBreakerCounters.WithLabelValues(metricName, "success") cb.errorCounter = m.CircuitBreakerCounters.WithLabelValues(metricName, "error") cb.overloadCounter = m.CircuitBreakerCounters.WithLabelValues(metricName, "overload") cb.fastFailCounter = m.CircuitBreakerCounters.WithLabelValues(metricName, "fast_fail") - return cb } // ChangeSettings changes the CircuitBreaker settings. From debceaf3c78350ab37b8cbc6bcf3b3470cb60b8d Mon Sep 17 00:00:00 2001 From: JmPotato Date: Mon, 27 Jan 2025 20:49:59 +0800 Subject: [PATCH 28/33] client/router: implement the query region gRPC client (#8939) ref tikv/pd#8690 Implement the router stream update logic. Signed-off-by: JmPotato Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/client.go | 46 +-- client/clients/router/client.go | 434 +++++++++++++++++++++++ client/clients/router/request.go | 114 ++++++ client/go.mod | 2 +- client/go.sum | 4 +- client/inner_client.go | 10 + client/pkg/connectionctx/manager.go | 3 +- tests/integrations/client/client_test.go | 83 ++++- 8 files changed, 670 insertions(+), 26 deletions(-) create mode 100644 client/clients/router/request.go diff --git a/client/client.go b/client/client.go index fa0a1473ba7..e5f6442780b 100644 --- a/client/client.go +++ b/client/client.go @@ -143,12 +143,11 @@ var _ Client = (*client)(nil) // serviceModeKeeper is for service mode switching. type serviceModeKeeper struct { - // RMutex here is for the future usage that there might be multiple goroutines - // triggering service mode switching concurrently. sync.RWMutex serviceMode pdpb.ServiceMode tsoClient *tso.Cli tsoSvcDiscovery sd.ServiceDiscovery + routerClient *router.Cli } func (k *serviceModeKeeper) close() { @@ -570,21 +569,16 @@ func (c *client) GetMinTS(ctx context.Context) (physical int64, logical int64, e return minTS.Physical, minTS.Logical, nil } -func handleRegionResponse(res *pdpb.GetRegionResponse) *router.Region { - if res.Region == nil { - return nil - } +// EnableRouterClient enables the router client. +// This is only for test currently. +func (c *client) EnableRouterClient() { + c.inner.initRouterClient() +} - r := &router.Region{ - Meta: res.Region, - Leader: res.Leader, - PendingPeers: res.PendingPeers, - Buckets: res.Buckets, - } - for _, s := range res.DownPeers { - r.DownPeers = append(r.DownPeers, s.Peer) - } - return r +func (c *client) getRouterClient() *router.Cli { + c.inner.RLock() + defer c.inner.RUnlock() + return c.inner.routerClient } // GetRegionFromMember implements the RPCClient interface. @@ -623,7 +617,7 @@ func (c *client) GetRegionFromMember(ctx context.Context, key []byte, memberURLs errorMsg := fmt.Sprintf("[pd] can't get region info from member URLs: %+v", memberURLs) return nil, errors.WithStack(errors.New(errorMsg)) } - return handleRegionResponse(resp), nil + return router.ConvertToRegion(resp), nil } // GetRegion implements the RPCClient interface. @@ -637,6 +631,10 @@ func (c *client) GetRegion(ctx context.Context, key []byte, opts ...opt.GetRegio ctx, cancel := context.WithTimeout(ctx, c.inner.option.Timeout) defer cancel() + if routerClient := c.getRouterClient(); routerClient != nil { + return routerClient.GetRegion(ctx, key, opts...) + } + options := &opt.GetRegionOp{} for _, opt := range opts { opt(options) @@ -663,7 +661,7 @@ func (c *client) GetRegion(ctx context.Context, key []byte, opts ...opt.GetRegio if err = c.respForErr(metrics.CmdFailedDurationGetRegion, start, err, resp.GetHeader()); err != nil { return nil, err } - return handleRegionResponse(resp), nil + return router.ConvertToRegion(resp), nil } // GetPrevRegion implements the RPCClient interface. @@ -677,6 +675,10 @@ func (c *client) GetPrevRegion(ctx context.Context, key []byte, opts ...opt.GetR ctx, cancel := context.WithTimeout(ctx, c.inner.option.Timeout) defer cancel() + if routerClient := c.getRouterClient(); routerClient != nil { + return routerClient.GetPrevRegion(ctx, key, opts...) + } + options := &opt.GetRegionOp{} for _, opt := range opts { opt(options) @@ -703,7 +705,7 @@ func (c *client) GetPrevRegion(ctx context.Context, key []byte, opts ...opt.GetR if err = c.respForErr(metrics.CmdFailedDurationGetPrevRegion, start, err, resp.GetHeader()); err != nil { return nil, err } - return handleRegionResponse(resp), nil + return router.ConvertToRegion(resp), nil } // GetRegionByID implements the RPCClient interface. @@ -717,6 +719,10 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64, opts ...opt ctx, cancel := context.WithTimeout(ctx, c.inner.option.Timeout) defer cancel() + if routerClient := c.getRouterClient(); routerClient != nil { + return routerClient.GetRegionByID(ctx, regionID, opts...) + } + options := &opt.GetRegionOp{} for _, opt := range opts { opt(options) @@ -744,7 +750,7 @@ func (c *client) GetRegionByID(ctx context.Context, regionID uint64, opts ...opt if err = c.respForErr(metrics.CmdFailedDurationGetRegionByID, start, err, resp.GetHeader()); err != nil { return nil, err } - return handleRegionResponse(resp), nil + return router.ConvertToRegion(resp), nil } // ScanRegions implements the RPCClient interface. diff --git a/client/clients/router/client.go b/client/clients/router/client.go index 48cebfa950e..0038c42dff8 100644 --- a/client/clients/router/client.go +++ b/client/clients/router/client.go @@ -18,12 +18,30 @@ import ( "context" "encoding/hex" "net/url" + "runtime/trace" + "sync" + "sync/atomic" + "time" + + "github.com/opentracing/opentracing-go" + "go.uber.org/zap" + "google.golang.org/grpc" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/kvproto/pkg/pdpb" + "github.com/pingcap/log" + "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/opt" + "github.com/tikv/pd/client/pkg/batch" + cctx "github.com/tikv/pd/client/pkg/connectionctx" + "github.com/tikv/pd/client/pkg/retry" + sd "github.com/tikv/pd/client/servicediscovery" ) +// defaultMaxRouterRequestBatchSize is the default max size of the router request batch. +const defaultMaxRouterRequestBatchSize = 10000 + // Region contains information of a region's meta and its peers. type Region struct { Meta *metapb.Region @@ -33,6 +51,33 @@ type Region struct { Buckets *metapb.Buckets } +type regionResponse interface { + GetRegion() *metapb.Region + GetLeader() *metapb.Peer + GetDownPeers() []*pdpb.PeerStats + GetPendingPeers() []*metapb.Peer + GetBuckets() *metapb.Buckets +} + +// ConvertToRegion converts the region response to the region. +func ConvertToRegion(res regionResponse) *Region { + region := res.GetRegion() + if region == nil { + return nil + } + + r := &Region{ + Meta: region, + Leader: res.GetLeader(), + PendingPeers: res.GetPendingPeers(), + Buckets: res.GetBuckets(), + } + for _, s := range res.GetDownPeers() { + r.DownPeers = append(r.DownPeers, s.Peer) + } + return r +} + // KeyRange defines a range of keys in bytes. type KeyRange struct { StartKey []byte @@ -92,3 +137,392 @@ type Client interface { // The returned regions are flattened, even there are key ranges located in the same region, only one region will be returned. BatchScanRegions(ctx context.Context, keyRanges []KeyRange, limit int, opts ...opt.GetRegionOption) ([]*Region, error) } + +// Cli is the implementation of the router client. +type Cli struct { + ctx context.Context + cancel context.CancelFunc + wg sync.WaitGroup + option *opt.Option + + svcDiscovery sd.ServiceDiscovery + // leaderURL is the URL of the router leader. + leaderURL atomic.Value + // conCtxMgr is used to store the context of the router stream connection(s). + conCtxMgr *cctx.Manager[pdpb.PD_QueryRegionClient] + // updateConnectionCh is used to trigger the connection update actively. + updateConnectionCh chan struct{} + // bo is the backoffer for the router client. + bo *retry.Backoffer + + reqPool *sync.Pool + requestCh chan *Request + batchController *batch.Controller[*Request] +} + +// NewClient returns a new router client. +func NewClient( + ctx context.Context, + svcDiscovery sd.ServiceDiscovery, + option *opt.Option, +) *Cli { + ctx, cancel := context.WithCancel(ctx) + c := &Cli{ + ctx: ctx, + cancel: cancel, + svcDiscovery: svcDiscovery, + option: option, + conCtxMgr: cctx.NewManager[pdpb.PD_QueryRegionClient](), + updateConnectionCh: make(chan struct{}, 1), + bo: retry.InitialBackoffer( + sd.UpdateMemberBackOffBaseTime, + sd.UpdateMemberMaxBackoffTime, + sd.UpdateMemberTimeout, + ), + reqPool: &sync.Pool{ + New: func() any { + return &Request{ + done: make(chan error, 1), + } + }, + }, + requestCh: make(chan *Request, defaultMaxRouterRequestBatchSize*2), + batchController: batch.NewController(defaultMaxRouterRequestBatchSize, requestFinisher(nil), nil), + } + c.leaderURL.Store(svcDiscovery.GetServingURL()) + c.svcDiscovery.ExecAndAddLeaderSwitchedCallback(c.updateLeaderURL) + c.svcDiscovery.AddMembersChangedCallback(c.scheduleUpdateConnection) + + c.wg.Add(2) + go c.connectionDaemon() + go c.dispatcher() + + return c +} + +func (c *Cli) newRequest(ctx context.Context) *Request { + req := c.reqPool.Get().(*Request) + req.requestCtx = ctx + req.clientCtx = c.ctx + req.pool = c.reqPool + + return req +} + +func requestFinisher(resp *pdpb.QueryRegionResponse) batch.FinisherFunc[*Request] { + var keyIdx, prevKeyIdx int + return func(_ int, req *Request, err error) { + requestCtx := req.requestCtx + defer trace.StartRegion(requestCtx, "pdclient.regionReqDone").End() + + if err != nil { + req.tryDone(err) + return + } + + var id uint64 + if req.key != nil { + id = resp.KeyIdMap[keyIdx] + keyIdx++ + } else if req.prevKey != nil { + id = resp.PrevKeyIdMap[prevKeyIdx] + prevKeyIdx++ + } else if req.id != 0 { + id = req.id + } + if region, ok := resp.RegionsById[id]; ok { + req.region = ConvertToRegion(region) + } + req.tryDone(err) + } +} + +func (c *Cli) cancelCollectedRequests(err error) { + c.batchController.FinishCollectedRequests(requestFinisher(nil), err) +} + +func (c *Cli) doneCollectedRequests(resp *pdpb.QueryRegionResponse) { + c.batchController.FinishCollectedRequests(requestFinisher(resp), nil) +} + +// Close closes the router client. +func (c *Cli) Close() { + if c == nil { + return + } + log.Info("[router] closing router client") + + c.cancel() + c.wg.Wait() + + log.Info("[router] router client is closed") +} + +func (c *Cli) getLeaderURL() string { + url := c.leaderURL.Load() + if url == nil { + return "" + } + return url.(string) +} + +func (c *Cli) updateLeaderURL(url string) error { + oldURL := c.getLeaderURL() + if oldURL == url { + return nil + } + c.leaderURL.Store(url) + c.scheduleUpdateConnection() + + log.Info("[router] switch the router leader serving url", + zap.String("old-url", oldURL), zap.String("new-url", url)) + return nil +} + +// getLeaderClientConn returns the leader gRPC client connection. +func (c *Cli) getLeaderClientConn() (*grpc.ClientConn, string) { + url := c.getLeaderURL() + if len(url) == 0 { + c.svcDiscovery.ScheduleCheckMemberChanged() + return nil, "" + } + cc, ok := c.svcDiscovery.GetClientConns().Load(url) + if !ok { + return nil, url + } + return cc.(*grpc.ClientConn), url +} + +// scheduleUpdateConnection is used to schedule an update to the connection(s). +func (c *Cli) scheduleUpdateConnection() { + select { + case c.updateConnectionCh <- struct{}{}: + default: + } +} + +// connectionDaemon is used to update the router leader/primary/backup connection(s) in background. +// It aims to provide a seamless connection updating for the router client to keep providing the +// router service without interruption. +func (c *Cli) connectionDaemon() { + defer c.wg.Done() + updaterCtx, updaterCancel := context.WithCancel(c.ctx) + defer updaterCancel() + updateTicker := time.NewTicker(sd.MemberUpdateInterval) + defer updateTicker.Stop() + + log.Info("[router] connection daemon is started") + for { + c.updateConnection(updaterCtx) + select { + case <-updaterCtx.Done(): + log.Info("[router] connection daemon is exiting") + return + case <-updateTicker.C: + case <-c.updateConnectionCh: + } + } +} + +// updateConnection is used to get the leader client connection and update the connection context if it does not exist before. +func (c *Cli) updateConnection(ctx context.Context) { + cc, url := c.getLeaderClientConn() + if cc == nil || len(url) == 0 { + log.Warn("[router] got an invalid leader client connection", zap.String("url", url)) + return + } + if c.conCtxMgr.Exist(url) { + log.Debug("[router] the router leader remains unchanged", zap.String("url", url)) + return + } + stream, err := pdpb.NewPDClient(cc).QueryRegion(ctx) + if err != nil { + log.Error("[router] failed to create the router stream connection", errs.ZapError(err)) + } + c.conCtxMgr.Store(ctx, url, stream) + // TODO: support the forwarding mechanism for the router client. + // TODO: support sending the router requests to the follower nodes. +} + +func (c *Cli) dispatcher() { + defer c.wg.Done() + + var ( + stream pdpb.PD_QueryRegionClient + streamURL string + streamCtx context.Context + timeoutTimer *time.Timer + resetTimeoutTimer = func() { + if timeoutTimer == nil { + timeoutTimer = time.NewTimer(c.option.Timeout) + } else { + timeoutTimer.Reset(c.option.Timeout) + } + } + ctx, cancel = context.WithCancel(c.ctx) + ) + + log.Info("[router] dispatcher is started") + defer func() { + log.Info("[router] dispatcher is exiting") + cancel() + if timeoutTimer != nil { + timeoutTimer.Stop() + } + log.Info("[router] dispatcher exited") + }() +batchLoop: + for { + select { + case <-ctx.Done(): + return + default: + } + + // Step 1: Fetch the pending router requests in batch. + err := c.batchController.FetchPendingRequests(ctx, c.requestCh, nil, 0) + if err != nil { + if err == context.Canceled { + log.Info("[router] stop fetching the pending router requests due to context canceled") + } else { + log.Error("[router] failed to fetch the pending router requests", errs.ZapError(err)) + } + return + } + + // Step 2: Choose a stream connection to send the router request. + resetTimeoutTimer() + connectionCtxChoosingLoop: + for { + // Check if the dispatcher is canceled or the timeout timer is triggered. + select { + case <-ctx.Done(): + return + case <-timeoutTimer.C: + log.Error("[router] router stream connection is not ready until timeout, abort the batch") + c.svcDiscovery.ScheduleCheckMemberChanged() + c.batchController.FinishCollectedRequests(requestFinisher(nil), err) + continue batchLoop + default: + } + // Choose a stream connection to send the router request later. + connectionCtx := c.conCtxMgr.GetConnectionCtx() + if connectionCtx == nil { + log.Info("[router] router stream connection is not ready") + c.updateConnection(ctx) + continue connectionCtxChoosingLoop + } + streamCtx, streamURL, stream = connectionCtx.Ctx, connectionCtx.StreamURL, connectionCtx.Stream + // Check if the stream connection is canceled. + select { + case <-streamCtx.Done(): + log.Info("[router] router stream connection is canceled", zap.String("stream-url", streamURL)) + c.conCtxMgr.Release(streamURL) + continue connectionCtxChoosingLoop + default: + } + // The stream connection is ready, break the loop. + break connectionCtxChoosingLoop + } + + // Step 3: Dispatch the router requests to the stream connection. + // TODO: timeout handling if the stream takes too long to process the requests. + err = c.processRequests(stream) + if err != nil { + if !c.handleProcessRequestError(ctx, streamURL, err) { + return + } + } + } +} + +func (c *Cli) processRequests(stream pdpb.PD_QueryRegionClient) error { + var ( + requests = c.batchController.GetCollectedRequests() + traceRegions = make([]*trace.Region, 0, len(requests)) + spans = make([]opentracing.Span, 0, len(requests)) + ) + for _, req := range requests { + traceRegions = append(traceRegions, trace.StartRegion(req.requestCtx, "pdclient.regionReqSend")) + if span := opentracing.SpanFromContext(req.requestCtx); span != nil && span.Tracer() != nil { + spans = append(spans, span.Tracer().StartSpan("pdclient.processRegionRequests", opentracing.ChildOf(span.Context()))) + } + } + defer func() { + for i := range spans { + spans[i].Finish() + } + for i := range traceRegions { + traceRegions[i].End() + } + }() + + queryReq := &pdpb.QueryRegionRequest{ + Header: &pdpb.RequestHeader{ + ClusterId: c.svcDiscovery.GetClusterID(), + }, + Keys: make([][]byte, 0, len(requests)), + PrevKeys: make([][]byte, 0, len(requests)), + Ids: make([]uint64, 0, len(requests)), + } + for _, req := range requests { + if !queryReq.NeedBuckets && req.needBuckets { + queryReq.NeedBuckets = true + } + if req.key != nil { + queryReq.Keys = append(queryReq.Keys, req.key) + } else if req.prevKey != nil { + queryReq.PrevKeys = append(queryReq.PrevKeys, req.prevKey) + } else if req.id != 0 { + queryReq.Ids = append(queryReq.Ids, req.id) + } else { + panic("invalid region query request received") + } + } + err := stream.Send(queryReq) + if err != nil { + return err + } + resp, err := stream.Recv() + if err != nil { + return err + } + c.doneCollectedRequests(resp) + return nil +} + +func (c *Cli) handleProcessRequestError( + ctx context.Context, + streamURL string, + err error, +) bool { + log.Error("[router] failed to process the router requests", + zap.String("stream-url", streamURL), + errs.ZapError(err)) + c.cancelCollectedRequests(err) + + select { + case <-ctx.Done(): + return false + default: + } + + // Delete the stream connection context. + c.conCtxMgr.Release(streamURL) + if errs.IsLeaderChange(err) { + // If the leader changes, we better call `CheckMemberChanged` blockingly to + // ensure the next round of router requests can be sent to the new leader. + if err := c.bo.Exec(ctx, c.svcDiscovery.CheckMemberChanged); err != nil { + select { + case <-ctx.Done(): + return false + default: + } + } + } else { + // For other errors, we can just schedule a member change check asynchronously. + c.svcDiscovery.ScheduleCheckMemberChanged() + } + + return true +} diff --git a/client/clients/router/request.go b/client/clients/router/request.go new file mode 100644 index 00000000000..4578514597d --- /dev/null +++ b/client/clients/router/request.go @@ -0,0 +1,114 @@ +// Copyright 2024 TiKV Project Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package router + +import ( + "context" + "runtime/trace" + "sync" + + "github.com/pingcap/errors" + + "github.com/tikv/pd/client/opt" +) + +// Request is a region info request. +type Request struct { + requestCtx context.Context + clientCtx context.Context + + // Key field represents this is a `GetRegion` request. + key []byte + // PrevKey field represents this is a `GetPrevRegion` request. + prevKey []byte + // ID field represents this is a `GetRegionByID` request. + id uint64 + + // NeedBuckets field represents whether the request needs to get the region buckets. + needBuckets bool + + done chan error + // region will be set after the request is done. + region *Region + + // Runtime fields. + pool *sync.Pool +} + +func (req *Request) tryDone(err error) { + select { + case req.done <- err: + default: + } +} + +func (req *Request) wait() (*Region, error) { + // TODO: introduce the metrics. + select { + case err := <-req.done: + defer req.pool.Put(req) + defer trace.StartRegion(req.requestCtx, "pdclient.regionReqDone").End() + if err != nil { + return nil, errors.WithStack(err) + } + return req.region, nil + case <-req.requestCtx.Done(): + return nil, errors.WithStack(req.requestCtx.Err()) + case <-req.clientCtx.Done(): + return nil, errors.WithStack(req.clientCtx.Err()) + } +} + +// GetRegion implements the Client interface. +func (c *Cli) GetRegion(ctx context.Context, key []byte, opts ...opt.GetRegionOption) (*Region, error) { + req := c.newRequest(ctx) + req.key = key + options := &opt.GetRegionOp{} + for _, opt := range opts { + opt(options) + } + req.needBuckets = options.NeedBuckets + + c.requestCh <- req + return req.wait() +} + +// GetPrevRegion implements the Client interface. +func (c *Cli) GetPrevRegion(ctx context.Context, key []byte, opts ...opt.GetRegionOption) (*Region, error) { + req := c.newRequest(ctx) + req.prevKey = key + options := &opt.GetRegionOp{} + for _, opt := range opts { + opt(options) + } + req.needBuckets = options.NeedBuckets + + c.requestCh <- req + return req.wait() +} + +// GetRegionByID implements the Client interface. +func (c *Cli) GetRegionByID(ctx context.Context, regionID uint64, opts ...opt.GetRegionOption) (*Region, error) { + req := c.newRequest(ctx) + req.id = regionID + options := &opt.GetRegionOp{} + for _, opt := range opts { + opt(options) + } + req.needBuckets = options.NeedBuckets + + c.requestCh <- req + return req.wait() +} diff --git a/client/go.mod b/client/go.mod index 78aef084ff7..a84bf303be1 100644 --- a/client/go.mod +++ b/client/go.mod @@ -10,7 +10,7 @@ require ( github.com/opentracing/opentracing-go v1.2.0 github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 - github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 + github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 github.com/prometheus/client_golang v1.20.5 github.com/stretchr/testify v1.9.0 diff --git a/client/go.sum b/client/go.sum index 4cca5ba3ad5..2873e4f550c 100644 --- a/client/go.sum +++ b/client/go.sum @@ -49,8 +49,8 @@ github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c h1:xpW9bvK+HuuTm github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c/go.mod h1:X2r9ueLEUZgtx2cIogM0v4Zj5uvvzhuuiu7Pn8HzMPg= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86 h1:tdMsjOqUR7YXHoBitzdebTvOjs/swniBTOLy5XiMtuE= github.com/pingcap/failpoint v0.0.0-20240528011301-b51a646c7c86/go.mod h1:exzhVYca3WRtd6gclGNErRWb1qEgff3LYta0LvRmON4= -github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037 h1:xYNSJjYNur4Dr5bV+9BXK9n5E0T1zlcAN25XX68+mOg= -github.com/pingcap/kvproto v0.0.0-20241120071417-b5b7843d9037/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= +github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1 h1:rTAyiswGyWSGHJVa4Mkhdi8YfGqfA4LrUVKsH9nrJ8E= +github.com/pingcap/kvproto v0.0.0-20250117122752-2b87602a94a1/go.mod h1:rXxWk2UnwfUhLXha1jxRWPADw9eMZGWEWCg92Tgmb/8= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8IDP+SZrdhV1Kibl9KrHxJ9eciw= github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/client/inner_client.go b/client/inner_client.go index 7ce8f42386e..181ee2c9d52 100644 --- a/client/inner_client.go +++ b/client/inner_client.go @@ -27,6 +27,7 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/pingcap/log" + "github.com/tikv/pd/client/clients/router" "github.com/tikv/pd/client/clients/tso" "github.com/tikv/pd/client/errs" "github.com/tikv/pd/client/metrics" @@ -73,6 +74,15 @@ func (c *innerClient) init(updateKeyspaceIDCb sd.UpdateKeyspaceIDFunc) error { return nil } +func (c *innerClient) initRouterClient() { + c.Lock() + defer c.Unlock() + if c.routerClient != nil { + return + } + c.routerClient = router.NewClient(c.ctx, c.serviceDiscovery, c.option) +} + func (c *innerClient) setServiceMode(newMode pdpb.ServiceMode) { c.Lock() defer c.Unlock() diff --git a/client/pkg/connectionctx/manager.go b/client/pkg/connectionctx/manager.go index 04c1eb13d3a..fede8baf723 100644 --- a/client/pkg/connectionctx/manager.go +++ b/client/pkg/connectionctx/manager.go @@ -16,9 +16,8 @@ package connectionctx import ( "context" + "math/rand" "sync" - - "golang.org/x/exp/rand" ) type connectionCtx[T any] struct { diff --git a/tests/integrations/client/client_test.go b/tests/integrations/client/client_test.go index af8cdc00a7e..0f972cb2b8f 100644 --- a/tests/integrations/client/client_test.go +++ b/tests/integrations/client/client_test.go @@ -20,6 +20,7 @@ import ( "encoding/json" "fmt" "math" + "math/rand" "os" "path" "reflect" @@ -1105,6 +1106,10 @@ func bootstrapServer(re *require.Assertions, header *pdpb.RequestHeader, client re.Equal(pdpb.ErrorType_OK, resp.GetHeader().GetError().GetType()) } +func (suite *clientTestSuite) SetupTest() { + suite.grpcSvr.DirectlyGetRaftCluster().ResetRegionCache() +} + func (suite *clientTestSuite) TestGetRegion() { re := suite.Require() regionID := regionIDAllocator.alloc() @@ -1204,7 +1209,6 @@ func (suite *clientTestSuite) TestGetPrevRegion() { err := suite.regionHeartbeat.Send(req) re.NoError(err) } - time.Sleep(500 * time.Millisecond) for i := range 20 { testutil.Eventually(re, func() bool { r, err := suite.client.GetPrevRegion(context.Background(), []byte{byte(i)}) @@ -1338,6 +1342,83 @@ func (suite *clientTestSuite) TestGetRegionByID() { }) } +func (suite *clientTestSuite) TestGetRegionConcurrently() { + suite.client.(interface{ EnableRouterClient() }).EnableRouterClient() + + re := suite.Require() + ctx, cancel := context.WithCancel(suite.ctx) + defer cancel() + + regions := make([]*metapb.Region, 0, 2) + for i := range 2 { + regionID := regionIDAllocator.alloc() + region := &metapb.Region{ + Id: regionID, + RegionEpoch: &metapb.RegionEpoch{ + ConfVer: 1, + Version: 1, + }, + StartKey: []byte{byte(i)}, + EndKey: []byte{byte(i + 1)}, + Peers: peers, + } + re.NoError(suite.regionHeartbeat.Send(&pdpb.RegionHeartbeatRequest{ + Header: newHeader(), + Region: region, + Leader: peers[0], + })) + regions = append(regions, region) + } + + const concurrency = 1000 + + wg := sync.WaitGroup{} + wg.Add(concurrency) + for range concurrency { + go func() { + defer wg.Done() + switch rand.Intn(3) { + case 0: + region := regions[0] + testutil.Eventually(re, func() bool { + r, err := suite.client.GetRegion(ctx, region.GetStartKey()) + re.NoError(err) + if r == nil { + return false + } + return reflect.DeepEqual(region, r.Meta) && + reflect.DeepEqual(peers[0], r.Leader) && + r.Buckets == nil + }) + case 1: + testutil.Eventually(re, func() bool { + r, err := suite.client.GetPrevRegion(ctx, regions[1].GetStartKey()) + re.NoError(err) + if r == nil { + return false + } + return reflect.DeepEqual(regions[0], r.Meta) && + reflect.DeepEqual(peers[0], r.Leader) && + r.Buckets == nil + }) + case 2: + region := regions[0] + testutil.Eventually(re, func() bool { + r, err := suite.client.GetRegionByID(ctx, region.GetId()) + re.NoError(err) + if r == nil { + return false + } + return reflect.DeepEqual(region, r.Meta) && + reflect.DeepEqual(peers[0], r.Leader) && + r.Buckets == nil + }) + } + }() + } + wg.Wait() +} + func (suite *clientTestSuite) TestGetStore() { re := suite.Require() cluster := suite.srv.GetRaftCluster() From f6d4ca57efc2cea024aeabf6a9cccfecc22d8870 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Wed, 5 Feb 2025 18:45:03 +0800 Subject: [PATCH 29/33] client: expose callback function (#9032) ref tikv/pd#8690 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/servicediscovery/callbacks.go | 11 ++++++----- client/servicediscovery/mock_service_discovery.go | 4 ++-- client/servicediscovery/service_discovery.go | 8 ++++---- client/servicediscovery/tso_service_discovery.go | 6 +++--- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/client/servicediscovery/callbacks.go b/client/servicediscovery/callbacks.go index c21217f7e44..ea25fcb7de5 100644 --- a/client/servicediscovery/callbacks.go +++ b/client/servicediscovery/callbacks.go @@ -20,7 +20,8 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" ) -type leaderSwitchedCallbackFunc func(string) error +// LeaderSwitchedCallbackFunc is the callback function for leader switched event +type LeaderSwitchedCallbackFunc func(string) error // serviceCallbacks contains all the callback functions for service discovery events type serviceCallbacks struct { @@ -28,7 +29,7 @@ type serviceCallbacks struct { // serviceModeUpdateCb will be called when the service mode gets updated serviceModeUpdateCb func(pdpb.ServiceMode) // leaderSwitchedCbs will be called after the leader switched - leaderSwitchedCbs []leaderSwitchedCallbackFunc + leaderSwitchedCbs []LeaderSwitchedCallbackFunc // membersChangedCbs will be called after there is any membership change in the // leader and followers membersChangedCbs []func() @@ -36,7 +37,7 @@ type serviceCallbacks struct { func newServiceCallbacks() *serviceCallbacks { return &serviceCallbacks{ - leaderSwitchedCbs: make([]leaderSwitchedCallbackFunc, 0), + leaderSwitchedCbs: make([]LeaderSwitchedCallbackFunc, 0), membersChangedCbs: make([]func(), 0), } } @@ -47,7 +48,7 @@ func (c *serviceCallbacks) setServiceModeUpdateCallback(cb func(pdpb.ServiceMode c.serviceModeUpdateCb = cb } -func (c *serviceCallbacks) addLeaderSwitchedCallback(cb leaderSwitchedCallbackFunc) { +func (c *serviceCallbacks) addLeaderSwitchedCallback(cb LeaderSwitchedCallbackFunc) { c.Lock() defer c.Unlock() c.leaderSwitchedCbs = append(c.leaderSwitchedCbs, cb) @@ -72,7 +73,7 @@ func (c *serviceCallbacks) onServiceModeUpdate(mode pdpb.ServiceMode) { func (c *serviceCallbacks) onLeaderSwitched(leader string) error { c.RLock() - cbs := make([]leaderSwitchedCallbackFunc, len(c.leaderSwitchedCbs)) + cbs := make([]LeaderSwitchedCallbackFunc, len(c.leaderSwitchedCbs)) copy(cbs, c.leaderSwitchedCbs) c.RUnlock() diff --git a/client/servicediscovery/mock_service_discovery.go b/client/servicediscovery/mock_service_discovery.go index 362626243f7..779ca832e35 100644 --- a/client/servicediscovery/mock_service_discovery.go +++ b/client/servicediscovery/mock_service_discovery.go @@ -101,10 +101,10 @@ func (*mockServiceDiscovery) ScheduleCheckMemberChanged() {} func (*mockServiceDiscovery) CheckMemberChanged() error { return nil } // ExecAndAddLeaderSwitchedCallback implements the ServiceDiscovery interface. -func (*mockServiceDiscovery) ExecAndAddLeaderSwitchedCallback(leaderSwitchedCallbackFunc) {} +func (*mockServiceDiscovery) ExecAndAddLeaderSwitchedCallback(LeaderSwitchedCallbackFunc) {} // AddLeaderSwitchedCallback implements the ServiceDiscovery interface. -func (*mockServiceDiscovery) AddLeaderSwitchedCallback(leaderSwitchedCallbackFunc) {} +func (*mockServiceDiscovery) AddLeaderSwitchedCallback(LeaderSwitchedCallbackFunc) {} // AddMembersChangedCallback implements the ServiceDiscovery interface. func (*mockServiceDiscovery) AddMembersChangedCallback(func()) {} diff --git a/client/servicediscovery/service_discovery.go b/client/servicediscovery/service_discovery.go index 3657a5229c2..146a08aa381 100644 --- a/client/servicediscovery/service_discovery.go +++ b/client/servicediscovery/service_discovery.go @@ -130,11 +130,11 @@ type ServiceDiscovery interface { // in a quorum-based cluster or among the primary/secondaries in a primary/secondary configured cluster. CheckMemberChanged() error // ExecAndAddLeaderSwitchedCallback executes the callback once and adds it to the callback list then. - ExecAndAddLeaderSwitchedCallback(cb leaderSwitchedCallbackFunc) + ExecAndAddLeaderSwitchedCallback(cb LeaderSwitchedCallbackFunc) // AddLeaderSwitchedCallback adds callbacks which will be called when the leader // in a quorum-based cluster or the primary in a primary/secondary configured cluster // is switched. - AddLeaderSwitchedCallback(cb leaderSwitchedCallbackFunc) + AddLeaderSwitchedCallback(cb LeaderSwitchedCallbackFunc) // AddMembersChangedCallback adds callbacks which will be called when any leader/follower // in a quorum-based cluster or any primary/secondary in a primary/secondary configured cluster // is changed. @@ -780,7 +780,7 @@ func (c *serviceDiscovery) CheckMemberChanged() error { } // ExecAndAddLeaderSwitchedCallback executes the callback once and adds it to the callback list then. -func (c *serviceDiscovery) ExecAndAddLeaderSwitchedCallback(callback leaderSwitchedCallbackFunc) { +func (c *serviceDiscovery) ExecAndAddLeaderSwitchedCallback(callback LeaderSwitchedCallbackFunc) { url := c.getLeaderURL() if len(url) > 0 { if err := callback(url); err != nil { @@ -794,7 +794,7 @@ func (c *serviceDiscovery) ExecAndAddLeaderSwitchedCallback(callback leaderSwitc // AddLeaderSwitchedCallback adds callbacks which will be called when the leader // in a quorum-based cluster or the primary in a primary/secondary configured cluster // is switched. -func (c *serviceDiscovery) AddLeaderSwitchedCallback(callback leaderSwitchedCallbackFunc) { +func (c *serviceDiscovery) AddLeaderSwitchedCallback(callback LeaderSwitchedCallbackFunc) { c.callbacks.addLeaderSwitchedCallback(callback) } diff --git a/client/servicediscovery/tso_service_discovery.go b/client/servicediscovery/tso_service_discovery.go index 06401427522..70b823bc5af 100644 --- a/client/servicediscovery/tso_service_discovery.go +++ b/client/servicediscovery/tso_service_discovery.go @@ -141,7 +141,7 @@ type tsoServiceDiscovery struct { clientConns sync.Map // Store as map[string]*grpc.ClientConn // tsoLeaderUpdatedCb will be called when the TSO leader is updated. - tsoLeaderUpdatedCb leaderSwitchedCallbackFunc + tsoLeaderUpdatedCb LeaderSwitchedCallbackFunc checkMembershipCh chan struct{} @@ -359,7 +359,7 @@ func (c *tsoServiceDiscovery) CheckMemberChanged() error { } // ExecAndAddLeaderSwitchedCallback executes the callback once and adds it to the callback list then. -func (c *tsoServiceDiscovery) ExecAndAddLeaderSwitchedCallback(callback leaderSwitchedCallbackFunc) { +func (c *tsoServiceDiscovery) ExecAndAddLeaderSwitchedCallback(callback LeaderSwitchedCallbackFunc) { url := c.getPrimaryURL() if len(url) > 0 { if err := callback(url); err != nil { @@ -371,7 +371,7 @@ func (c *tsoServiceDiscovery) ExecAndAddLeaderSwitchedCallback(callback leaderSw // AddLeaderSwitchedCallback adds callbacks which will be called when the primary in // a primary/secondary configured cluster is switched. -func (*tsoServiceDiscovery) AddLeaderSwitchedCallback(leaderSwitchedCallbackFunc) {} +func (*tsoServiceDiscovery) AddLeaderSwitchedCallback(LeaderSwitchedCallbackFunc) {} // AddMembersChangedCallback adds callbacks which will be called when any primary/secondary // in a primary/secondary configured cluster is changed. From ac4e640fac030d0e04809d170c1b8ce264db1118 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 5 Feb 2025 21:48:27 +0800 Subject: [PATCH 30/33] tso: add retry for UpdateTSO (#9021) close tikv/pd#9020 Signed-off-by: lhy1024 Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- pkg/tso/global_allocator.go | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkg/tso/global_allocator.go b/pkg/tso/global_allocator.go index ea91bcdfadc..9462d70e3a2 100644 --- a/pkg/tso/global_allocator.go +++ b/pkg/tso/global_allocator.go @@ -140,8 +140,21 @@ func (gta *GlobalTSOAllocator) IsInitialize() bool { } // UpdateTSO is used to update the TSO in memory and the time window in etcd. -func (gta *GlobalTSOAllocator) UpdateTSO() error { - return gta.timestampOracle.UpdateTimestamp() +func (gta *GlobalTSOAllocator) UpdateTSO() (err error) { + // When meet network partition, we need to manually retry to update the global tso, + // next request succeeds with the new endpoint, according to https://github.com/etcd-io/etcd/issues/8711 + maxRetryCount := 3 + for range maxRetryCount { + err = gta.timestampOracle.UpdateTimestamp() + if err == nil { + return nil + } + log.Warn("try to update the global tso but failed", errs.ZapError(err)) + // Etcd client retry with roundRobinQuorumBackoff https://github.com/etcd-io/etcd/blob/d62cdeee4863001b09e772ed013eb1342a1d0f89/client/v3/client.go#L488 + // And its default interval is 25ms, so we sleep 50ms here. https://github.com/etcd-io/etcd/blob/d62cdeee4863001b09e772ed013eb1342a1d0f89/client/v3/options.go#L53 + time.Sleep(50 * time.Millisecond) + } + return } // SetTSO sets the physical part with given TSO. From 0c43ce53a895e35eb73a1a730e026b4db3384397 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Fri, 7 Feb 2025 11:15:55 +0800 Subject: [PATCH 31/33] *: fix default log file size (#9038) close tikv/pd#9037 Signed-off-by: Ryan Leung --- cmd/pd-server/main.go | 3 +-- pkg/mcs/scheduling/server/server.go | 2 +- pkg/mcs/scheduling/server/testutil.go | 2 +- pkg/mcs/tso/server/server.go | 2 +- pkg/utils/logutil/log.go | 4 ++-- server/api/server_test.go | 2 +- server/config/config_test.go | 15 +++++++++++++++ server/testutil.go | 2 +- tests/cluster.go | 2 +- tests/testutil.go | 2 +- tools/pd-api-bench/main.go | 2 +- tools/pd-heartbeat-bench/main.go | 2 +- tools/pd-simulator/main.go | 2 +- 13 files changed, 28 insertions(+), 14 deletions(-) diff --git a/cmd/pd-server/main.go b/cmd/pd-server/main.go index e9abef1fd4a..d45d2f641d2 100644 --- a/cmd/pd-server/main.go +++ b/cmd/pd-server/main.go @@ -209,7 +209,7 @@ func start(cmd *cobra.Command, args []string, services ...string) { // Check the PD version first before running. server.CheckAndGetPDVersion() // New zap logger - err = logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + err = logutil.SetupLogger(&cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) if err == nil { log.ReplaceGlobals(cfg.Logger, cfg.LogProps) } else { @@ -223,7 +223,6 @@ func start(cmd *cobra.Command, args []string, services ...string) { for _, msg := range cfg.WarningMsgs { log.Warn(msg) } - // TODO: Make it configurable if it has big impact on performance. grpcprometheus.EnableHandlingTimeHistogram() diff --git a/pkg/mcs/scheduling/server/server.go b/pkg/mcs/scheduling/server/server.go index 482bc48d2b4..2987df8f817 100644 --- a/pkg/mcs/scheduling/server/server.go +++ b/pkg/mcs/scheduling/server/server.go @@ -597,7 +597,7 @@ func CreateServerWrapper(cmd *cobra.Command, args []string) { } // New zap logger - err = logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + err = logutil.SetupLogger(&cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) if err == nil { log.ReplaceGlobals(cfg.Logger, cfg.LogProps) } else { diff --git a/pkg/mcs/scheduling/server/testutil.go b/pkg/mcs/scheduling/server/testutil.go index 794a1bf520c..6bb1250cbd8 100644 --- a/pkg/mcs/scheduling/server/testutil.go +++ b/pkg/mcs/scheduling/server/testutil.go @@ -31,7 +31,7 @@ import ( // NewTestServer creates a resource manager server for testing. func NewTestServer(ctx context.Context, re *require.Assertions, cfg *config.Config) (*Server, testutil.CleanupFunc, error) { // New zap logger - err := logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + err := logutil.SetupLogger(&cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) re.NoError(err) log.ReplaceGlobals(cfg.Logger, cfg.LogProps) // Flushing any buffered log entries diff --git a/pkg/mcs/tso/server/server.go b/pkg/mcs/tso/server/server.go index ebd0cca8344..32a2c8c69cf 100644 --- a/pkg/mcs/tso/server/server.go +++ b/pkg/mcs/tso/server/server.go @@ -414,7 +414,7 @@ func CreateServerWrapper(cmd *cobra.Command, args []string) { } // New zap logger - err = logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + err = logutil.SetupLogger(&cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) if err == nil { log.ReplaceGlobals(cfg.Logger, cfg.LogProps) } else { diff --git a/pkg/utils/logutil/log.go b/pkg/utils/logutil/log.go index 1620f9e3b70..14da95e19ff 100644 --- a/pkg/utils/logutil/log.go +++ b/pkg/utils/logutil/log.go @@ -72,12 +72,12 @@ func StringToZapLogLevel(level string) zapcore.Level { // SetupLogger setup the logger. func SetupLogger( - logConfig log.Config, + logConfig *log.Config, logger **zap.Logger, logProps **log.ZapProperties, redactInfoLogType RedactInfoLogType, ) error { - lg, p, err := log.InitLogger(&logConfig, zap.AddStacktrace(zapcore.FatalLevel)) + lg, p, err := log.InitLogger(logConfig, zap.AddStacktrace(zapcore.FatalLevel)) if err != nil { return errs.ErrInitLogger.Wrap(err) } diff --git a/server/api/server_test.go b/server/api/server_test.go index 9b99a6c32f8..3e52ffbfd5f 100644 --- a/server/api/server_test.go +++ b/server/api/server_test.go @@ -90,7 +90,7 @@ func mustNewCluster(re *require.Assertions, num int, opts ...func(cfg *config.Co ch := make(chan *server.Server, num) for _, cfg := range cfgs { go func(cfg *config.Config) { - err := logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + err := logutil.SetupLogger(&cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) re.NoError(err) zapLogOnce.Do(func() { log.ReplaceGlobals(cfg.Logger, cfg.LogProps) diff --git a/server/config/config_test.go b/server/config/config_test.go index b035ab22178..19b3bc53030 100644 --- a/server/config/config_test.go +++ b/server/config/config_test.go @@ -166,6 +166,7 @@ leader-schedule-limit = 0 flagSet := pflag.NewFlagSet("test", pflag.ContinueOnError) flagSet.StringP("log-level", "L", "info", "log level: debug, info, warn, error, fatal (default 'info')") + flagSet.StringP("log-file", "", "pd.log", "log file path") flagSet.Parse(nil) cfg := NewConfig() err := cfg.Parse(flagSet) @@ -174,6 +175,8 @@ leader-schedule-limit = 0 re.NoError(err) err = cfg.Adjust(&meta, false) re.NoError(err) + err = logutil.SetupLogger(&cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + re.NoError(err) // When invalid, use default values. host, err := os.Hostname() @@ -189,6 +192,9 @@ leader-schedule-limit = 0 // When undefined, use default values. re.True(cfg.PreVote) re.Equal("info", cfg.Log.Level) + re.Equal(300, cfg.Log.File.MaxSize) + re.Equal(0, cfg.Log.File.MaxDays) + re.Equal(0, cfg.Log.File.MaxBackups) re.Equal(uint64(0), cfg.Schedule.MaxMergeRegionKeys) re.Equal("http://127.0.0.1:9090", cfg.PDServerCfg.MetricStorage) @@ -252,9 +258,15 @@ tso-update-physical-interval = "15s" cfgData = ` [log] level = "debug" + +[log.file] +max-size = 100 +max-days = 10 +max-backups = 5 ` flagSet = pflag.NewFlagSet("testlog", pflag.ContinueOnError) flagSet.StringP("log-level", "L", "info", "log level: debug, info, warn, error, fatal (default 'info')") + flagSet.StringP("log-file", "", "pd.log", "log file path") flagSet.Parse(nil) cfg = NewConfig() err = cfg.Parse(flagSet) @@ -264,6 +276,9 @@ level = "debug" err = cfg.Adjust(&meta, false) re.NoError(err) re.Equal("debug", cfg.Log.Level) + re.Equal(100, cfg.Log.File.MaxSize) + re.Equal(10, cfg.Log.File.MaxDays) + re.Equal(5, cfg.Log.File.MaxBackups) } func TestMigrateFlags(t *testing.T) { diff --git a/server/testutil.go b/server/testutil.go index d8bb9bb1cd9..cacaa233138 100644 --- a/server/testutil.go +++ b/server/testutil.go @@ -86,7 +86,7 @@ func NewTestSingleConfig(c *assertutil.Checker) *config.Config { cfg.TickInterval = typeutil.NewDuration(100 * time.Millisecond) cfg.ElectionInterval = typeutil.NewDuration(3 * time.Second) cfg.LeaderPriorityCheckInterval = typeutil.NewDuration(100 * time.Millisecond) - err := logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + err := logutil.SetupLogger(&cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) c.AssertNil(err) zapLogOnce.Do(func() { log.ReplaceGlobals(cfg.Logger, cfg.LogProps) diff --git a/tests/cluster.go b/tests/cluster.go index 66d8ce60e7b..01bbf729ab1 100644 --- a/tests/cluster.go +++ b/tests/cluster.go @@ -82,7 +82,7 @@ var zapLogOnce sync.Once func NewTestServer(ctx context.Context, cfg *config.Config, services []string) (*TestServer, error) { // disable the heartbeat async runner in test cfg.Schedule.EnableHeartbeatConcurrentRunner = false - err := logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) + err := logutil.SetupLogger(&cfg.Log, &cfg.Logger, &cfg.LogProps, cfg.Security.RedactInfoLog) if err != nil { return nil, err } diff --git a/tests/testutil.go b/tests/testutil.go index 406e09345b9..7b00193cd29 100644 --- a/tests/testutil.go +++ b/tests/testutil.go @@ -97,7 +97,7 @@ var once sync.Once func InitLogger(logConfig log.Config, logger *zap.Logger, logProps *log.ZapProperties, redactInfoLog logutil.RedactInfoLogType) (err error) { once.Do(func() { // Setup the logger. - err = logutil.SetupLogger(logConfig, &logger, &logProps, redactInfoLog) + err = logutil.SetupLogger(&logConfig, &logger, &logProps, redactInfoLog) if err != nil { return } diff --git a/tools/pd-api-bench/main.go b/tools/pd-api-bench/main.go index 0d9235d5055..37568bf041a 100644 --- a/tools/pd-api-bench/main.go +++ b/tools/pd-api-bench/main.go @@ -96,7 +96,7 @@ func main() { default: log.Fatal("parse cmd flags error", zap.Error(err)) } - err = logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, logutil.RedactInfoLogOFF) + err = logutil.SetupLogger(&cfg.Log, &cfg.Logger, &cfg.LogProps, logutil.RedactInfoLogOFF) if err == nil { log.ReplaceGlobals(cfg.Logger, cfg.LogProps) } else { diff --git a/tools/pd-heartbeat-bench/main.go b/tools/pd-heartbeat-bench/main.go index 60b1db6734d..d74a693ea3d 100644 --- a/tools/pd-heartbeat-bench/main.go +++ b/tools/pd-heartbeat-bench/main.go @@ -479,7 +479,7 @@ func main() { } // New zap logger - err = logutil.SetupLogger(cfg.Log, &cfg.Logger, &cfg.LogProps, logutil.RedactInfoLogOFF) + err = logutil.SetupLogger(&cfg.Log, &cfg.Logger, &cfg.LogProps, logutil.RedactInfoLogOFF) if err == nil { log.ReplaceGlobals(cfg.Logger, cfg.LogProps) } else { diff --git a/tools/pd-simulator/main.go b/tools/pd-simulator/main.go index 59a58f0a00b..a8381e38ef9 100644 --- a/tools/pd-simulator/main.go +++ b/tools/pd-simulator/main.go @@ -114,7 +114,7 @@ func run(simCase string, simConfig *sc.SimConfig) { // NewSingleServer creates a pd server for simulator. func NewSingleServer(ctx context.Context, simConfig *sc.SimConfig) (*server.Server, testutil.CleanupFunc) { - err := logutil.SetupLogger(simConfig.ServerConfig.Log, &simConfig.ServerConfig.Logger, &simConfig.ServerConfig.LogProps, simConfig.ServerConfig.Security.RedactInfoLog) + err := logutil.SetupLogger(&simConfig.ServerConfig.Log, &simConfig.ServerConfig.Logger, &simConfig.ServerConfig.LogProps, simConfig.ServerConfig.Security.RedactInfoLog) if err == nil { log.ReplaceGlobals(simConfig.ServerConfig.Logger, simConfig.ServerConfig.LogProps) } else { From ce761611dc77e63e536d9d49168e0808edd1b333 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Sat, 8 Feb 2025 14:35:49 +0800 Subject: [PATCH 32/33] metrics: add metrics for the internal pd client calls (#9043) close tikv/pd#9039 Signed-off-by: Ryan Leung --- client/metrics/metrics.go | 42 ++++++++++++++++++-- client/servicediscovery/service_discovery.go | 9 +++++ 2 files changed, 48 insertions(+), 3 deletions(-) diff --git a/client/metrics/metrics.go b/client/metrics/metrics.go index da7637b19be..9e1981446ee 100644 --- a/client/metrics/metrics.go +++ b/client/metrics/metrics.go @@ -64,9 +64,11 @@ func InitAndRegisterMetrics(constLabels prometheus.Labels) { } var ( - cmdDuration *prometheus.HistogramVec - cmdFailedDuration *prometheus.HistogramVec - requestDuration *prometheus.HistogramVec + cmdDuration *prometheus.HistogramVec + cmdFailedDuration *prometheus.HistogramVec + internalCmdDuration *prometheus.HistogramVec + internalCmdFailedDuration *prometheus.HistogramVec + requestDuration *prometheus.HistogramVec // TSOBestBatchSize is the histogram of the best batch size of TSO requests. TSOBestBatchSize prometheus.Histogram @@ -105,6 +107,26 @@ func initMetrics(constLabels prometheus.Labels) { Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13), }, []string{"type"}) + internalCmdDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "pd_client", + Subsystem: "internal_cmd", + Name: "handle_cmds_duration_seconds", + Help: "Bucketed histogram of processing time (s) of handled success internal cmds.", + ConstLabels: constLabels, + Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13), + }, []string{"type"}) + + internalCmdFailedDuration = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Namespace: "pd_client", + Subsystem: "internal_cmd", + Name: "handle_failed_cmds_duration_seconds", + Help: "Bucketed histogram of processing time (s) of failed handled internal cmds.", + ConstLabels: constLabels, + Buckets: prometheus.ExponentialBuckets(0.0005, 2, 13), + }, []string{"type"}) + requestDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Namespace: "pd_client", @@ -228,6 +250,12 @@ var ( CmdFailedDurationUpdateGCSafePointV2 prometheus.Observer CmdFailedDurationUpdateServiceSafePointV2 prometheus.Observer + InternalCmdDurationGetClusterInfo prometheus.Observer + InternalCmdDurationGetMembers prometheus.Observer + + InternalCmdFailedDurationGetClusterInfo prometheus.Observer + InternalCmdFailedDurationGetMembers prometheus.Observer + // RequestDurationTSO records the durations of the successful TSO requests. RequestDurationTSO prometheus.Observer // RequestFailedDurationTSO records the durations of the failed TSO requests. @@ -281,6 +309,12 @@ func initCmdDurations() { CmdFailedDurationUpdateGCSafePointV2 = cmdFailedDuration.WithLabelValues("update_gc_safe_point_v2") CmdFailedDurationUpdateServiceSafePointV2 = cmdFailedDuration.WithLabelValues("update_service_safe_point_v2") + InternalCmdDurationGetClusterInfo = internalCmdDuration.WithLabelValues("get_cluster_info") + InternalCmdDurationGetMembers = internalCmdDuration.WithLabelValues("get_members") + + InternalCmdFailedDurationGetClusterInfo = internalCmdFailedDuration.WithLabelValues("get_cluster_info") + InternalCmdFailedDurationGetMembers = internalCmdFailedDuration.WithLabelValues("get_members") + RequestDurationTSO = requestDuration.WithLabelValues("tso") RequestFailedDurationTSO = requestDuration.WithLabelValues("tso-failed") } @@ -288,6 +322,8 @@ func initCmdDurations() { func registerMetrics() { prometheus.MustRegister(cmdDuration) prometheus.MustRegister(cmdFailedDuration) + prometheus.MustRegister(internalCmdDuration) + prometheus.MustRegister(internalCmdFailedDuration) prometheus.MustRegister(requestDuration) prometheus.MustRegister(TSOBestBatchSize) prometheus.MustRegister(TSOBatchSize) diff --git a/client/servicediscovery/service_discovery.go b/client/servicediscovery/service_discovery.go index 146a08aa381..e26c081cea7 100644 --- a/client/servicediscovery/service_discovery.go +++ b/client/servicediscovery/service_discovery.go @@ -38,6 +38,7 @@ import ( "github.com/tikv/pd/client/constants" "github.com/tikv/pd/client/errs" + "github.com/tikv/pd/client/metrics" "github.com/tikv/pd/client/opt" "github.com/tikv/pd/client/pkg/retry" "github.com/tikv/pd/client/pkg/utils/grpcutil" @@ -909,12 +910,16 @@ func (c *serviceDiscovery) getClusterInfo(ctx context.Context, url string, timeo if err != nil { return nil, err } + start := time.Now() + defer func() { metrics.InternalCmdDurationGetClusterInfo.Observe(time.Since(start).Seconds()) }() clusterInfo, err := pdpb.NewPDClient(cc).GetClusterInfo(ctx, &pdpb.GetClusterInfoRequest{}) if err != nil { + metrics.InternalCmdFailedDurationGetClusterInfo.Observe(time.Since(start).Seconds()) attachErr := errors.Errorf("error:%s target:%s status:%s", err, cc.Target(), cc.GetState().String()) return nil, errs.ErrClientGetClusterInfo.Wrap(attachErr).GenWithStackByCause() } if clusterInfo.GetHeader().GetError() != nil { + metrics.InternalCmdFailedDurationGetClusterInfo.Observe(time.Since(start).Seconds()) attachErr := errors.Errorf("error:%s target:%s status:%s", clusterInfo.GetHeader().GetError().String(), cc.Target(), cc.GetState().String()) return nil, errs.ErrClientGetClusterInfo.Wrap(attachErr).GenWithStackByCause() } @@ -928,12 +933,16 @@ func (c *serviceDiscovery) getMembers(ctx context.Context, url string, timeout t if err != nil { return nil, err } + start := time.Now() + defer func() { metrics.InternalCmdDurationGetMembers.Observe(time.Since(start).Seconds()) }() members, err := pdpb.NewPDClient(cc).GetMembers(ctx, &pdpb.GetMembersRequest{}) if err != nil { + metrics.InternalCmdFailedDurationGetMembers.Observe(time.Since(start).Seconds()) attachErr := errors.Errorf("error:%s target:%s status:%s", err, cc.Target(), cc.GetState().String()) return nil, errs.ErrClientGetMember.Wrap(attachErr).GenWithStackByCause() } if members.GetHeader().GetError() != nil { + metrics.InternalCmdFailedDurationGetMembers.Observe(time.Since(start).Seconds()) attachErr := errors.Errorf("error:%s target:%s status:%s", members.GetHeader().GetError().String(), cc.Target(), cc.GetState().String()) return nil, errs.ErrClientGetMember.Wrap(attachErr).GenWithStackByCause() } From 3af31b2fc064bbc3bf5f5889cef277962eb63fd6 Mon Sep 17 00:00:00 2001 From: Ryan Leung Date: Sat, 8 Feb 2025 16:46:18 +0800 Subject: [PATCH 33/33] metrics: fix typo (#9046) ref tikv/pd#8678 Signed-off-by: Ryan Leung Co-authored-by: ti-chi-bot[bot] <108142056+ti-chi-bot[bot]@users.noreply.github.com> --- client/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/client/metrics/metrics.go b/client/metrics/metrics.go index 9e1981446ee..d168f9b46f3 100644 --- a/client/metrics/metrics.go +++ b/client/metrics/metrics.go @@ -200,7 +200,7 @@ func initMetrics(constLabels prometheus.Labels) { Name: "circuit_breaker_count", Help: "Circuit breaker counters", ConstLabels: constLabels, - }, []string{"name", "success"}) + }, []string{"name", "event"}) } // CmdDurationXXX and CmdFailedDurationXXX are the durations of the client commands.