==================== Test output for //br/pkg/utils:utils_test (shard 25 of 33): === RUN TestTaskRegisterFailedGrant logger.go:146: 2024-05-06T02:26:08.677Z INFO m0 LISTEN GRPC {"member": "m0", "grpcAddr": "localhost:m0", "m.Name": "m0"} logger.go:146: 2024-05-06T02:26:08.677Z INFO m0 launching a member {"member": "m0", "name": "m0", "advertise-peer-urls": ["unix://127.0.0.1:2100200035"], "listen-client-urls": ["unix://127.0.0.1:2100300035"], "grpc-url": "unix://localhost:m0"} logger.go:146: 2024-05-06T02:26:08.678Z INFO m0 opened backend db {"member": "m0", "path": "/tmp/TestTaskRegisterFailedGrant221039868/002/etcd1199801676/member/snap/db", "took": "467.139µs"} logger.go:146: 2024-05-06T02:26:08.680Z INFO m0 starting local member {"member": "m0", "local-member-id": "f0c778fdf2c164b0", "cluster-id": "546fc962f846716a"} logger.go:146: 2024-05-06T02:26:08.680Z INFO m0.raft f0c778fdf2c164b0 switched to configuration voters=() {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.680Z INFO m0.raft f0c778fdf2c164b0 became follower at term 0 {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.680Z INFO m0.raft newRaft f0c778fdf2c164b0 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0] {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.680Z INFO m0.raft f0c778fdf2c164b0 became follower at term 1 {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.680Z INFO m0.raft f0c778fdf2c164b0 switched to configuration voters=(17349969121562944688) {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.680Z WARN m0 simple token is not cryptographically signed {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.681Z INFO m0 kvstore restored {"member": "m0", "current-rev": 1} logger.go:146: 2024-05-06T02:26:08.681Z INFO m0 enabled backend quota with default value {"member": "m0", "quota-name": "v3-applier", "quota-size-bytes": 2147483648, "quota-size": "2.1 GB"} logger.go:146: 2024-05-06T02:26:08.681Z INFO m0 starting etcd server {"member": "m0", "local-member-id": "f0c778fdf2c164b0", "local-server-version": "3.5.12", "cluster-version": "to_be_decided"} logger.go:146: 2024-05-06T02:26:08.681Z INFO grpc [[core] [Server #1] Server created] logger.go:146: 2024-05-06T02:26:08.681Z INFO m0 started as single-node; fast-forwarding election ticks {"member": "m0", "local-member-id": "f0c778fdf2c164b0", "forward-ticks": 9, "forward-duration": "90ms", "election-ticks": 10, "election-timeout": "100ms"} logger.go:146: 2024-05-06T02:26:08.682Z INFO m0.raft f0c778fdf2c164b0 switched to configuration voters=(17349969121562944688) {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.682Z INFO m0 added member {"member": "m0", "cluster-id": "546fc962f846716a", "local-member-id": "f0c778fdf2c164b0", "added-peer-id": "f0c778fdf2c164b0", "added-peer-peer-urls": ["unix://127.0.0.1:2100200035"]} logger.go:146: 2024-05-06T02:26:08.684Z INFO grpc [[core] [Server #2] Server created] logger.go:146: 2024-05-06T02:26:08.684Z INFO grpc [[core] [Server #2 ListenSocket #3] ListenSocket created] logger.go:146: 2024-05-06T02:26:08.684Z INFO grpc [[core] [Server #1 ListenSocket #4] ListenSocket created] logger.go:146: 2024-05-06T02:26:08.684Z INFO m0 launched a member {"member": "m0", "name": "m0", "advertise-peer-urls": ["unix://127.0.0.1:2100200035"], "listen-client-urls": ["unix://127.0.0.1:2100300035"], "grpc-url": "unix://localhost:m0"} logger.go:146: 2024-05-06T02:26:08.731Z INFO m0.raft f0c778fdf2c164b0 is starting a new election at term 1 {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.732Z INFO m0.raft f0c778fdf2c164b0 became candidate at term 2 {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.732Z INFO m0.raft f0c778fdf2c164b0 received MsgVoteResp from f0c778fdf2c164b0 at term 2 {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.732Z INFO m0.raft f0c778fdf2c164b0 became leader at term 2 {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.732Z INFO m0.raft raft.node: f0c778fdf2c164b0 elected leader f0c778fdf2c164b0 at term 2 {"member": "m0"} logger.go:146: 2024-05-06T02:26:08.732Z INFO m0 published local member to cluster through raft {"member": "m0", "local-member-id": "f0c778fdf2c164b0", "local-member-attributes": "{Name:m0 ClientURLs:[unix://127.0.0.1:2100300035]}", "request-path": "/0/members/f0c778fdf2c164b0/attributes", "cluster-id": "546fc962f846716a", "publish-timeout": "5.2s"} logger.go:146: 2024-05-06T02:26:08.732Z INFO m0 setting up initial cluster version using v2 API {"member": "m0", "cluster-version": "3.5"} logger.go:146: 2024-05-06T02:26:08.734Z INFO m0 set initial cluster version {"member": "m0", "cluster-id": "546fc962f846716a", "local-member-id": "f0c778fdf2c164b0", "cluster-version": "3.5"} logger.go:146: 2024-05-06T02:26:08.734Z INFO m0 enabled capabilities for version {"member": "m0", "cluster-version": "3.5"} logger.go:146: 2024-05-06T02:26:08.734Z INFO m0 cluster version is updated {"member": "m0", "cluster-version": "3.5"} cluster.go:266: - m0 -> f0c778fdf2c164b0 (unix://localhost:m0) logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[core] [Channel #5] Channel created] logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[core] [Channel #5] original dial target is: "etcd-endpoints://0xc0008bb340/localhost:m0"] logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[core] [Channel #5] parsed dial target is: resolver.Target{URL:url.URL{Scheme:"etcd-endpoints", Opaque:"", User:(*url.Userinfo)(nil), Host:"0xc0008bb340", Path:"/localhost:m0", RawPath:"", OmitHost:false, ForceQuery:false, RawQuery:"", Fragment:"", RawFragment:""}}] logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[core] [Channel #5] Channel authority set to "localhost:m0"] logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[core] [Channel #5] Resolver state updated: { "Addresses": [ { "Addr": "unix:localhost:m0", "ServerName": "localhost:m0", "Attributes": null, "BalancerAttributes": null, "Metadata": null } ], "Endpoints": [ { "Addresses": [ { "Addr": "unix:localhost:m0", "ServerName": "localhost:m0", "Attributes": null, "BalancerAttributes": null, "Metadata": null } ], "Attributes": null } ], "ServiceConfig": { "Config": { "Config": null, "LB": "round_robin", "Methods": {} }, "Err": null }, "Attributes": null } (service config updated; resolver returned new addresses)] logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[core] [Channel #5] Channel switches to new LB policy "round_robin"] logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[balancer] base.baseBalancer: got new ClientConn state: {{[{Addr: "unix:localhost:m0", ServerName: "localhost:m0", }] [{[{Addr: "unix:localhost:m0", ServerName: "localhost:m0", }] }] 0xc000053f60 } }] logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[core] [Channel #5 SubChannel #6] Subchannel created] logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[roundrobin] roundrobinPicker: Build called with info: {map[]}] logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[core] [Channel #5] Channel Connectivity change to CONNECTING] logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[core] [Channel #5 SubChannel #6] Subchannel Connectivity change to CONNECTING] logger.go:146: 2024-05-06T02:26:08.742Z INFO grpc [[core] [Channel #5 SubChannel #6] Subchannel picks a new address "unix:localhost:m0" to connect] logger.go:146: 2024-05-06T02:26:08.742Z INFO grpc [[balancer] base.baseBalancer: handle SubConn state change: 0xc000fe4ba0, CONNECTING] logger.go:146: 2024-05-06T02:26:08.741Z INFO grpc [[core] [Channel #5] Channel exiting idle mode] logger.go:146: 2024-05-06T02:26:08.750Z INFO grpc [[core] [Channel #5 SubChannel #6] Subchannel Connectivity change to READY] logger.go:146: 2024-05-06T02:26:08.750Z INFO grpc [[balancer] base.baseBalancer: handle SubConn state change: 0xc000fe4ba0, READY] logger.go:146: 2024-05-06T02:26:08.750Z INFO grpc [[roundrobin] roundrobinPicker: Build called with info: {map[SubConn(id:6):{{Addr: "unix:localhost:m0", ServerName: "localhost:m0", }}]}] logger.go:146: 2024-05-06T02:26:08.750Z INFO grpc [[core] [Channel #5] Channel Connectivity change to READY] [2024/05/06 02:26:08.753 +00:00] [WARN] [register.go:219] ["the keepalive channel is closed, try to recreate it"] [2024/05/06 02:26:08.754 +00:00] [WARN] [register.go:235] ["failed to grant lease"] [error=failpoint-error] [errorVerbose="failpoint-error\ngithub.com/pingcap/tidb/br/pkg/utils.(*taskRegister).keepaliveLoop\n\tbr/pkg/utils/register.go:227\nruntime.goexit\n\tsrc/runtime/asm_amd64.s:1650"] [2024/05/06 02:26:18.757 +00:00] [WARN] [register.go:235] ["failed to grant lease"] [error=failpoint-error] [errorVerbose="failpoint-error\ngithub.com/pingcap/tidb/br/pkg/utils.(*taskRegister).keepaliveLoop\n\tbr/pkg/utils/register.go:227\nruntime.goexit\n\tsrc/runtime/asm_amd64.s:1650"] register_test.go:119: Error Trace: br/pkg/utils/register_test.go:119 Error: Should be true Test: TestTaskRegisterFailedGrant logger.go:146: 2024-05-06T02:26:28.760Z INFO grpc [[core] [Channel #5] Channel Connectivity change to SHUTDOWN] logger.go:146: 2024-05-06T02:26:28.760Z INFO grpc [[core] [Channel #5] Closing the name resolver] logger.go:146: 2024-05-06T02:26:28.760Z INFO grpc [[core] [Channel #5] ccBalancerWrapper: closing] logger.go:146: 2024-05-06T02:26:28.760Z INFO grpc [[core] [Channel #5 SubChannel #6] Subchannel Connectivity change to SHUTDOWN] logger.go:146: 2024-05-06T02:26:28.760Z INFO grpc [[core] [Channel #5 SubChannel #6] Subchannel deleted] logger.go:146: 2024-05-06T02:26:28.760Z INFO grpc [[transport] [client-transport 0xc000919680] Closing: rpc error: code = Canceled desc = grpc: the client connection is closing] logger.go:146: 2024-05-06T02:26:28.760Z INFO grpc [[core] [Channel #5] Channel deleted] logger.go:146: 2024-05-06T02:26:28.760Z INFO m0 terminating a member {"member": "m0", "name": "m0", "advertise-peer-urls": ["unix://127.0.0.1:2100200035"], "listen-client-urls": ["unix://127.0.0.1:2100300035"], "grpc-url": "unix://localhost:m0"} logger.go:146: 2024-05-06T02:26:28.760Z INFO grpc [[core] [Server #1 ListenSocket #4] ListenSocket deleted] logger.go:146: 2024-05-06T02:26:28.760Z WARN m0.client retrying of unary invoker failed {"member": "m0", "target": "etcd-endpoints://0xc0008bb340/localhost:m0", "attempt": 0, "error": "rpc error: code = Canceled desc = grpc: the client connection is closing"} logger.go:146: 2024-05-06T02:26:28.761Z INFO grpc [[transport] [client-transport 0xc000919680] loopyWriter exiting with error: connection error: desc = "transport is closing"] logger.go:146: 2024-05-06T02:26:28.761Z INFO grpc [[transport] [server-transport 0xc000e941a0] Closing: EOF] logger.go:146: 2024-05-06T02:26:28.761Z INFO grpc [[transport] [server-transport 0xc000e941a0] loopyWriter exiting with error: transport closed by client] logger.go:146: 2024-05-06T02:26:28.761Z INFO grpc [[core] [Server #2 ListenSocket #3] ListenSocket deleted] logger.go:146: 2024-05-06T02:26:28.761Z INFO m0 verification of persisted state {"member": "m0", "data-dir": "/tmp/TestTaskRegisterFailedGrant221039868/002/etcd1199801676"} logger.go:146: 2024-05-06T02:26:28.770Z INFO m0 verification: consistentIndex OK {"member": "m0", "backend-consistent-index": 12, "hardstate-commit": 12} logger.go:146: 2024-05-06T02:26:28.772Z INFO m0 verification of persisted state successful {"member": "m0", "data-dir": "/tmp/TestTaskRegisterFailedGrant221039868/002/etcd1199801676"} logger.go:146: 2024-05-06T02:26:28.775Z INFO m0 terminated a member {"member": "m0", "name": "m0", "advertise-peer-urls": ["unix://127.0.0.1:2100200035"], "listen-client-urls": ["unix://127.0.0.1:2100300035"], "grpc-url": "unix://localhost:m0"} --- FAIL: TestTaskRegisterFailedGrant (20.10s) FAIL panic: Log in goroutine after TestTaskRegisterFailedGrant has completed: 2024-05-06T02:26:28.785Z WARN m0.client retrying of unary invoker failed {"member": "m0", "target": "etcd-endpoints://0xc0008bb340/localhost:m0", "attempt": 1, "error": "rpc error: code = Canceled desc = grpc: the client connection is closing"} goroutine 358 [running]: testing.(*common).logDepth(0xc000183520, {0xc000517c00, 0xf4}, 0x3) GOROOT/src/testing/testing.go:1022 +0x4c5 testing.(*common).log(...) GOROOT/src/testing/testing.go:1004 testing.(*common).Logf(0xc000183520, {0x1e0e3d0?, 0x4101e5?}, {0xc00118afc0?, 0x183bc80?, 0x7f43b93f2601?}) GOROOT/src/testing/testing.go:1055 +0x54 go.uber.org/zap/zaptest.TestingWriter.Write({{0x7f43b93ecfe0?, 0xc000183520?}, 0x0?}, {0xc0009aac00?, 0xf5, 0xc00118afa0?}) external/org_uber_go_zap/zaptest/logger.go:146 +0xdc go.uber.org/zap/zapcore.(*ioCore).Write(0xc00098a450, {0x1, {0xc1862dd52ed6effd, 0x4b161f00f, 0x3799aa0}, {0xc0004474e0, 0x9}, {0x1b8b52e, 0x20}, {0x0, ...}, ...}, ...) external/org_uber_go_zap/zapcore/core.go:99 +0xb5 go.uber.org/zap/zapcore.(*CheckedEntry).Write(0xc000e961a0, {0xc0005a5200, 0x3, 0x3}) external/org_uber_go_zap/zapcore/entry.go:253 +0x1dc go.uber.org/zap.(*Logger).Warn(0xc00130d100?, {0x1b8b52e?, 0x4?}, {0xc0005a5200, 0x3, 0x3}) external/org_uber_go_zap/logger.go:255 +0x51 go.etcd.io/etcd/client/v3.(*Client).unaryClientInterceptor.func1({0x1e344a0?, 0xc000ae04b0?}, {0x1b6cd83, 0x14}, {0x1ad9a20, 0xc000946b40}, {0x1a9e800, 0xc001614450}, 0xc000972700, 0x1c24000, ...) external/io_etcd_go_etcd_client_v3/retry_interceptor.go:62 +0x90e google.golang.org/grpc.(*ClientConn).Invoke(0xc000972700, {0x1e344a0?, 0xc000ae04b0?}, {0x1b6cd83?, 0xc000946b40?}, {0x1ad9a20?, 0xc000946b40?}, {0x1a9e800?, 0xc001614450?}, {0x37991a0, ...}) external/org_golang_google_grpc/call.go:35 +0x205 go.etcd.io/etcd/api/v3/etcdserverpb.(*kVClient).Put(0xc000ae2030, {0x1e344a0, 0xc000ae04b0}, 0x412785?, {0x37991a0, 0x3, 0x3}) external/io_etcd_go_etcd_api_v3/etcdserverpb/rpc.pb.go:6469 +0xc5 go.etcd.io/etcd/client/v3.(*retryKVClient).Put(0xc000700000?, {0x1e344a0?, 0xc000ae04b0?}, 0xc000b87800?, {0x37991a0?, 0x20?, 0x0?}) external/io_etcd_go_etcd_client_v3/retry.go:109 +0x2a go.etcd.io/etcd/client/v3.(*kv).Do(_, {_, _}, {0x2, {0xc0001df560, 0x1e, 0x20}, {0x0, 0x0, 0x0}, ...}) external/io_etcd_go_etcd_client_v3/kv.go:156 +0x2d4 go.etcd.io/etcd/client/v3.(*kv).Put(0x1e146c8?, {0x1e344a0, 0xc000ae04b0}, {0xc0001df160?, 0x0?}, {0x0?, 0x0?}, {0xc000ae2170, 0x1, 0x1}) external/io_etcd_go_etcd_client_v3/kv.go:114 +0xd0 github.com/pingcap/tidb/br/pkg/utils.(*taskRegister).keepaliveLoop(0xc000ae0320, {0x1e344a0, 0xc000ae04b0}, 0xc0002b2120) br/pkg/utils/register.go:245 +0x951 created by github.com/pingcap/tidb/br/pkg/utils.(*taskRegister).RegisterTask in goroutine 109 br/pkg/utils/register.go:183 +0x245 -- Coverage runner: Not collecting coverage for failed test. The following commands failed with status 2 /home/jenkins/.tidb/tmp/5cfc4a16c2c6d36cce346a7a12411ff4/sandbox/linux-sandbox/6512/execroot/__main__/bazel-out/k8-fastbuild/bin/br/pkg/utils/utils_test_/utils_test.runfiles/__main__/br/pkg/utils/utils_test_/utils_test ================================================================================