2680e7c4-a912-47b9-b7d4-3b2bae24d7a4
/tidb/cdc/default/default/changefeed/info/89e873a8-9196-4744-b15e-5e845aa2de97
{"upstream-id":7362107735913147282,"namespace":"default","changefeed-id":"89e873a8-9196-4744-b15e-5e845aa2de97","sink-uri":"pulsar://127.0.0.1:6650/ticdc-owner-resign-test-7229?protocol=canal-json\u0026enable-tidb-extension=true","create-time":"2024-04-26T17:38:27.289995389+08:00","start-ts":449347402330537986,"target-ts":0,"admin-job-type":0,"sort-engine":"","sort-dir":"","config":{"memory-quota":1073741824,"case-sensitive":false,"force-replicate":false,"check-gc-safe-point":true,"enable-sync-point":false,"ignore-ineligible-table":false,"bdr-mode":false,"sync-point-interval":600000000000,"sync-point-retention":86400000000000,"filter":{"rules":["*.*"],"ignore-txn-start-ts":null,"event-filters":null},"mounter":{"worker-num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include-commit-ts":false,"binary-encoding-method":"base64"},"encoder-concurrency":32,"terminator":"\r\n","date-separator":"day","enable-partition-separator":true,"enable-kafka-sink-v2":false,"only-output-updated-columns":false,"delete-only-output-handle-key-columns":false,"pulsar-config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance-timeout-in-sec":150},"consistent":{"level":"none","max-log-size":64,"flush-interval":2000,"meta-flush-interval":200,"encoding-worker-num":16,"flush-worker-num":8,"storage":"","use-file-backend":false,"compression":"","memory-usage":{"memory-quota-percentage":50,"event-cache-percentage":0}},"scheduler":{"enable-table-across-nodes":false,"region-threshold":100000,"write-key-threshold":0,"region-per-span":0},"integrity":{"integrity-check-level":"none","corruption-handle-level":"warn"},"changefeed-error-stuck-duration":1800000000000,"sql-mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced-status":{"synced-check-interval":300,"checkpoint-interval":15}},"state":"normal","error":null,"warning":null,"creator-version":"v7.5.1-16-g445a4712c","epoch":449347402409443329}
/tidb/cdc/default/default/changefeed/status/89e873a8-9196-4744-b15e-5e845aa2de97
{"checkpoint-ts":449347405083312130,"min-table-barrier-ts":449347405083312130,"admin-job-type":0}
/tidb/cdc/default/default/task/position/2680e7c4-a912-47b9-b7d4-3b2bae24d7a4/89e873a8-9196-4744-b15e-5e845aa2de97
{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}
/tidb/cdc/default/default/task/position/b8922c6b-893d-4c27-b16d-cb2ece1fde81/89e873a8-9196-4744-b15e-5e845aa2de97
{"checkpoint-ts":0,"resolved-ts":0,"count":0,"error":null,"warning":null}
/tidb/cdc/default/default/upstream/7362107735913147282
{"id":7362107735913147282,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
cdc cli capture list --server http://127.0.0.1:8302 |jq '.|length'|grep -E '^2$'
2
run task successfully
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
Starting Upstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/cli/run.sh using Sink-Type: pulsar... <<=================
The 1 times to try to start tidb cluster...
check diff failed 1-th time, retry later
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
check diff failed 2-th time, retry later
start tidb cluster in /tmp/tidb_cdc_test/cli
Starting Upstream PD...
Release Version: v7.5.1-5-g584533652
Edition: Community
Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b
Git Branch: release-7.5
UTC Build Time: 2024-04-03 10:04:14
Starting Downstream PD...
Release Version: v7.5.1-5-g584533652
Edition: Community
Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b
Git Branch: release-7.5
UTC Build Time: 2024-04-03 10:04:14
Verifying upstream PD is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c6715fdd80007 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-2m115, pid:3808, start at 2024-04-26 17:38:44.760976917 +0800 CST m=+5.485027329 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:40:44.773 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:38:44.775 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:28:44.775 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
[2024/04/26 17:38:45.993 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0015581c0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
check diff successfully
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c6715fdd80007 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-2m115, pid:3808, start at 2024-04-26 17:38:44.760976917 +0800 CST m=+5.485027329 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:40:44.773 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:38:44.775 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:28:44.775 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c6715ff3c0014 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-2m115, pid:3896, start at 2024-04-26 17:38:44.86050084 +0800 CST m=+5.486070779 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:40:44.871 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:38:44.865 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:28:44.865 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-12-g9002cc34d
Edition: Community
Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314
Git Branch: HEAD
UTC Build Time: 2024-04-18 07:24:48
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time: 2024-04-18 07:28:40
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/drop_many_tables/tiflash/log/error.log
arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/drop_many_tables/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c67162c1c0015 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-2djhk, pid:3769, start at 2024-04-26 17:38:47.708136576 +0800 CST m=+10.278684905 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:40:47.715 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:38:47.687 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:28:47.687 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c67162c1c0015 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-2djhk, pid:3769, start at 2024-04-26 17:38:47.708136576 +0800 CST m=+10.278684905 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:40:47.715 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:38:47.687 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:28:47.687 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c67162cc40007 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-2djhk, pid:3847, start at 2024-04-26 17:38:47.736344735 +0800 CST m=+10.236142732 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:40:47.745 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:38:47.729 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:28:47.729 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-12-g9002cc34d
Edition: Community
Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314
Git Branch: HEAD
UTC Build Time: 2024-04-18 07:24:48
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time: 2024-04-18 07:28:40
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/error.log
arg matches is ArgMatches { args: {"addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/ddl_puller_lag/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990
Git Commit Branch: release-7.5
UTC Build Time: 2024-04-25 09:30:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990
Git Commit Branch: release-7.5
UTC Build Time: 2024-04-25 09:30:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5147.out cli tso query --pd=http://127.0.0.1:2379ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
[2024/04/26 17:38:50.866 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc00031c1c0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
Starting Upstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
+ set +x
+ tso='449347408357228545
PASS
coverage: 1.9% of statements in github.com/pingcap/tiflow/...'
+ echo 449347408357228545 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
[Fri Apr 26 17:38:51 CST 2024] <<<<<< START cdc server in drop_many_tables case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ [[ no != \n\o ]]
+ GO_FAILPOINTS=
+ (( i = 0 ))
+ (( i <= 50 ))
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.51705172.out server --log-file /tmp/tidb_cdc_test/drop_many_tables/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/drop_many_tables/cdc_data --cluster-id default
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5109.out cli tso query --pd=http://127.0.0.1:2379ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
[2024/04/26 17:38:53.249 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc00031c1c0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
[2024/04/26 17:38:53.412 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc00092e1c0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"]
[2024/04/26 17:38:53.413 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc00037f180/127.0.0.1:2379] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"]
+ set +x
+ tso='449347408910352385
PASS
coverage: 1.9% of statements in github.com/pingcap/tiflow/...'
+ echo 449347408910352385 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
[Fri Apr 26 17:38:53 CST 2024] <<<<<< START cdc server in ddl_puller_lag case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ [[ no != \n\o ]]
+ GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorDDLResolved=1*sleep(180000)'
+ (( i = 0 ))
+ (( i <= 50 ))
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.51555157.out server --log-file /tmp/tidb_cdc_test/ddl_puller_lag/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/ddl_puller_lag/cdc_data --cluster-id default
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 26 Apr 2024 09:38:54 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/c6339c5f-f084-441c-85d6-9d8a5f2eddab
{"id":"c6339c5f-f084-441c-85d6-9d8a5f2eddab","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c55a1ff6
c6339c5f-f084-441c-85d6-9d8a5f2eddab
/tidb/cdc/default/default/upstream/7362107880503902962
{"id":7362107880503902962,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/c6339c5f-f084-441c-85d6-9d8a5f2eddab
{"id":"c6339c5f-f084-441c-85d6-9d8a5f2eddab","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c55a1ff6
c6339c5f-f084-441c-85d6-9d8a5f2eddab
/tidb/cdc/default/default/upstream/7362107880503902962
{"id":7362107880503902962,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/c6339c5f-f084-441c-85d6-9d8a5f2eddab
{"id":"c6339c5f-f084-441c-85d6-9d8a5f2eddab","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c55a1ff6
c6339c5f-f084-441c-85d6-9d8a5f2eddab
/tidb/cdc/default/default/upstream/7362107880503902962
{"id":7362107880503902962,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.drop_many_tables.cli.5218.out cli changefeed create --start-ts=449347408357228545 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-drop-tables-test-20271?protocol=canal-json&enable-tidb-extension=true'
Create changefeed successfully!
ID: 07e461f5-dc83-41ee-8873-e22f5e343687
Info: {"upstream_id":7362107880503902962,"namespace":"default","id":"07e461f5-dc83-41ee-8873-e22f5e343687","sink_uri":"pulsar://127.0.0.1:6650/ticdc-drop-tables-test-20271?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-04-26T17:38:54.956798402+08:00","start_ts":449347408357228545,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-16-g445a4712c","resolved_ts":449347408357228545,"checkpoint_ts":449347408357228545,"checkpoint_time":"2024-04-26 17:38:49.976"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
+ set +x
[Fri Apr 26 17:38:56 CST 2024] <<<<<< START Pulsar consumer in drop_many_tables case >>>>>>
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
check diff successfully
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 26 Apr 2024 09:38:56 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/bd7584a0-0672-43d0-b0fb-7b9db9e2ddd6
{"id":"bd7584a0-0672-43d0-b0fb-7b9db9e2ddd6","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c54f090c
bd7584a0-0672-43d0-b0fb-7b9db9e2ddd6
/tidb/cdc/default/default/upstream/7362107874169378087
{"id":7362107874169378087,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/bd7584a0-0672-43d0-b0fb-7b9db9e2ddd6
{"id":"bd7584a0-0672-43d0-b0fb-7b9db9e2ddd6","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c54f090c
bd7584a0-0672-43d0-b0fb-7b9db9e2ddd6
/tidb/cdc/default/default/upstream/7362107874169378087
{"id":7362107874169378087,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/bd7584a0-0672-43d0-b0fb-7b9db9e2ddd6
{"id":"bd7584a0-0672-43d0-b0fb-7b9db9e2ddd6","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c54f090c
bd7584a0-0672-43d0-b0fb-7b9db9e2ddd6
/tidb/cdc/default/default/upstream/7362107874169378087
{"id":7362107874169378087,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.ddl_puller_lag.cli.5199.out cli changefeed create --start-ts=449347408910352385 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-ddl-puller-lag-test-20297?protocol=canal-json&enable-tidb-extension=true'
Create changefeed successfully!
ID: e92c1931-89af-47b7-a741-9a546ff8ae6c
Info: {"upstream_id":7362107874169378087,"namespace":"default","id":"e92c1931-89af-47b7-a741-9a546ff8ae6c","sink_uri":"pulsar://127.0.0.1:6650/ticdc-ddl-puller-lag-test-20297?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-04-26T17:38:57.315890025+08:00","start_ts":449347408910352385,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-16-g445a4712c","resolved_ts":449347408910352385,"checkpoint_ts":449347408910352385,"checkpoint_time":"2024-04-26 17:38:52.086"}
PASS
table drop_tables.c not exists for 1-th check, retry later
wait process cdc.test exit for 1-th time...
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
wait process cdc.test exit for 2-th time...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
wait process cdc.test exit for 3-th time...
cdc.test: no process found
wait process cdc.test exit for 4-th time...
process cdc.test already exit
[Fri Apr 26 17:38:58 CST 2024] <<<<<< run test case owner_resign success! >>>>>>
table drop_tables.c not exists for 2-th check, retry later
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
+ set +x
[Fri Apr 26 17:38:59 CST 2024] <<<<<< START Pulsar consumer in ddl_puller_lag case >>>>>>
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c6716d2a40013 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-dsb23, pid:9940, start at 2024-04-26 17:38:58.375407212 +0800 CST m=+6.611204455 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:40:58.385 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:38:58.345 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:28:58.345 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c6716d2a40013 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-dsb23, pid:9940, start at 2024-04-26 17:38:58.375407212 +0800 CST m=+6.611204455 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:40:58.385 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:38:58.345 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:28:58.345 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c6716d52c0014 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-dsb23, pid:10028, start at 2024-04-26 17:38:58.534552714 +0800 CST m=+6.705118499 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:40:58.543 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:38:58.507 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:28:58.507 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-12-g9002cc34d
Edition: Community
Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314
Git Branch: HEAD
UTC Build Time: 2024-04-18 07:24:48
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time: 2024-04-18 07:28:40
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/cli/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/cli/tiflash/log/error.log
arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/cli/tiflash/db/proxy"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/cli/tiflash/log/proxy.log"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/cli/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
table drop_tables.c not exists for 3-th check, retry later
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11322.out cli tso query --pd=http://127.0.0.1:2379ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
table drop_tables.c not exists for 4-th check, retry later
+ set +x
+ tso='449347411625902081
PASS
coverage: 1.9% of statements in github.com/pingcap/tiflow/...'
+ echo 449347411625902081 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
[Fri Apr 26 17:39:04 CST 2024] <<<<<< START cdc server in cli case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ GO_FAILPOINTS=
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ [[ no != \n\o ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.1136811370.out server --log-file /tmp/tidb_cdc_test/cli/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/cli/cdc_data --cluster-id default
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
table drop_tables.c not exists for 5-th check, retry later
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
table drop_tables.c exists
check diff successfully
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 26 Apr 2024 09:39:07 GMT
< Content-Length: 657
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/2e080b64-cc09-4b43-8b72-e5785485ffa4
{"id":"2e080b64-cc09-4b43-8b72-e5785485ffa4","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c586e807
2e080b64-cc09-4b43-8b72-e5785485ffa4
/tidb/cdc/default/default/upstream/7362107930350332633
{"id":7362107930350332633,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/2e080b64-cc09-4b43-8b72-e5785485ffa4
{"id":"2e080b64-cc09-4b43-8b72-e5785485ffa4","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c586e807
2e080b64-cc09-4b43-8b72-e5785485ffa4
/tidb/cdc/default/default/upstream/7362107930350332633
{"id":7362107930350332633,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/2e080b64-cc09-4b43-8b72-e5785485ffa4
{"id":"2e080b64-cc09-4b43-8b72-e5785485ffa4","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c586e807
2e080b64-cc09-4b43-8b72-e5785485ffa4
/tidb/cdc/default/default/upstream/7362107930350332633
{"id":7362107930350332633,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2779,http://127.0.0.1:2679,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11425.out cli changefeed create --start-ts=449347411625902081 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-cli-test-27989?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name
[WARN] --tz is deprecated in changefeed settings.
Create changefeed successfully!
ID: custom-changefeed-name
Info: {"upstream_id":7362107930350332633,"namespace":"default","id":"custom-changefeed-name","sink_uri":"pulsar://127.0.0.1:6650/ticdc-cli-test-27989?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-04-26T17:39:07.851252924+08:00","start_ts":449347411625902081,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-16-g445a4712c","resolved_ts":449347411625902081,"checkpoint_ts":449347411625902081,"checkpoint_time":"2024-04-26 17:39:02.445"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
wait process cdc.test exit for 1-th time...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
cdc.test: no process found
wait process cdc.test exit for 2-th time...
process cdc.test already exit
[Fri Apr 26 17:39:08 CST 2024] <<<<<< run test case drop_many_tables success! >>>>>>
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_etcd_worker_delay/run.sh using Sink-Type: pulsar... <<=================
+ set +x
[Fri Apr 26 17:39:09 CST 2024] <<<<<< START Pulsar consumer in cli case >>>>>>
table test.simple not exists for 1-th check, retry later
The 1 times to try to start tidb cluster...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
table test.simple exists
table test.`simple-dash` not exists for 1-th check, retry later
[2024/04/26 17:39:11.462 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0015581c0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
[2024/04/26 17:39:11.563 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0015581c0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
table test.`simple-dash` exists
+ endpoints=http://127.0.0.1:2379
+ changefeed_id=custom-changefeed-name
+ expected_state=normal
+ error_msg=null
+ tls_dir=
+ [[ http://127.0.0.1:2379 =~ https ]]
++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s
+ info='{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "normal",
"checkpoint_tso": 449347412058439707,
"checkpoint_time": "2024-04-26 17:39:04.095",
"error": null
}'
+ echo '{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "normal",
"checkpoint_tso": 449347412058439707,
"checkpoint_time": "2024-04-26 17:39:04.095",
"error": null
}'
{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "normal",
"checkpoint_tso": 449347412058439707,
"checkpoint_time": "2024-04-26 17:39:04.095",
"error": null
}
++ echo '{' '"upstream_id":' 7362107930350332633, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449347412058439707, '"checkpoint_time":' '"2024-04-26' '17:39:04.095",' '"error":' null '}'
++ jq -r .state
+ state=normal
+ [[ ! normal == \n\o\r\m\a\l ]]
++ echo '{' '"upstream_id":' 7362107930350332633, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449347412058439707, '"checkpoint_time":' '"2024-04-26' '17:39:04.095",' '"error":' null '}'
++ jq -r .error.message
+ message=null
+ [[ ! null =~ null ]]
changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379start tidb cluster in /tmp/tidb_cdc_test/processor_etcd_worker_delay
Starting Upstream PD...
Release Version: v7.5.1-5-g584533652
Edition: Community
Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b
Git Branch: release-7.5
UTC Build Time: 2024-04-03 10:04:14
Starting Downstream PD...
changefeed count 1 check pass, pd_addr: http://127.0.0.1:2679
changefeed count 1 check pass, pd_addr: http://127.0.0.1:2779Release Version: v7.5.1-5-g584533652
Edition: Community
Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b
Git Branch: release-7.5
UTC Build Time: 2024-04-03 10:04:14
Verifying upstream PD is started...
[2024/04/26 17:39:14.039 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0015581c0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
changefeed count 1 check pass, pd_addr: http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
Error: [CDC:ErrChangefeedUpdateRefused]changefeed update error: can only update changefeed config when it is stopped or failed
update changefeed config should fail when changefeed is running, got Diff of changefeed config:
{Type:update Path:[Config CaseSensitive] From:false To:true}
{Type:update Path:[Config SyncPointInterval] From:<nil> To:0xc002e3eb20}
{Type:update Path:[Config SyncPointRetention] From:<nil> To:0xc002e3eb28}
{Type:update Path:[Config Consistent] From:<nil> To:0xc0009a41c0}
{Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true}
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11690.out cli changefeed --changefeed-id custom-changefeed-name pause
PASS
coverage: 2.0% of statements in github.com/pingcap/tiflow/...
[2024/04/26 17:39:16.566 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc00031c1c0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
Verifying downstream PD is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
[2024/04/26 17:39:16.861 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0015581c0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
+ set +x
Starting Upstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990
Git Commit Branch: release-7.5
UTC Build Time: 2024-04-25 09:30:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990
Git Commit Branch: release-7.5
UTC Build Time: 2024-04-25 09:30:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
[2024/04/26 17:39:18.000 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc00092e1c0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"]
[2024/04/26 17:39:18.014 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc00037f180/127.0.0.1:2379] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"]
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/processor_stop_delay/run.sh using Sink-Type: pulsar... <<=================
The 1 times to try to start tidb cluster...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
Starting Upstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
+ endpoints=http://127.0.0.1:2379
+ changefeed_id=custom-changefeed-name
+ expected_state=stopped
+ error_msg=null
+ tls_dir=
+ [[ http://127.0.0.1:2379 =~ https ]]
++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s
+ info='{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "stopped",
"checkpoint_tso": 449347414876487683,
"checkpoint_time": "2024-04-26 17:39:14.845",
"error": null
}'
+ echo '{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "stopped",
"checkpoint_tso": 449347414876487683,
"checkpoint_time": "2024-04-26 17:39:14.845",
"error": null
}'
{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "stopped",
"checkpoint_tso": 449347414876487683,
"checkpoint_time": "2024-04-26 17:39:14.845",
"error": null
}
++ echo '{' '"upstream_id":' 7362107930350332633, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449347414876487683, '"checkpoint_time":' '"2024-04-26' '17:39:14.845",' '"error":' null '}'
++ jq -r .state
+ state=stopped
+ [[ ! stopped == \s\t\o\p\p\e\d ]]
++ echo '{' '"upstream_id":' 7362107930350332633, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"stopped",' '"checkpoint_tso":' 449347414876487683, '"checkpoint_time":' '"2024-04-26' '17:39:14.845",' '"error":' null '}'
++ jq -r .error.message
+ message=null
+ [[ ! null =~ null ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11760.out cli changefeed update --pd=http://127.0.0.1:2379,http://127.0.0.1:2679,http://127.0.0.1:2779 --config=/tmp/tidb_cdc_test/cli/changefeed.toml --no-confirm --changefeed-id custom-changefeed-name
Diff of changefeed config:
{Type:update Path:[Config CaseSensitive] From:false To:true}
{Type:update Path:[Config SyncPointInterval] From:<nil> To:0xc002e142d8}
{Type:update Path:[Config SyncPointRetention] From:<nil> To:0xc002e142e8}
{Type:update Path:[Config Consistent] From:<nil> To:0xc000a56150}
{Type:update Path:[Config Scheduler EnableTableAcrossNodes] From:false To:true}
Update changefeed config successfully!
ID: custom-changefeed-name
Info: {"upstream_id":7362107930350332633,"namespace":"default","id":"custom-changefeed-name","sink_uri":"pulsar://127.0.0.1:6650/ticdc-cli-test-27989?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-04-26T17:39:07.851252924+08:00","start_ts":449347411625902081,"admin_job_type":1,"config":{"memory_quota":1073741824,"case_sensitive":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","encoder_concurrency":32,"terminator":"\r\n","enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"stopped","creator_version":"v7.5.1-16-g445a4712c","resolved_ts":449347414876487683,"checkpoint_ts":449347414876487683,"checkpoint_time":"2024-04-26 17:39:14.845"}
PASS
coverage: 2.9% of statements in github.com/pingcap/tiflow/...
start tidb cluster in /tmp/tidb_cdc_test/processor_stop_delay
Starting Upstream PD...
Release Version: v7.5.1-5-g584533652
Edition: Community
Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b
Git Branch: release-7.5
UTC Build Time: 2024-04-03 10:04:14
Starting Downstream PD...
Release Version: v7.5.1-5-g584533652
Edition: Community
Git Commit Hash: 58453365285465cd90bc4472cff2bad7ce4d764b
Git Branch: release-7.5
UTC Build Time: 2024-04-03 10:04:14
Verifying upstream PD is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
+ set +x
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11793.out cli changefeed --changefeed-id custom-changefeed-name resume
PASS
coverage: 2.1% of statements in github.com/pingcap/tiflow/...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990
Git Commit Branch: release-7.5
UTC Build Time: 2024-04-25 09:30:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: 46c2b1e4fa7bc9a71e9294770766c2f2b8bb8990
Git Commit Branch: release-7.5
UTC Build Time: 2024-04-25 09:30:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
+ set +x
Starting Upstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v7.5.1-43-ge406d5780b
Edition: Community
Git Commit Hash: e406d5780b18a1f2aaf6230cde4b6403991e228d
Git Branch: release-7.5
UTC Build Time: 2024-04-22 09:56:10
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c67186870000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-jx727, pid:1089, start at 2024-04-26 17:39:24.351671265 +0800 CST m=+73.927354434 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:41:24.363 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:39:24.367 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:29:24.367 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c67186870000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-jx727, pid:1089, start at 2024-04-26 17:39:24.351671265 +0800 CST m=+73.927354434 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:41:24.363 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:39:24.367 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:29:24.367 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c671862e00004 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-jx727, pid:1317, start at 2024-04-26 17:39:23.965014649 +0800 CST m=+73.470323579 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:41:23.973 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:39:23.960 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:29:23.960 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-12-g9002cc34d
Edition: Community
Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314
Git Branch: HEAD
UTC Build Time: 2024-04-18 07:24:48
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time: 2024-04-18 07:28:40
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log
arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/resolve_lock/tiflash-proxy.toml"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
+ endpoints=http://127.0.0.1:2379
+ changefeed_id=custom-changefeed-name
+ expected_state=normal
+ error_msg=null
+ tls_dir=
+ [[ http://127.0.0.1:2379 =~ https ]]
++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s
+ info='{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "normal",
"checkpoint_tso": 449347418022215682,
"checkpoint_time": "2024-04-26 17:39:26.845",
"error": null
}'
+ echo '{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "normal",
"checkpoint_tso": 449347418022215682,
"checkpoint_time": "2024-04-26 17:39:26.845",
"error": null
}'
{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "normal",
"checkpoint_tso": 449347418022215682,
"checkpoint_time": "2024-04-26 17:39:26.845",
"error": null
}
++ echo '{' '"upstream_id":' 7362107930350332633, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449347418022215682, '"checkpoint_time":' '"2024-04-26' '17:39:26.845",' '"error":' null '}'
++ jq -r .state
+ state=normal
+ [[ ! normal == \n\o\r\m\a\l ]]
++ echo '{' '"upstream_id":' 7362107930350332633, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449347418022215682, '"checkpoint_time":' '"2024-04-26' '17:39:26.845",' '"error":' null '}'
++ jq -r .error.message
+ message=null
+ [[ ! null =~ null ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11859.out cli changefeed --changefeed-id custom-changefeed-name remove
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
Changefeed remove successfully.
ID: custom-changefeed-name
CheckpointTs: 449347418022215682
SinkURI: pulsar://127.0.0.1:6650/ticdc-cli-test-27989?protocol=canal-json&enable-tidb-extension=true
PASS
coverage: 2.1% of statements in github.com/pingcap/tiflow/...
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2835.out cli tso query --pd=http://127.0.0.1:2379ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c67189ef80013 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-tnjjp, pid:3574, start at 2024-04-26 17:39:27.852476024 +0800 CST m=+8.134822268 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:41:27.868 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:39:27.857 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:29:27.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c67189ef80013 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-tnjjp, pid:3574, start at 2024-04-26 17:39:27.852476024 +0800 CST m=+8.134822268 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:41:27.868 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:39:27.857 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:29:27.857 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c6718a3fc0006 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-tnjjp, pid:3607, start at 2024-04-26 17:39:28.150704979 +0800 CST m=+8.309348131 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:41:28.171 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:39:28.177 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:29:28.177 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
+ set +x
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-12-g9002cc34d
Edition: Community
Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314
Git Branch: HEAD
UTC Build Time: 2024-04-18 07:24:48
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time: 2024-04-18 07:28:40
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/error.log
arg matches is ArgMatches { args: {"engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash-proxy.toml"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_etcd_worker_delay/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
+ set +x
+ tso='449347418552270852
PASS
coverage: 1.9% of statements in github.com/pingcap/tiflow/...'
+ echo 449347418552270852 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
[Fri Apr 26 17:39:30 CST 2024] <<<<<< START cdc server in resolve_lock case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ GO_FAILPOINTS=
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.28662868.out server --log-file /tmp/tidb_cdc_test/resolve_lock/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/resolve_lock/cdc_data --cluster-id default
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ [[ no != \n\o ]]
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
table big_txn.usertable1 not exists for 1-th check, retry later
table big_txn.usertable1 not exists for 2-th check, retry later
table big_txn.usertable1 not exists for 3-th check, retry later
table big_txn.usertable1 not exists for 4-th check, retry later
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c6718c7e8000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-2m115, pid:6323, start at 2024-04-26 17:39:30.445983044 +0800 CST m=+5.279424384 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:41:30.453 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:39:30.426 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:29:30.426 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
table big_txn.usertable1 not exists for 5-th check, retry later
[Fri Apr 26 17:39:32 CST 2024] <<<<<< START cdc server in processor_etcd_worker_delay case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ [[ no != \n\o ]]
+ (( i = 0 ))
+ (( i <= 50 ))
+ GO_FAILPOINTS='github.com/pingcap/tiflow/pkg/orchestrator/ProcessorEtcdDelay=return(true)'
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_etcd_worker_delay.48514853.out server --log-file /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_etcd_worker_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
changefeed count 0 check pass, pd_addr: http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.11911.out cli changefeed create '--sink-uri=pulsar://127.0.0.1:6650/ticdc-cli-test-27989?protocol=canal-json&enable-tidb-extension=true' --tz=Asia/Shanghai -c=custom-changefeed-name
[WARN] --tz is deprecated in changefeed settings.
Create changefeed successfully!
ID: custom-changefeed-name
Info: {"upstream_id":7362107930350332633,"namespace":"default","id":"custom-changefeed-name","sink_uri":"pulsar://127.0.0.1:6650/ticdc-cli-test-27989?protocol=canal-json\u0026enable-tidb-extension=true","create_time":"2024-04-26T17:39:33.405451631+08:00","start_ts":449347419713044481,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"pulsar_config":{"connection-timeout":5,"operation-timeout":30,"batching-max-messages":1000,"batching-max-publish-delay":10,"send-timeout":30},"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-16-g445a4712c","resolved_ts":449347419713044481,"checkpoint_ts":449347419713044481,"checkpoint_time":"2024-04-26 17:39:33.295"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c6718c7e8000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-2m115, pid:6323, start at 2024-04-26 17:39:30.445983044 +0800 CST m=+5.279424384 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:41:30.453 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:39:30.426 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:29:30.426 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63c6718c8980014 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:p-tiflow-release-7-5-pull-cdc-integration-pulsar-test-253-2m115, pid:6411, start at 2024-04-26 17:39:30.516174462 +0800 CST m=+5.290957325 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240426-17:41:30.526 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240426-17:39:30.520 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240426-17:29:30.520 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-12-g9002cc34d
Edition: Community
Git Commit Hash: 9002cc34d3b593a718b6c5260ba18f30a45ab314
Git Branch: HEAD
UTC Build Time: 2024-04-18 07:24:48
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time: 2024-04-18 07:28:40
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/error.log
arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-12-g9002cc34d"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["9002cc34d3b593a718b6c5260ba18f30a45ab314"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/log/proxy.log"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/processor_stop_delay/tiflash-proxy.toml"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
table big_txn.usertable1 not exists for 6-th check, retry later
+ set +x
[2024/04/26 17:39:34.998 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc0015581c0/127.0.0.1:2479] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
[2024/04/26 17:39:35.018 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc00031c1c0/127.0.0.1:2379] [attempt=0] [error="rpc error: code = DeadlineExceeded desc = context deadline exceeded"]
[Fri Apr 26 17:39:35 CST 2024] <<<<<< START cdc server in processor_stop_delay case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/processor/processorStopDelay=1*sleep(10000)'
+ [[ no != \n\o ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.processor_stop_delay.76597661.out server --log-file /tmp/tidb_cdc_test/processor_stop_delay/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/processor_stop_delay/cdc_data --cluster-id default --addr 127.0.0.1:8300 --pd http://127.0.0.1:2379
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 26 Apr 2024 09:39:35 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/0b08b3de-2bf0-4e6a-8f64-f366ade27049
{"id":"0b08b3de-2bf0-4e6a-8f64-f366ade27049","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c5f162f5
0b08b3de-2bf0-4e6a-8f64-f366ade27049
/tidb/cdc/default/default/upstream/7362108050205241545
{"id":7362108050205241545,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/0b08b3de-2bf0-4e6a-8f64-f366ade27049
{"id":"0b08b3de-2bf0-4e6a-8f64-f366ade27049","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c5f162f5
0b08b3de-2bf0-4e6a-8f64-f366ade27049
/tidb/cdc/default/default/upstream/7362108050205241545
{"id":7362108050205241545,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/0b08b3de-2bf0-4e6a-8f64-f366ade27049
{"id":"0b08b3de-2bf0-4e6a-8f64-f366ade27049","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c5f162f5
0b08b3de-2bf0-4e6a-8f64-f366ade27049
/tidb/cdc/default/default/upstream/7362108050205241545
{"id":7362108050205241545,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
[Fri Apr 26 17:39:36 CST 2024] <<<<<< START Pulsar consumer in processor_etcd_worker_delay case >>>>>>
[2024/04/26 17:39:36.424 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc00037f180/127.0.0.1:2379] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"]
< HTTP/1.1 200 OK
< Date: Fri, 26 Apr 2024 09:39:36 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/1e3b3dca-4272-4d95-a4b5-9f17268dd125
{"id":"1e3b3dca-4272-4d95-a4b5-9f17268dd125","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c49b1415
1e3b3dca-4272-4d95-a4b5-9f17268dd125
/tidb/cdc/default/default/upstream/7362107683958403961
{"id":7362107683958403961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/1e3b3dca-4272-4d95-a4b5-9f17268dd125
{"id":"1e3b3dca-4272-4d95-a4b5-9f17268dd125","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c49b1415
1e3b3dca-4272-4d95-a4b5-9f17268dd125
/tidb/cdc/default/default/upstream/7362107683958403961
{"id":7362107683958403961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/1e3b3dca-4272-4d95-a4b5-9f17268dd125
{"id":"1e3b3dca-4272-4d95-a4b5-9f17268dd125","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c49b1415
1e3b3dca-4272-4d95-a4b5-9f17268dd125
/tidb/cdc/default/default/upstream/7362107683958403961
{"id":7362107683958403961,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.resolve_lock.cli.2926.out cli changefeed create --start-ts=449347418552270852 '--sink-uri=pulsar://127.0.0.1:6650/ticdc-resolve-lock-test-3875?protocol=canal-json&enable-tidb-extension=true'
table big_txn.usertable1 exists
+ endpoints=http://127.0.0.1:2379
+ changefeed_id=custom-changefeed-name
+ expected_state=normal
+ error_msg=null
+ tls_dir=
+ [[ http://127.0.0.1:2379 =~ https ]]
++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c custom-changefeed-name -s
+ info='{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "normal",
"checkpoint_tso": 449347420643917830,
"checkpoint_time": "2024-04-26 17:39:36.846",
"error": null
}'
+ echo '{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "normal",
"checkpoint_tso": 449347420643917830,
"checkpoint_time": "2024-04-26 17:39:36.846",
"error": null
}'
{
"upstream_id": 7362107930350332633,
"namespace": "default",
"id": "custom-changefeed-name",
"state": "normal",
"checkpoint_tso": 449347420643917830,
"checkpoint_time": "2024-04-26 17:39:36.846",
"error": null
}
++ echo '{' '"upstream_id":' 7362107930350332633, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449347420643917830, '"checkpoint_time":' '"2024-04-26' '17:39:36.846",' '"error":' null '}'
++ jq -r .state
+ state=normal
+ [[ ! normal == \n\o\r\m\a\l ]]
++ echo '{' '"upstream_id":' 7362107930350332633, '"namespace":' '"default",' '"id":' '"custom-changefeed-name",' '"state":' '"normal",' '"checkpoint_tso":' 449347420643917830, '"checkpoint_time":' '"2024-04-26' '17:39:36.846",' '"error":' null '}'
++ jq -r .error.message
+ message=null
+ [[ ! null =~ null ]]
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 26 Apr 2024 09:39:38 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/00741435-f5d3-400b-b158-203a9995642a
{"id":"00741435-f5d3-400b-b158-203a9995642a","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c60d6cf4
00741435-f5d3-400b-b158-203a9995642a
/tidb/cdc/default/default/upstream/7362108078581261498
{"id":7362108078581261498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/00741435-f5d3-400b-b158-203a9995642a
{"id":"00741435-f5d3-400b-b158-203a9995642a","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c60d6cf4
00741435-f5d3-400b-b158-203a9995642a
/tidb/cdc/default/default/upstream/7362108078581261498
{"id":7362108078581261498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/00741435-f5d3-400b-b158-203a9995642a
{"id":"00741435-f5d3-400b-b158-203a9995642a","address":"127.0.0.1:8300","version":"v7.5.1-16-g445a4712c"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f19c60d6cf4
00741435-f5d3-400b-b158-203a9995642a
/tidb/cdc/default/default/upstream/7362108078581261498
{"id":7362108078581261498,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
[Fri Apr 26 17:39:38 CST 2024] <<<<<< START Pulsar consumer in processor_stop_delay case >>>>>>
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.12005.out cli unsafe delete-service-gc-safepoint
table processor_stop_delay.t not exists for 1-th check, retry later
Confirm that you know what this command will do and use it at your own risk [Y/N]
CDC service GC safepoint truncated in PD!
PASS
coverage: 2.0% of statements in github.com/pingcap/tiflow/...
+ set +x
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.12030.out cli unsafe reset --no-confirm --pd=http://127.0.0.1:2379
reset and all metadata truncated in PD!
PASS
table processor_stop_delay.t exists
check diff failed 1-th time, retry later
coverage: 2.1% of statements in github.com/pingcap/tiflow/...
+ set +x
[2024/04/26 17:39:42.572 +08:00] [WARN] [retry_interceptor.go:62] ["retrying of unary invoker failed"] [target=etcd-endpoints://0xc00037f180/127.0.0.1:2379] [attempt=0] [error="rpc error: code = NotFound desc = etcdserver: requested lease not found"]
Error: [CDC:ErrInternalServerError]internal server error: rpc error: code = Unknown desc = [PD:tso:ErrGenerateTimestamp]generate timestamp failed, requested pd is not leader of cluster
warning: GOCOVERDIR not set, no coverage data emitted
check diff successfully
check diff failed 1-th time, retry later
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.12124.out cli unsafe resolve-lock --region=28
PASS
coverage: 2.0% of statements in github.com/pingcap/tiflow/...
+ set +x
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.cli.cli.12147.out cli unsafe resolve-lock --region=28 --ts=449347422150983685
check diff failed 2-th time, retry later
PASS
coverage: 2.0% of statements in github.com/pingcap/tiflow/...
+ set +x
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 218 100 212 100 6 141k 4106 --:--:-- --:--:-- --:--:-- 207k
{
"error_msg": "[CDC:ErrAPIInvalidParam]invalid log level: json: cannot unmarshal string into Go value of type struct { Level string \"json:\\\"log_level\\\"\" }",
"error_code": "CDC:ErrAPIInvalidParam"
check diff failed 3-th time, retry later
table processor_delay.t1 exists
table processor_delay.t2 exists
table processor_delay.t3 exists
table processor_delay.t4 not exists for 1-th check, retry later
table processor_delay.t4 exists
table processor_delay.t5 not exists for 1-th check, retry later
check diff failed 4-th time, retry later
} % Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 176 100 176 0 0 173k 0 --:--:-- --:--:-- --:--:-- 171k
{
"version": "v7.5.1-16-g445a4712c",
"git_hash": "445a4712c4a4945beb3d9584e431ee578b954b72",
"id": "8f08a00f-e03b-46b6-b0d1-104357592638",
"pid": 11373,
"is_owner": true
}wait process cdc.test exit for 1-th time...
table processor_delay.t5 exists
table processor_delay.t6 not exists for 1-th check, retry later
wait process cdc.test exit for 2-th time...
wait process cdc.test exit for 3-th time...
check diff failed 5-th time, retry later
cdc.test: no process found
wait process cdc.test exit for 4-th time...
process cdc.test already exit
[Fri Apr 26 17:39:54 CST 2024] <<<<<< run test case cli success! >>>>>>
table processor_delay.t6 exists
table processor_delay.t7 exists
table processor_delay.t8 not exists for 1-th check, retry later
check diff failed 6-th time, retry later
table processor_delay.t8 exists
table processor_delay.t9 not exists for 1-th check, retry later
check diff successfully
wait process cdc.test exit for 1-th time...
table processor_delay.t9 exists
table processor_delay.t10 not exists for 1-th check, retry later
wait process cdc.test exit for 2-th time...
wait process cdc.test exit for 3-th time...
process cdc.test already exit
[Fri Apr 26 17:39:59 CST 2024] <<<<<< run test case processor_stop_delay success! >>>>>>
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
Post stage
[Pipeline] sh
table processor_delay.t10 exists
table processor_delay.t11 not exists for 1-th check, retry later
+ ls /tmp/tidb_cdc_test/
cov.resolve_lock.cli.2835.out
resolve_lock
++ find /tmp/tidb_cdc_test/ -type f -name '*.log'
+ tar -cvzf log-G11.tar.gz /tmp/tidb_cdc_test/resolve_lock/cdc.log /tmp/tidb_cdc_test/resolve_lock/tikv1.log /tmp/tidb_cdc_test/resolve_lock/down_pd.log /tmp/tidb_cdc_test/resolve_lock/pd1/region-meta/000001.log /tmp/tidb_cdc_test/resolve_lock/pd1/hot-region/000001.log /tmp/tidb_cdc_test/resolve_lock/stdout.log /tmp/tidb_cdc_test/resolve_lock/tikv_down/db/000005.log /tmp/tidb_cdc_test/resolve_lock/tidb_down.log /tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log /tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log /tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log /tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy/db/000005.log /tmp/tidb_cdc_test/resolve_lock/tikv3/db/000005.log /tmp/tidb_cdc_test/resolve_lock/tidb.log /tmp/tidb_cdc_test/resolve_lock/tikv2.log /tmp/tidb_cdc_test/resolve_lock/down_pd/region-meta/000001.log /tmp/tidb_cdc_test/resolve_lock/down_pd/hot-region/000001.log /tmp/tidb_cdc_test/resolve_lock/tikv1/db/000005.log /tmp/tidb_cdc_test/resolve_lock/pd1.log /tmp/tidb_cdc_test/resolve_lock/tidb_other.log /tmp/tidb_cdc_test/resolve_lock/tikv2/db/000005.log /tmp/tidb_cdc_test/resolve_lock/tidb-slow.log /tmp/tidb_cdc_test/resolve_lock/tikv3.log /tmp/tidb_cdc_test/resolve_lock/tikv_down.log
tar: Removing leading `/' from member names
/tmp/tidb_cdc_test/resolve_lock/cdc.log
/tmp/tidb_cdc_test/resolve_lock/tikv1.log
/tmp/tidb_cdc_test/resolve_lock/down_pd.log
/tmp/tidb_cdc_test/resolve_lock/pd1/region-meta/000001.log
/tmp/tidb_cdc_test/resolve_lock/pd1/hot-region/000001.log
/tmp/tidb_cdc_test/resolve_lock/stdout.log
/tmp/tidb_cdc_test/resolve_lock/tikv_down/db/000005.log
/tmp/tidb_cdc_test/resolve_lock/tidb_down.log
/tmp/tidb_cdc_test/resolve_lock/tiflash/log/error.log
/tmp/tidb_cdc_test/resolve_lock/tiflash/log/server.log
/tmp/tidb_cdc_test/resolve_lock/tiflash/log/proxy.log
/tmp/tidb_cdc_test/resolve_lock/tiflash/db/proxy/db/000005.log
/tmp/tidb_cdc_test/resolve_lock/tikv3/db/000005.log
/tmp/tidb_cdc_test/resolve_lock/tidb.log
/tmp/tidb_cdc_test/resolve_lock/tikv2.log
/tmp/tidb_cdc_test/resolve_lock/down_pd/region-meta/000001.log
/tmp/tidb_cdc_test/resolve_lock/down_pd/hot-region/000001.log
/tmp/tidb_cdc_test/resolve_lock/tikv1/db/000005.log
/tmp/tidb_cdc_test/resolve_lock/pd1.log
/tmp/tidb_cdc_test/resolve_lock/tidb_other.log
/tmp/tidb_cdc_test/resolve_lock/tikv2/db/000005.log
/tmp/tidb_cdc_test/resolve_lock/tidb-slow.log
/tmp/tidb_cdc_test/resolve_lock/tikv3.log
/tmp/tidb_cdc_test/resolve_lock/tikv_down.log
+ ls -alh log-G11.tar.gz
-rw-r--r-- 1 jenkins jenkins 2.7M Apr 26 17:40 log-G11.tar.gz
[Pipeline] archiveArtifacts
Archiving artifacts
Recording fingerprints
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
table processor_delay.t11 exists
table processor_delay.t12 not exists for 1-th check, retry later
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G11'
Sending interrupt signal to process
Killing processes
kill finished with exit code 0
Sending interrupt signal to process
Killing processes
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_pulsar_test/tiflow/tests/integration_tests/savepoint/run.sh using Sink-Type: pulsar... <<=================
The 1 times to try to start tidb cluster...
kill finished with exit code 0
Sending interrupt signal to process
Killing processes
kill finished with exit code 0
Sending interrupt signal to process
Killing processes
script returned exit code 143
script returned exit code 143
kill finished with exit code 0
Sending interrupt signal to process
Killing processes
kill finished with exit code 0
[Pipeline] }
Cache not saved (inner-step execution failed)
table processor_delay.t12 exists
table processor_delay.t13 not exists for 1-th check, retry later
[Pipeline] }
{"level":"warn","ts":1714124405.2007833,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002d70700/127.0.0.1:2479","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"}
{"level":"warn","ts":1714124405.2039964,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc0021f96c0/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: read tcp 127.0.0.1:36234->127.0.0.1:2379: read: connection reset by peer"}
Cache not saved (inner-step execution failed)
script returned exit code 143
[Pipeline] // cache
[Pipeline] // cache
[Pipeline] }
[Pipeline] }
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] // dir
[Pipeline] // dir
[Pipeline] // cache
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] // timeout
[Pipeline] // timeout
[Pipeline] // dir
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] // stage
[Pipeline] // stage
[Pipeline] // timeout
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] // container
[Pipeline] // container
[Pipeline] // stage
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] // withEnv
[Pipeline] // container
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] // node
[Pipeline] // node
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] // podTemplate
[Pipeline] // node
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] // withEnv
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] }
[Pipeline] }
[Pipeline] // stage
[Pipeline] // stage
[Pipeline] // withEnv
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G03'
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G09'
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G16'
{"level":"warn","ts":1714124404.3516245,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002935340/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"}
{"level":"warn","ts":1714124406.3522856,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002935340/127.0.0.1:2379","attempt":1,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""}
script returned exit code 143
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
{"level":"warn","ts":1714124404.8973317,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002344e00/127.0.0.1:2379","attempt":0,"error":"rpc error: code = Unavailable desc = error reading from server: EOF"}
{"level":"warn","ts":1714124406.8977237,"caller":"v3@v3.5.10/retry_interceptor.go:62","msg":"retrying of unary invoker failed","target":"etcd-endpoints://0xc002344e00/127.0.0.1:2379","attempt":1,"error":"rpc error: code = DeadlineExceeded desc = latest balancer error: last connection error: connection error: desc = \"transport: Error while dialing: dial tcp 127.0.0.1:2379: connect: connection refused\""}
script returned exit code 143
[Pipeline] // stage
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G04'
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G14'
[Pipeline] // parallel
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] End of Pipeline
ERROR: script returned exit code 1
Finished: FAILURE
![[Jenkins]](/jenkins/static/62b7ca6c/images/svgs/logo.svg){kind=logo}
