+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.5235052352.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Wed, 08 May 2024 08:41:07 GMT
< Content-Length: 612
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/1df4ee5f-867b-470e-91c1-a47e628aa0f9
{"id":"1df4ee5f-867b-470e-91c1-a47e628aa0f9","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f575cc95a90
1df4ee5f-867b-470e-91c1-a47e628aa0f9
/tidb/cdc/default/default/upstream/7366545286647601465
{"id":7366545286647601465,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/1df4ee5f-867b-470e-91c1-a47e628aa0f9
{"id":"1df4ee5f-867b-470e-91c1-a47e628aa0f9","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f575cc95a90
1df4ee5f-867b-470e-91c1-a47e628aa0f9
/tidb/cdc/default/default/upstream/7366545286647601465
{"id":7366545286647601465,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/1df4ee5f-867b-470e-91c1-a47e628aa0f9
{"id":"1df4ee5f-867b-470e-91c1-a47e628aa0f9","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f575cc95a90
1df4ee5f-867b-470e-91c1-a47e628aa0f9
/tidb/cdc/default/default/upstream/7366545286647601465
{"id":7366545286647601465,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ config_path=conf/changefeed-redo.toml
+ SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1'
+ run_cdc_cli changefeed create --start-ts=449618290768084993 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.52400.out cli changefeed create --start-ts=449618290768084993 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml
Create changefeed successfully!
ID: test-1
Info: {"upstream_id":7366545286647601465,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-08T16:41:08.165589638+08:00","start_ts":449618290768084993,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-4-gbe6459908","resolved_ts":449618290768084993,"checkpoint_ts":449618290768084993,"checkpoint_time":"2024-05-08 16:41:04.368"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
+ set +x
+ run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);'
+ check_table_exists test.t1 127.0.0.1 3306
table test.t1 not exists for 1-th check, retry later
table test.t1 exists
+ sleep 5
+ kill_tikv
++ ps aux
++ grep tikv-server
++ grep /tmp/tidb_cdc_test/synced_status
+ info='jenkins 51402 6.8 0.3 3171628 1448716 ? Sl 16:40 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20160 --status-addr 127.0.0.1:20181 --log-file /tmp/tidb_cdc_test/synced_status/tikv1.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv1
jenkins 51403 6.9 0.3 3158312 1449720 ? Sl 16:40 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20161 --status-addr 127.0.0.1:20182 --log-file /tmp/tidb_cdc_test/synced_status/tikv2.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv2
jenkins 51404 6.8 0.3 3168556 1448424 ? Sl 16:40 0:04 tikv-server --pd 127.0.0.1:2379 -A 127.0.0.1:20162 --status-addr 127.0.0.1:20183 --log-file /tmp/tidb_cdc_test/synced_status/tikv3.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv3
jenkins 51406 7.0 0.3 3144488 1444932 ? Sl 16:40 0:04 tikv-server --pd 127.0.0.1:2479 -A 127.0.0.1:21160 --status-addr 127.0.0.1:21180 --log-file /tmp/tidb_cdc_test/synced_status/tikv_down.log --log-level debug -C /tmp/tidb_cdc_test/synced_status/tikv-config.toml -s /tmp/tidb_cdc_test/synced_status/tikv_down'
++ ps aux
++ grep tikv-server
++ grep /tmp/tidb_cdc_test/synced_status
++ awk '{print $2}'
++ xargs kill -9
++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 243 100 243 0 0 5080 0 --:--:-- --:--:-- --:--:-- 5170
+ synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-08 16:42:01.269","puller_resolved_ts":"2024-05-08 16:42:00.268","last_synced_ts":"2024-05-08 16:41:57.468","now_ts":"2024-05-08 16:42:04.000","info":"The data syncing is not finished, please wait"}'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-08' '16:42:01.269","puller_resolved_ts":"2024-05-08' '16:42:00.268","last_synced_ts":"2024-05-08' '16:41:57.468","now_ts":"2024-05-08' '16:42:04.000","info":"The' data syncing is not finished, please 'wait"}'
++ jq .synced
+ status=false
+ '[' false '!=' false ']'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-08' '16:42:01.269","puller_resolved_ts":"2024-05-08' '16:42:00.268","last_synced_ts":"2024-05-08' '16:41:57.468","now_ts":"2024-05-08' '16:42:04.000","info":"The' data syncing is not finished, please 'wait"}'
++ jq -r .info
+ info='The data syncing is not finished, please wait'
+ target_message='The data syncing is not finished, please wait'
+ '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']'
+ sleep 130
++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 723 100 723 0 0 10667 0 --:--:-- --:--:-- --:--:-- 10791
+ synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-08 16:42:04.268","puller_resolved_ts":"2024-05-08 16:42:04.268","last_synced_ts":"2024-05-08 16:41:57.468","now_ts":"2024-05-08 16:44:14.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-08' '16:42:04.268","puller_resolved_ts":"2024-05-08' '16:42:04.268","last_synced_ts":"2024-05-08' '16:41:57.468","now_ts":"2024-05-08' '16:44:14.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}'
++ jq .synced
+ status=false
+ '[' false '!=' false ']'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-08' '16:42:04.268","puller_resolved_ts":"2024-05-08' '16:42:04.268","last_synced_ts":"2024-05-08' '16:41:57.468","now_ts":"2024-05-08' '16:44:14.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}'
++ jq -r .info
+ info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait'
+ target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait'
+ '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']'
+ cleanup_process cdc.test
cdc.test: no process found
wait process cdc.test exit for 1-th time...
process cdc.test already exit
+ stop_tidb_cluster
+ run_case_with_unavailable_tidb conf/changefeed-redo.toml
+ rm -rf /tmp/tidb_cdc_test/synced_status
+ mkdir -p /tmp/tidb_cdc_test/synced_status
+ start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
The 1 times to try to start tidb cluster...
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
start tidb cluster in /tmp/tidb_cdc_test/synced_status
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d810850000e Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:53446, start at 2024-05-08 16:44:37.280409065 +0800 CST m=+4.421125611 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:46:37.287 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:44:37.268 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:34:37.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d810850000e Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:53446, start at 2024-05-08 16:44:37.280409065 +0800 CST m=+4.421125611 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:46:37.287 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:44:37.268 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:34:37.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d810924000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:53533, start at 2024-05-08 16:44:37.335869284 +0800 CST m=+4.398081965 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:46:37.341 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:44:37.321 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:34:37.321 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v6.5.8-7-g0be447c11
Edition: Community
Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa
Git Branch: HEAD
UTC Build Time: 2024-03-30 14:02:19
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a
Git Commit Branch: HEAD
UTC Build Time: 2024-03-30 14:05:17
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log
Starting CDC state checker...
# github.com/pingcap/tiflow/tests/utils/cdc_state_checker
./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState)
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
start tidb cluster failed
The 2 times to try to start tidb cluster...
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
start tidb cluster in /tmp/tidb_cdc_test/synced_status
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d810850000e Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:53446, start at 2024-05-08 16:44:37.280409065 +0800 CST m=+4.421125611 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:46:37.287 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:44:37.268 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:34:37.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d810850000e Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:53446, start at 2024-05-08 16:44:37.280409065 +0800 CST m=+4.421125611 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:46:37.287 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:44:37.268 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:34:37.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d810924000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:53533, start at 2024-05-08 16:44:37.335869284 +0800 CST m=+4.398081965 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:46:37.341 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:44:37.321 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:34:37.321 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%'
start tidb cluster failed
The 3 times to try to start tidb cluster...
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
start tidb cluster in /tmp/tidb_cdc_test/synced_status
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d810850000e Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:53446, start at 2024-05-08 16:44:37.280409065 +0800 CST m=+4.421125611 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:46:37.287 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:44:37.268 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:34:37.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d810850000e Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:53446, start at 2024-05-08 16:44:37.280409065 +0800 CST m=+4.421125611 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:46:37.287 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:44:37.268 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:34:37.268 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d810924000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:53533, start at 2024-05-08 16:44:37.335869284 +0800 CST m=+4.398081965 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:46:37.341 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:44:37.321 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:34:37.321 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%'
start tidb cluster failed
+ cd /tmp/tidb_cdc_test/synced_status
++ run_cdc_cli_tso_query 127.0.0.1 2379
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.57646.out cli tso query --pd=http://127.0.0.1:2379
+ set +x
+ tso='449618358391537665
PASS
coverage: 1.9% of statements in github.com/pingcap/tiflow/...'
+ echo 449618358391537665 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
+ start_ts=449618358391537665
+ run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test
[Wed May 8 16:45:22 CST 2024] <<<<<< START cdc server in synced_status case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ [[ no != \n\o ]]
+ GO_FAILPOINTS=
+ (( i = 0 ))
+ (( i <= 50 ))
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.5769557697.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Wed, 08 May 2024 08:45:25 GMT
< Content-Length: 612
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/3db823de-e199-4718-9f8d-50d182964046
{"id":"3db823de-e199-4718-9f8d-50d182964046","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f5760c90b90
3db823de-e199-4718-9f8d-50d182964046
/tidb/cdc/default/default/upstream/7366546956092724543
{"id":7366546956092724543,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/3db823de-e199-4718-9f8d-50d182964046
{"id":"3db823de-e199-4718-9f8d-50d182964046","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f5760c90b90
3db823de-e199-4718-9f8d-50d182964046
/tidb/cdc/default/default/upstream/7366546956092724543
{"id":7366546956092724543,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/3db823de-e199-4718-9f8d-50d182964046
{"id":"3db823de-e199-4718-9f8d-50d182964046","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f5760c90b90
3db823de-e199-4718-9f8d-50d182964046
/tidb/cdc/default/default/upstream/7366546956092724543
{"id":7366546956092724543,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ config_path=conf/changefeed-redo.toml
+ SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1'
+ run_cdc_cli changefeed create --start-ts=449618358391537665 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.57760.out cli changefeed create --start-ts=449618358391537665 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml
Create changefeed successfully!
ID: test-1
Info: {"upstream_id":7366546956092724543,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-08T16:45:26.052523127+08:00","start_ts":449618358391537665,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-4-gbe6459908","resolved_ts":449618358391537665,"checkpoint_ts":449618358391537665,"checkpoint_time":"2024-05-08 16:45:22.331"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
+ set +x
+ run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);'
+ check_table_exists test.t1 127.0.0.1 3306
table test.t1 not exists for 1-th check, retry later
table test.t1 exists
+ sleep 5
+ kill_tidb
++ ps aux
++ grep tidb-server
++ grep /tmp/tidb_cdc_test/synced_status
+ info='jenkins 57489 0.9 0.0 2124052 137312 ? Sl 16:45 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1715157919945352255.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log
jenkins 57493 0.7 0.0 1926356 114064 ? Sl 16:45 0:00 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1715157919948643275.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log
jenkins 57585 1.0 0.0 2279200 141528 ? Sl 16:45 0:00 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1715157920030331420.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log'
++ ps aux
++ grep tidb-server
++ grep /tmp/tidb_cdc_test/synced_status
++ awk '{print $2}'
++ xargs kill -9
++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 243 100 243 0 0 10464 0 --:--:-- --:--:-- --:--:-- 10565
+ synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-08 16:46:23.531","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-08 16:46:19.130","now_ts":"2024-05-08 16:46:26.000","info":"The data syncing is not finished, please wait"}'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-08' '16:46:23.531","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-08' '16:46:19.130","now_ts":"2024-05-08' '16:46:26.000","info":"The' data syncing is not finished, please 'wait"}'
++ jq .synced
+ status=false
+ '[' false '!=' false ']'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-08' '16:46:23.531","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-08' '16:46:19.130","now_ts":"2024-05-08' '16:46:26.000","info":"The' data syncing is not finished, please 'wait"}'
++ jq -r .info
+ info='The data syncing is not finished, please wait'
+ target_message='The data syncing is not finished, please wait'
+ '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']'
+ sleep 130
++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 221 100 221 0 0 5972 0 --:--:-- --:--:-- --:--:-- 6138
+ synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-08 16:48:33.631","puller_resolved_ts":"2024-05-08 16:48:27.631","last_synced_ts":"2024-05-08 16:46:19.130","now_ts":"2024-05-08 16:48:36.000","info":"Data syncing is finished"}'
++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-08' '16:48:33.631","puller_resolved_ts":"2024-05-08' '16:48:27.631","last_synced_ts":"2024-05-08' '16:46:19.130","now_ts":"2024-05-08' '16:48:36.000","info":"Data' syncing is 'finished"}'
++ jq .synced
+ status=true
+ '[' true '!=' true ']'
++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-08' '16:48:33.631","puller_resolved_ts":"2024-05-08' '16:48:27.631","last_synced_ts":"2024-05-08' '16:46:19.130","now_ts":"2024-05-08' '16:48:36.000","info":"Data' syncing is 'finished"}'
++ jq -r .info
+ info='Data syncing is finished'
+ target_message='Data syncing is finished'
+ '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']'
+ cleanup_process cdc.test
cdc.test: no process found
wait process cdc.test exit for 1-th time...
process cdc.test already exit
+ stop_tidb_cluster
+ run_case_with_failpoint conf/changefeed-redo.toml
+ rm -rf /tmp/tidb_cdc_test/synced_status
+ mkdir -p /tmp/tidb_cdc_test/synced_status
+ start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
The 1 times to try to start tidb cluster...
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
start tidb cluster in /tmp/tidb_cdc_test/synced_status
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9130340015 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:58905, start at 2024-05-08 16:49:01.993737617 +0800 CST m=+4.257273827 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:51:01.999 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:49:01.965 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:39:01.965 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9130340015 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:58905, start at 2024-05-08 16:49:01.993737617 +0800 CST m=+4.257273827 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:51:01.999 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:49:01.965 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:39:01.965 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9130f00016 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:59001, start at 2024-05-08 16:49:02.044412267 +0800 CST m=+4.211714494 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:51:02.050 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:49:02.012 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:39:02.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v6.5.8-7-g0be447c11
Edition: Community
Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa
Git Branch: HEAD
UTC Build Time: 2024-03-30 14:02:19
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a
Git Commit Branch: HEAD
UTC Build Time: 2024-03-30 14:05:17
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log
Starting CDC state checker...
# github.com/pingcap/tiflow/tests/utils/cdc_state_checker
./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState)
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
start tidb cluster failed
The 2 times to try to start tidb cluster...
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
start tidb cluster in /tmp/tidb_cdc_test/synced_status
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9130340015 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:58905, start at 2024-05-08 16:49:01.993737617 +0800 CST m=+4.257273827 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:51:01.999 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:49:01.965 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:39:01.965 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9130340015 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:58905, start at 2024-05-08 16:49:01.993737617 +0800 CST m=+4.257273827 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:51:01.999 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:49:01.965 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:39:01.965 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9130f00016 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:59001, start at 2024-05-08 16:49:02.044412267 +0800 CST m=+4.211714494 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:51:02.050 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:49:02.012 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:39:02.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%'
start tidb cluster failed
The 3 times to try to start tidb cluster...
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
start tidb cluster in /tmp/tidb_cdc_test/synced_status
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9130340015 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:58905, start at 2024-05-08 16:49:01.993737617 +0800 CST m=+4.257273827 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:51:01.999 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:49:01.965 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:39:01.965 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9130340015 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:58905, start at 2024-05-08 16:49:01.993737617 +0800 CST m=+4.257273827 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:51:01.999 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:49:01.965 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:39:01.965 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9130f00016 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:59001, start at 2024-05-08 16:49:02.044412267 +0800 CST m=+4.211714494 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:51:02.050 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:49:02.012 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:39:02.012 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%'
start tidb cluster failed
+ cd /tmp/tidb_cdc_test/synced_status
+ export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)'
+ GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)'
++ run_cdc_cli_tso_query 127.0.0.1 2379
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.63012.out cli tso query --pd=http://127.0.0.1:2379
+ set +x
+ tso='449618428174532609
PASS
coverage: 1.9% of statements in github.com/pingcap/tiflow/...'
+ echo 449618428174532609 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
+ start_ts=449618428174532609
+ run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test
[Wed May 8 16:49:48 CST 2024] <<<<<< START cdc server in synced_status case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ [[ no != \n\o ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.6305663058.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Wed, 08 May 2024 08:49:51 GMT
< Content-Length: 612
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/9eed1ed1-0183-43e0-b9b9-40bb93f361e5
{"id":"9eed1ed1-0183-43e0-b9b9-40bb93f361e5","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f5764d9a18e
9eed1ed1-0183-43e0-b9b9-40bb93f361e5
/tidb/cdc/default/default/upstream/7366548091157381909
{"id":7366548091157381909,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/9eed1ed1-0183-43e0-b9b9-40bb93f361e5
{"id":"9eed1ed1-0183-43e0-b9b9-40bb93f361e5","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f5764d9a18e
9eed1ed1-0183-43e0-b9b9-40bb93f361e5
/tidb/cdc/default/default/upstream/7366548091157381909
{"id":7366548091157381909,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/9eed1ed1-0183-43e0-b9b9-40bb93f361e5
{"id":"9eed1ed1-0183-43e0-b9b9-40bb93f361e5","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f5764d9a18e
9eed1ed1-0183-43e0-b9b9-40bb93f361e5
/tidb/cdc/default/default/upstream/7366548091157381909
{"id":7366548091157381909,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ config_path=conf/changefeed-redo.toml
+ SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1'
+ run_cdc_cli changefeed create --start-ts=449618428174532609 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.63119.out cli changefeed create --start-ts=449618428174532609 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml
Create changefeed successfully!
ID: test-1
Info: {"upstream_id":7366548091157381909,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-08T16:49:52.215050229+08:00","start_ts":449618428174532609,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v6.5.9-4-gbe6459908","resolved_ts":449618428174532609,"checkpoint_ts":449618428174532609,"checkpoint_time":"2024-05-08 16:49:48.532"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
+ set +x
+ sleep 20
++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 723 100 723 0 0 21203 0 --:--:-- --:--:-- --:--:-- 21264
+ synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-08 16:49:48.532","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-08 16:50:12.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-08' '16:49:48.532","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-08' '16:50:12.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}'
++ jq .synced
+ status=false
+ '[' false '!=' false ']'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-08' '16:49:48.532","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-08' '16:50:12.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}'
++ jq -r .info
+ info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait'
+ target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait'
+ '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']'
+ export GO_FAILPOINTS=
+ GO_FAILPOINTS=
+ cleanup_process cdc.test
cdc.test: no process found
wait process cdc.test exit for 1-th time...
process cdc.test already exit
+ stop_tidb_cluster
+ check_logs /tmp/tidb_cdc_test/synced_status
++ date
+ echo '[Wed May 8 16:50:25 CST 2024] <<<<<< run test case synced_status success! >>>>>>'
[Wed May 8 16:50:25 CST 2024] <<<<<< run test case synced_status success! >>>>>>
+ stop_tidb_cluster
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/run.sh using Sink-Type: mysql... <<=================
The 1 times to try to start tidb cluster...
start tidb cluster in /tmp/tidb_cdc_test/syncpoint
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d974414000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:64079, start at 2024-05-08 16:50:41.554772747 +0800 CST m=+4.228292137 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:52:41.561 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:50:41.541 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:40:41.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d974414000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:64079, start at 2024-05-08 16:50:41.554772747 +0800 CST m=+4.228292137 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:52:41.561 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:50:41.541 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:40:41.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9743e80004 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:64178, start at 2024-05-08 16:50:41.534593125 +0800 CST m=+4.104663012 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:52:41.540 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:50:41.530 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:40:41.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v6.5.8-7-g0be447c11
Edition: Community
Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa
Git Branch: HEAD
UTC Build Time: 2024-03-30 14:02:19
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a
Git Commit Branch: HEAD
UTC Build Time: 2024-03-30 14:05:17
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/syncpoint/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/syncpoint/tiflash/log/error.log
Starting CDC state checker...
# github.com/pingcap/tiflow/tests/utils/cdc_state_checker
./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState)
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
start tidb cluster failed
The 2 times to try to start tidb cluster...
start tidb cluster in /tmp/tidb_cdc_test/syncpoint
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d974414000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:64079, start at 2024-05-08 16:50:41.554772747 +0800 CST m=+4.228292137 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:52:41.561 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:50:41.541 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:40:41.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d974414000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:64079, start at 2024-05-08 16:50:41.554772747 +0800 CST m=+4.228292137 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:52:41.561 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:50:41.541 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:40:41.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9743e80004 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:64178, start at 2024-05-08 16:50:41.534593125 +0800 CST m=+4.104663012 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:52:41.540 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:50:41.530 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:40:41.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%'
start tidb cluster failed
The 3 times to try to start tidb cluster...
start tidb cluster in /tmp/tidb_cdc_test/syncpoint
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d974414000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:64079, start at 2024-05-08 16:50:41.554772747 +0800 CST m=+4.228292137 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:52:41.561 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:50:41.541 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:40:41.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d974414000f Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:64079, start at 2024-05-08 16:50:41.554772747 +0800 CST m=+4.228292137 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:52:41.561 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:50:41.541 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:40:41.541 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5d9743e80004 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:64178, start at 2024-05-08 16:50:41.534593125 +0800 CST m=+4.104663012 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:52:41.540 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:50:41.530 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:40:41.530 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%'
start tidb cluster failed
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.68357.out cli tso query --pd=http://127.0.0.1:2379
+ set +x
+ tso='449618469676646401
PASS
coverage: 1.9% of statements in github.com/pingcap/tiflow/...'
+ echo 449618469676646401 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
[Wed May 8 16:52:27 CST 2024] <<<<<< START cdc server in syncpoint case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/DDLSinkInitializeSlowly=return(true)'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ [[ no != \n\o ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.6839968401.out server --log-file /tmp/tidb_cdc_test/syncpoint/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/syncpoint/cdc_data --cluster-id default
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Wed, 08 May 2024 08:52:30 GMT
< Content-Length: 612
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/395c6abc-4e27-4ebe-b4c8-6df46e34ac6e
{"id":"395c6abc-4e27-4ebe-b4c8-6df46e34ac6e","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f5766605f0f
395c6abc-4e27-4ebe-b4c8-6df46e34ac6e
/tidb/cdc/default/default/upstream/7366548526246790976
{"id":7366548526246790976,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/395c6abc-4e27-4ebe-b4c8-6df46e34ac6e
{"id":"395c6abc-4e27-4ebe-b4c8-6df46e34ac6e","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f5766605f0f
395c6abc-4e27-4ebe-b4c8-6df46e34ac6e
/tidb/cdc/default/default/upstream/7366548526246790976
{"id":7366548526246790976,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/395c6abc-4e27-4ebe-b4c8-6df46e34ac6e
{"id":"395c6abc-4e27-4ebe-b4c8-6df46e34ac6e","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f5766605f0f
395c6abc-4e27-4ebe-b4c8-6df46e34ac6e
/tidb/cdc/default/default/upstream/7366548526246790976
{"id":7366548526246790976,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.syncpoint.cli.68469.out cli changefeed create --start-ts=449618469676646401 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/syncpoint/conf/changefeed.toml
Create changefeed successfully!
ID: 30dcdcff-daf6-48db-8385-6de966e73ef3
Info: {"upstream_id":7366548526246790976,"namespace":"default","id":"30dcdcff-daf6-48db-8385-6de966e73ef3","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-08T16:52:30.593130295+08:00","start_ts":449618469676646401,"config":{"memory_quota":1073741824,"case_sensitive":false,"enable_old_value":true,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":true,"bdr_mode":false,"sync_point_interval":30000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"],"event_filters":null},"mounter":{"worker_num":16},"sink":{"protocol":"","schema_registry":"","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"column_selectors":null,"transaction_atomicity":"","encoder_concurrency":16,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"file_index_width":0,"kafka_config":null,"advance_timeout":150},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v6.5.9-4-gbe6459908","resolved_ts":449618469676646401,"checkpoint_ts":449618469676646401,"checkpoint_time":"2024-05-08 16:52:26.850"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
+ set +x
***************** properties *****************
"requestdistribution"="uniform"
"mysql.db"="testSync"
"scanproportion"="0"
"workload"="core"
"recordcount"="10"
"updateproportion"="0"
"dotransactions"="false"
"readallfields"="true"
"readproportion"="0"
"mysql.user"="root"
"operationcount"="0"
"mysql.host"="127.0.0.1"
"mysql.port"="4000"
"insertproportion"="0"
"threadcount"="2"
**********************************************
Run finished, takes 9.476086ms
INSERT - Takes(s): 0.0, Count: 10, OPS: 2040.8, Avg(us): 1818, Min(us): 1027, Max(us): 4495, 95th(us): 5000, 99th(us): 5000
***************** properties *****************
"dotransactions"="false"
"recordcount"="10"
"updateproportion"="0"
"readproportion"="0"
"mysql.user"="root"
"scanproportion"="0"
"insertproportion"="0"
"requestdistribution"="uniform"
"mysql.db"="testSync"
"readallfields"="true"
"operationcount"="0"
"workload"="core"
"mysql.port"="4000"
"mysql.host"="127.0.0.1"
"threadcount"="2"
**********************************************
Run finished, takes 3.844612ms
INSERT - Takes(s): 0.0, Count: 10, OPS: 3928.6, Avg(us): 675, Min(us): 449, Max(us): 1470, 95th(us): 2000, 99th(us): 2000
***************** properties *****************
"threadcount"="2"
"operationcount"="0"
"workload"="core"
"mysql.db"="testSync"
"mysql.host"="127.0.0.1"
"readproportion"="0"
"dotransactions"="false"
"readallfields"="true"
"updateproportion"="0"
"recordcount"="10"
"requestdistribution"="uniform"
"mysql.user"="root"
"scanproportion"="0"
"mysql.port"="4000"
"insertproportion"="0"
**********************************************
Run finished, takes 4.305263ms
INSERT - Takes(s): 0.0, Count: 10, OPS: 4055.0, Avg(us): 794, Min(us): 478, Max(us): 1820, 95th(us): 2000, 99th(us): 2000
table testSync.usertable exists
table testSync.simple1 exists
table testSync.simple2 exists
____________________________________
*************************** 1. row ***************************
primary_ts: 449618469676646401
secondary_ts: 449618472008941581
*************************** 2. row ***************************
primary_ts: 449618477540966400
secondary_ts: 449618477737050113
*************************** 3. row ***************************
primary_ts: 449618485405286400
secondary_ts: 449618485601107969
*************************** 4. row ***************************
primary_ts: 449618493269606400
secondary_ts: 449618493465427969
*************************** 5. row ***************************
primary_ts: 449618501133926400
secondary_ts: 449618501355962369
*************************** 6. row ***************************
primary_ts: 449618508998246400
secondary_ts: 449618509220282369
*************************** 7. row ***************************
primary_ts: 449618516862566400
secondary_ts: 449618517084602369
*************************** 8. row ***************************
primary_ts: 449618524726886400
secondary_ts: 449618524975136769
*************************** 9. row ***************************
primary_ts: 449618532591206400
secondary_ts: 449618532839456769
skip invalid syncpoint primary_ts: 449618469676646401, first_ddl_ts: 449618472468217859
check diff successfully
check diff successfully
check diff successfully
check diff successfully
check diff successfully
check diff successfully
check diff successfully
check diff successfully
check diff successfully
cdc.test: no process found
wait process cdc.test exit for 1-th time...
process cdc.test already exit
[Wed May 8 16:56:32 CST 2024] <<<<<< run test case syncpoint success! >>>>>>
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh using Sink-Type: mysql... <<=================
The 1 times to try to start tidb cluster...
start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5dae39a40016 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:70615, start at 2024-05-08 16:56:57.738317598 +0800 CST m=+4.257736780 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:58:57.744 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:56:57.705 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:46:57.705 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5dae39a40016 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:70615, start at 2024-05-08 16:56:57.738317598 +0800 CST m=+4.257736780 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:58:57.744 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:56:57.705 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:46:57.705 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5dae39980009 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:70707, start at 2024-05-08 16:56:57.71047691 +0800 CST m=+4.164271127 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:58:57.717 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:56:57.702 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:46:57.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v6.5.8-7-g0be447c11
Edition: Community
Git Commit Hash: 0be447c1167454e092fadf228aa2ab11cef3f6aa
Git Branch: HEAD
UTC Build Time: 2024-03-30 14:02:19
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 37eb1f1a09576a984f07a04fe7d9f2373935657a
Git Commit Branch: HEAD
UTC Build Time: 2024-03-30 14:05:17
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/tidb_mysql_test/tiflash/log/error.log
Starting CDC state checker...
# github.com/pingcap/tiflow/tests/utils/cdc_state_checker
./cdc_monitor.go:82:52: cannot use initState (variable of type *cdcReactorState) as type orchestrator.ReactorState in argument to orchestrator.NewEtcdWorker:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:32:12: impossible type assertion: state.(*cdcReactorState)
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:34:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:40:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:46:10: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
./reactor.go:49:9: cannot use r.state (variable of type *cdcReactorState) as type orchestrator.ReactorState in return statement:
*cdcReactorState does not implement orchestrator.ReactorState (missing UpdatePendingChange method)
start tidb cluster failed
The 2 times to try to start tidb cluster...
start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5dae39a40016 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:70615, start at 2024-05-08 16:56:57.738317598 +0800 CST m=+4.257736780 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:58:57.744 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:56:57.705 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:46:57.705 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5dae39a40016 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:70615, start at 2024-05-08 16:56:57.738317598 +0800 CST m=+4.257736780 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:58:57.744 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:56:57.705 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:46:57.705 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5dae39980009 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:70707, start at 2024-05-08 16:56:57.71047691 +0800 CST m=+4.164271127 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:58:57.717 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:56:57.702 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:46:57.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%'
start tidb cluster failed
The 3 times to try to start tidb cluster...
start tidb cluster in /tmp/tidb_cdc_test/tidb_mysql_test
Starting Upstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Starting Downstream PD...
Release Version: v6.5.9-1-g0ecafcb36
Edition: Community
Git Commit Hash: 0ecafcb3657b51a1ae14f7ca79d5aa6175d70371
Git Branch: release-6.5
UTC Build Time: 2024-04-26 06:25:13
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 6.5.9
Edition: Community
Git Commit Hash: 172a25c1447d3cca370becc0cd7ceda40cda65d5
Git Commit Branch: release-6.5
UTC Build Time: 2024-04-25 09:35:48
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v6.5.9-7-gfe6837a8c9
Edition: Community
Git Commit Hash: fe6837a8c93ff800bef169a9f2514e14f8655fdc
Git Branch: release-6.5
UTC Build Time: 2024-05-06 09:02:13
GoVersion: go1.19.13
Race Enabled: false
TiKV Min Version: 6.2.0-alpha
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5dae39a40016 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:70615, start at 2024-05-08 16:56:57.738317598 +0800 CST m=+4.257736780 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:58:57.744 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:56:57.705 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:46:57.705 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5dae39a40016 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:70615, start at 2024-05-08 16:56:57.738317598 +0800 CST m=+4.257736780 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:58:57.744 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:56:57.705 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:46:57.705 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 110 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
tikv_gc_leader_uuid 63d5dae39980009 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:pingcap-tiflow-release-6-5-pull-cdc-integration-test-170--00p27, pid:70707, start at 2024-05-08 16:56:57.71047691 +0800 CST m=+4.164271127 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240508-16:58:57.717 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 60m All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240508-16:56:57.702 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240508-16:46:57.702 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 1396 (HY000) at line 1: Operation CREATE USER failed for 'normal'@'%'
start tidb cluster failed
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.cli.74766.out cli tso query --pd=http://127.0.0.1:2379
+ set +x
+ tso='449618553469403138
PASS
coverage: 1.9% of statements in github.com/pingcap/tiflow/...'
+ echo 449618553469403138 PASS coverage: 1.9% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
[Wed May 8 16:57:46 CST 2024] <<<<<< START cdc server in tidb_mysql_test case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ GO_FAILPOINTS=
+ [[ no != \n\o ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.tidb_mysql_test.7481674818.out server --log-file /tmp/tidb_cdc_test/tidb_mysql_test/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/tidb_mysql_test/cdc_data --cluster-id default
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Wed, 08 May 2024 08:57:49 GMT
< Content-Length: 612
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/82aaad2d-fddd-4349-86a8-de6218baf238
{"id":"82aaad2d-fddd-4349-86a8-de6218baf238","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f576c241f8e
82aaad2d-fddd-4349-86a8-de6218baf238
/tidb/cdc/default/default/upstream/7366550135788635824
{"id":7366550135788635824,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/82aaad2d-fddd-4349-86a8-de6218baf238
{"id":"82aaad2d-fddd-4349-86a8-de6218baf238","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f576c241f8e
82aaad2d-fddd-4349-86a8-de6218baf238
/tidb/cdc/default/default/upstream/7366550135788635824
{"id":7366550135788635824,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/82aaad2d-fddd-4349-86a8-de6218baf238
{"id":"82aaad2d-fddd-4349-86a8-de6218baf238","address":"127.0.0.1:8300","version":"v6.5.9-4-gbe6459908"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f576c241f8e
82aaad2d-fddd-4349-86a8-de6218baf238
/tidb/cdc/default/default/upstream/7366550135788635824
{"id":7366550135788635824,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 142 0 0 100 142 0 1716 --:--:-- --:--:-- --:--:-- 1731
mysql_test start
go: downloading github.com/maxshuang/mysql-tester v0.0.2
go: downloading github.com/sirupsen/logrus v1.8.1
go: downloading github.com/pingcap/tidb/parser v0.0.0-20211122065349-4d16aabf900b
go: downloading github.com/pingcap/errors v0.11.5-0.20211009033009-93128226aaa3
go: downloading go.uber.org/atomic v1.9.0
go: downloading golang.org/x/sys v0.0.0-20211117180635-dee7805ff2e1
go: downloading go.uber.org/zap v1.19.1
go: downloading github.com/pingcap/log v0.0.0-20210906054005-afc726e70354
go: downloading golang.org/x/text v0.3.7
go: downloading github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0
go: downloading go.uber.org/multierr v1.7.0
build mysql_test successfully
run all mysql test cases
time="2024-05-08T16:58:08+08:00" level=info msg="running tests: [alter_table bigint composite_index concurrent_ddl date_formats drop mysql_replace]"
time="2024-05-08T16:58:08+08:00" level=warning msg="Create new db&{0 0xc000126000 0 {0 0} [0xc000210510] map[] 0 1 0xc00012a000 false map[0xc000210510:map[0xc000210510:true]] map[] 0 0 0 0 <nil> 0 0 0 0 0x48dd80}"
./t/alter_table.test: ok! 70 test cases passed, take time 6.299703453 s
time="2024-05-08T16:58:50+08:00" level=info msg="run test [alter_table] ok"
time="2024-05-08T16:58:50+08:00" level=warning msg="Create new db&{0 0xc000314380 0 {0 0} [0xc00012c480] map[] 0 1 0xc000300900 false map[0xc00012c480:map[0xc00012c480:true]] map[] 0 0 0 0 <nil> 0 0 0 0 0x48dd80}"
./t/bigint.test: ok! 138 test cases passed, take time 4.847756653 s
time="2024-05-08T16:58:55+08:00" level=info msg="run test [bigint] ok"
time="2024-05-08T16:58:55+08:00" level=warning msg="Create new db&{0 0xc0003140c0 0 {0 0} [0xc000210870] map[] 0 1 0xc000300660 false map[0xc000210870:map[0xc000210870:true]] map[] 0 0 0 0 <nil> 0 0 0 0 0x48dd80}"
./t/composite_index.test: ok! 12 test cases passed, take time 0.248689557 s
time="2024-05-08T16:58:55+08:00" level=info msg="run test [composite_index] ok"
time="2024-05-08T16:58:55+08:00" level=warning msg="Create new db&{0 0xc000014150 0 {0 0} [0xc00012c3f0] map[] 0 1 0xc0000387e0 false map[0xc00012c3f0:map[0xc00012c3f0:true]] map[] 0 0 0 0 <nil> 0 0 0 0 0x48dd80}"
./t/concurrent_ddl.test: ok! 8 test cases passed, take time 0.587103468 s
time="2024-05-08T16:58:56+08:00" level=info msg="run test [concurrent_ddl] ok"
time="2024-05-08T16:58:56+08:00" level=warning msg="Create new db&{0 0xc00060e050 0 {0 0} [0xc00031c120] map[] 0 1 0xc000606120 false map[0xc00031c120:map[0xc00031c120:true]] map[] 0 0 0 0 <nil> 0 0 0 0 0x48dd80}"
./t/date_formats.test: ok! 99 test cases passed, take time 0.971326952 s
time="2024-05-08T16:58:57+08:00" level=info msg="run test [date_formats] ok"
time="2024-05-08T16:58:57+08:00" level=warning msg="Create new db&{0 0xc000126028 0 {0 0} [0xc000416120] map[] 0 1 0xc0005a60c0 false map[0xc000416120:map[0xc000416120:true]] map[] 0 0 0 0 <nil> 0 0 0 0 0x48dd80}"
./t/drop.test: ok! 48 test cases passed, take time 1.922654541 s
time="2024-05-08T16:58:59+08:00" level=info msg="run test [drop] ok"
time="2024-05-08T16:58:59+08:00" level=warning msg="Create new db&{0 0xc000308118 0 {0 0} [0xc0004162d0] map[] 0 1 0xc000606cc0 false map[0xc0004162d0:map[0xc0004162d0:true]] map[] 0 0 0 0 <nil> 0 0 0 0 0x48dd80}"
./t/mysql_replace.test: ok! 16 test cases passed, take time 0.172603906 s
time="2024-05-08T16:58:59+08:00" level=info msg="run test [mysql_replace] ok"
Great, All tests passed
mysqltest end
table test.finish_mark not exists for 1-th check, retry later
table test.finish_mark not exists for 2-th check, retry later
table test.finish_mark not exists for 3-th check, retry later
table test.finish_mark not exists for 4-th check, retry later
table test.finish_mark not exists for 5-th check, retry later
table test.finish_mark not exists for 6-th check, retry later
table test.finish_mark not exists for 7-th check, retry later
Cancelling nested steps due to timeout
Sending interrupt signal to process
Killing processes
kill finished with exit code 0
Sending interrupt signal to process
Killing processes
kill finished with exit code 0
/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh: line 1: 76472 Terminated check_table_exists test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 300
/home/jenkins/agent/workspace/pingcap/tiflow/release-6.5/pull_cdc_integration_test/tiflow/tests/integration_tests/tidb_mysql_test/run.sh: line 1: 76516 Terminated check_table_exists test.finish_mark ${DOWN_TIDB_HOST} ${DOWN_TIDB_PORT} 300
script returned exit code 143
[Pipeline] }
Cache not saved (inner-step execution failed)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
Failed in branch Matrix - TEST_GROUP = 'G15'
[Pipeline] // parallel
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] End of Pipeline
ERROR: script returned exit code 1
Finished: FAILURE
![[Jenkins]](/jenkins/static/62b7ca6c/images/svgs/logo.svg){kind=logo}
