* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 10 May 2024 04:16:41 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/813037fc-8153-4cb0-9543-8fb2218c8a42
{"id":"813037fc-8153-4cb0-9543-8fb2218c8a42","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60b75fb7f7
813037fc-8153-4cb0-9543-8fb2218c8a42
/tidb/cdc/default/default/upstream/7367220039831963496
{"id":7367220039831963496,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/813037fc-8153-4cb0-9543-8fb2218c8a42
{"id":"813037fc-8153-4cb0-9543-8fb2218c8a42","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60b75fb7f7
813037fc-8153-4cb0-9543-8fb2218c8a42
/tidb/cdc/default/default/upstream/7367220039831963496
{"id":7367220039831963496,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/813037fc-8153-4cb0-9543-8fb2218c8a42
{"id":"813037fc-8153-4cb0-9543-8fb2218c8a42","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60b75fb7f7
813037fc-8153-4cb0-9543-8fb2218c8a42
/tidb/cdc/default/default/upstream/7367220039831963496
{"id":7367220039831963496,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ config_path=conf/changefeed-redo.toml
+ SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1'
+ run_cdc_cli changefeed create --start-ts=449659429765251073 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.31593.out cli changefeed create --start-ts=449659429765251073 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml
Create changefeed successfully!
ID: test-1
Info: {"upstream_id":7367220039831963496,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-10T12:16:42.079432613+08:00","start_ts":449659429765251073,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-28-gdd60f55b4","resolved_ts":449659429765251073,"checkpoint_ts":449659429765251073,"checkpoint_time":"2024-05-10 12:16:37.188"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
Cache restored successfully (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-578/tiflow-cdc)
3367684096 bytes in 5.96 secs (564974257 bytes/sec)
[Pipeline] {
[Pipeline] container
[Pipeline] {
[Pipeline] timeout
Timeout set to expire in 6 min 0 sec
[Pipeline] {
+ set +x
+ run_sql 'USE TEST;Create table t1(a int primary key, b int);insert into t1 values(1,2);insert into t1 values(2,3);'
[Pipeline] sh
+ echo Waiting for zookeeper to be ready...
Waiting for zookeeper to be ready...
+ nc -z localhost 2181
+ echo Waiting for kafka to be ready...
Waiting for kafka to be ready...
+ nc -z localhost 9092
+ echo Waiting for kafka-broker to be ready...
Waiting for kafka-broker to be ready...
+ echo dump
+ nc localhost 2181
+ grep brokers
+ awk {$1=$1;print}
+ grep -F -w /brokers/ids/1
/brokers/ids/1
+ check_table_exists test.t1 127.0.0.1 3306
table test.t1 not exists for 1-th check, retry later
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // container
[Pipeline] sh
+ rm -rf /tmp/tidb_cdc_test
+ mkdir -p /tmp/tidb_cdc_test
+ chmod +x ./tests/integration_tests/run_group.sh
+ ./tests/integration_tests/run_group.sh kafka G02
Run cases: consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table kafka_big_txn_v2 kafka_big_messages_v2 multi_tables_ddl_v2 multi_topics_v2 storage_cleanup csv_storage_basic csv_storage_multi_tables_ddl csv_storage_partition_table
PROW_JOB_ID=15c60a16-0828-4c1d-98a3-5adbca2c8986
JENKINS_NODE_COOKIE=6a95b286-ed7c-4526-9c92-127869882cad
BUILD_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/578/
GOLANG_VERSION=1.21.0
HOSTNAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70
HUDSON_SERVER_COOKIE=83ef27fe9acccc92
KUBERNETES_PORT=tcp://10.233.0.1:443
KUBERNETES_PORT_443_TCP_PORT=443
TERM=xterm
STAGE_NAME=Test
BUILD_TAG=jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-578
KUBERNETES_SERVICE_PORT=443
GIT_PREVIOUS_COMMIT=863d41c24385fea97437f7929cc9999da9cc38a6
JOB_SPEC={"type":"presubmit","job":"pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test","buildid":"1788774560061984768","prowjobid":"15c60a16-0828-4c1d-98a3-5adbca2c8986","refs":{"org":"pingcap","repo":"tiflow","base_ref":"release-7.5","base_sha":"06ba7aeaac3d69ead86fd48e1b8a0a08f13ff25e","pulls":[{"number":11000,"author":"ti-chi-bot","sha":"b417a77d3bfa2c9235cbca0d58dab40f95c1ca50","title":"codec(ticdc): add more logs to the avro encoding to help detect potential key encoding failed issure (#10901)"}]}}
KUBERNETES_SERVICE_HOST=10.233.0.1
WORKSPACE=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test
JOB_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/
RUN_CHANGES_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/578/display/redirect?page=changes
RUN_ARTIFACTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/578/display/redirect?page=artifacts
FILE_SERVER_URL=http://fileserver.pingcap.net
JENKINS_HOME=/var/jenkins_home
GIT_COMMIT=1a8e788f78e7e981ce0f02f2d020372bd9713cdd
PATH=/go/bin:/usr/local/go/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/_utils:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../bin:/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/../../scripts/bin
RUN_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/578/display/redirect
GOPROXY=http://goproxy.apps.svc,https://proxy.golang.org,direct
POD_CONTAINER=golang
PWD=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow
HUDSON_URL=https://do.pingcap.net/jenkins/
TICDC_COVERALLS_TOKEN=****
JOB_NAME=pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test
TZ=Asia/Shanghai
BUILD_DISPLAY_NAME=#578
TEST_GROUP=G02
JENKINS_URL=https://do.pingcap.net/jenkins/
BUILD_ID=1788774560061984768
TICDC_CODECOV_TOKEN=****
GOLANG_DOWNLOAD_SHA256=d0398903a16ba2232b389fb31032ddf57cac34efda306a0eebac34f0965a0742
JOB_BASE_NAME=pull_cdc_integration_kafka_test
GIT_PREVIOUS_SUCCESSFUL_COMMIT=863d41c24385fea97437f7929cc9999da9cc38a6
RUN_TESTS_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/578/display/redirect?page=tests
SHLVL=5
HOME=/home/jenkins
POD_LABEL=ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_578-5gxtv
GOROOT=/usr/local/go
GIT_BRANCH=origin/main
KUBERNETES_PORT_443_TCP_PROTO=tcp
TINI_VERSION=v0.19.0
CI=true
KUBERNETES_SERVICE_PORT_HTTPS=443
WORKSPACE_TMP=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test@tmp
EXECUTOR_NUMBER=0
JENKINS_SERVER_COOKIE=durable-596db45d714d4dc1c94a71a261cdbfe8374c069ecbc68b0020035d4750de0595
NODE_LABELS=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70 ap_tiflow_release-7_5_pull_cdc_integration_kafka_test_578-5gxtv
GIT_URL=https://github.com/PingCAP-QE/ci.git
HUDSON_HOME=/var/jenkins_home
CLASSPATH=
NODE_NAME=ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70
GOPATH=/go
JOB_DISPLAY_URL=https://do.pingcap.net/jenkins/job/pingcap/job/tiflow/job/release-7.5/job/pull_cdc_integration_kafka_test/display/redirect
BUILD_NUMBER=578
KUBERNETES_PORT_443_TCP_ADDR=10.233.0.1
KUBERNETES_PORT_443_TCP=tcp://10.233.0.1:443
GOLANG_DOWNLOAD_URL=https://dl.google.com/go/go1.21.0.linux-amd64.tar.gz
_=/usr/bin/env
find: '/tmp/tidb_cdc_test/*/*': No such file or directory
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_ddl/run.sh using Sink-Type: kafka... <<=================
[Fri May 10 12:16:44 CST 2024] <<<<<< run test case consistent_replicate_ddl success! >>>>>>
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
table test.t1 exists
+ sleep 5
find: '/tmp/tidb_cdc_test/*/*': No such file or directory
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_gbk/run.sh using Sink-Type: kafka... <<=================
* About to connect() to 127.0.0.1 port 24927 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:24927; Connection refused
* Closing connection 0
You are running an older version of MinIO released 3 years ago
Update: Run `mc admin update`
Attempting encryption of all config, IAM users and policies on MinIO backend
Endpoint: http://127.0.0.1:24927
Object API (Amazon S3 compatible):
Go: https://docs.min.io/docs/golang-client-quickstart-guide
Java: https://docs.min.io/docs/java-client-quickstart-guide
Python: https://docs.min.io/docs/python-client-quickstart-guide
JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide
.NET: https://docs.min.io/docs/dotnet-client-quickstart-guide
* About to connect() to 127.0.0.1 port 24927 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0)
> GET / HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:24927
> Accept: */*
>
< HTTP/1.1 403 Forbidden
< Accept-Ranges: bytes
< Content-Length: 226
< Content-Security-Policy: block-all-mixed-content
< Content-Type: application/xml
< Server: MinIO/RELEASE.2020-07-27T18-37-02Z
< Vary: Origin
< X-Amz-Request-Id: 17CE058CB81D74F5
< X-Xss-Protection: 1; mode=block
< Date: Fri, 10 May 2024 04:16:50 GMT
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
Bucket 's3://logbucket/' created
[Fri May 10 12:16:50 CST 2024] <<<<<< run test case consistent_replicate_gbk success! >>>>>>
Exiting on signal: INTERRUPT
+ kill_tidb
++ ps aux
++ grep tidb-server
++ grep /tmp/tidb_cdc_test/synced_status
+ info='jenkins 30134 3.7 0.0 2581928 164388 ? Sl 12:16 0:00 tidb-server -P 4000 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1715314586533484120.toml --store tikv --path 127.0.0.1:2379 --status=10080 --log-file /tmp/tidb_cdc_test/synced_status/tidb.log
jenkins 30138 11.2 0.0 2377956 187116 ? Sl 12:16 0:02 tidb-server -P 4001 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1715314586536393530.toml --store tikv --path 127.0.0.1:2379 --status=10081 --log-file /tmp/tidb_cdc_test/synced_status/tidb_other.log
jenkins 30211 11.8 0.0 2353464 214876 ? Sl 12:16 0:02 tidb-server -P 3306 -config /tmp/tidb_cdc_test/synced_status/tidb-config-1715314586634619323.toml --store tikv --path 127.0.0.1:2479 --status=20080 --log-file /tmp/tidb_cdc_test/synced_status/tidb_down.log'
++ ps aux
++ grep tidb-server
++ grep /tmp/tidb_cdc_test/synced_status
++ awk '{print $2}'
++ xargs kill -9
++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 243 100 243 0 0 3126 0 --:--:-- --:--:-- --:--:-- 3155
+ synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-10 12:16:49.637","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"2024-05-10 12:16:43.987","now_ts":"2024-05-10 12:16:51.000","info":"The data syncing is not finished, please wait"}'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-10' '12:16:49.637","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-10' '12:16:43.987","now_ts":"2024-05-10' '12:16:51.000","info":"The' data syncing is not finished, please 'wait"}'
++ jq .synced
+ status=false
+ '[' false '!=' false ']'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-10' '12:16:49.637","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"2024-05-10' '12:16:43.987","now_ts":"2024-05-10' '12:16:51.000","info":"The' data syncing is not finished, please 'wait"}'
++ jq -r .info
+ info='The data syncing is not finished, please wait'
+ target_message='The data syncing is not finished, please wait'
+ '[' 'The data syncing is not finished, please wait' '!=' 'The data syncing is not finished, please wait' ']'
+ sleep 130
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_nfs/run.sh using Sink-Type: kafka... <<=================
[Fri May 10 12:16:57 CST 2024] <<<<<< run test case consistent_replicate_nfs success! >>>>>>
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
find: '/tmp/tidb_cdc_test/*/*': No such file or directory
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file/run.sh using Sink-Type: kafka... <<=================
[Fri May 10 12:17:00 CST 2024] <<<<<< run test case consistent_replicate_storage_file success! >>>>>>
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
find: '/tmp/tidb_cdc_test/*/*': No such file or directory
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_file_large_value/run.sh using Sink-Type: kafka... <<=================
[Fri May 10 12:17:03 CST 2024] <<<<<< run test case consistent_replicate_storage_file_large_value success! >>>>>>
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
find: '/tmp/tidb_cdc_test/*/*': No such file or directory
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_replicate_storage_s3/run.sh using Sink-Type: kafka... <<=================
* About to connect() to 127.0.0.1 port 24927 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:24927; Connection refused
* Closing connection 0
You are running an older version of MinIO released 3 years ago
Update: Run `mc admin update`
Attempting encryption of all config, IAM users and policies on MinIO backend
Endpoint: http://127.0.0.1:24927
Object API (Amazon S3 compatible):
Go: https://docs.min.io/docs/golang-client-quickstart-guide
Java: https://docs.min.io/docs/java-client-quickstart-guide
Python: https://docs.min.io/docs/python-client-quickstart-guide
JavaScript: https://docs.min.io/docs/javascript-client-quickstart-guide
.NET: https://docs.min.io/docs/dotnet-client-quickstart-guide
* About to connect() to 127.0.0.1 port 24927 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 24927 (#0)
> GET / HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:24927
> Accept: */*
>
< HTTP/1.1 403 Forbidden
< Accept-Ranges: bytes
< Content-Length: 226
< Content-Security-Policy: block-all-mixed-content
< Content-Type: application/xml
< Server: MinIO/RELEASE.2020-07-27T18-37-02Z
< Vary: Origin
< X-Amz-Request-Id: 17CE0590F8E4CD7F
< X-Xss-Protection: 1; mode=block
< Date: Fri, 10 May 2024 04:17:08 GMT
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
Bucket 's3://logbucket/' created
[Fri May 10 12:17:09 CST 2024] <<<<<< run test case consistent_replicate_storage_s3 success! >>>>>>
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
Exiting on signal: INTERRUPT
ERROR: Failed to launch ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-8gkh5
io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-8gkh5] in namespace [jenkins-tiflow].
at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939)
at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921)
at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97)
at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185)
at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297)
at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46)
at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/consistent_partition_table/run.sh using Sink-Type: kafka... <<=================
[Fri May 10 12:17:12 CST 2024] <<<<<< run test case consistent_partition_table success! >>>>>>
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
find: '/tmp/tidb_cdc_test/*/*': No such file or directory
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_txn_v2/run.sh using Sink-Type: kafka... <<=================
The 1 times to try to start tidb cluster...
start tidb cluster in /tmp/tidb_cdc_test/kafka_big_txn_v2
Starting Upstream PD...
Release Version: v7.5.1-6-g78f4254e3
Edition: Community
Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815
Git Branch: release-7.5
UTC Build Time: 2024-04-30 02:49:46
Starting Downstream PD...
Release Version: v7.5.1-6-g78f4254e3
Edition: Community
Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815
Git Branch: release-7.5
UTC Build Time: 2024-04-30 02:49:46
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: a0b1254aafa44634f4c96d339fb95f0180d1353f
Git Commit Branch: release-7.5
UTC Build Time: 2024-05-09 15:32:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: a0b1254aafa44634f4c96d339fb95f0180d1353f
Git Commit Branch: release-7.5
UTC Build Time: 2024-05-09 15:32:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v7.5.1-59-g12c763f3d2
Edition: Community
Git Commit Hash: 12c763f3d25b9cef659215d03bc73da78abe3c66
Git Branch: release-7.5
UTC Build Time: 2024-05-10 03:19:25
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v7.5.1-59-g12c763f3d2
Edition: Community
Git Commit Hash: 12c763f3d25b9cef659215d03bc73da78abe3c66
Git Branch: release-7.5
UTC Build Time: 2024-05-10 03:19:25
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82e1a5300012 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:1827, start at 2024-05-10 12:17:29.446600481 +0800 CST m=+5.239496011 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:19:29.454 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:17:29.420 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:07:29.420 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82e1a5300012 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:1827, start at 2024-05-10 12:17:29.446600481 +0800 CST m=+5.239496011 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:19:29.454 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:17:29.420 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:07:29.420 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82e1a83c0004 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:1963, start at 2024-05-10 12:17:29.618581677 +0800 CST m=+5.272999397 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:19:29.626 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:17:29.615 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:07:29.615 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-18-g0ebf3734b
Edition: Community
Git Commit Hash: 0ebf3734bec7d0c7c12e7b2d6c1565e9d6166e86
Git Branch: HEAD
UTC Build Time: 2024-05-08 07:04:56
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time: 2024-05-08 07:09:50
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/log/error.log
arg matches is ArgMatches { args: {"log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/log/proxy.log"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["0ebf3734bec7d0c7c12e7b2d6c1565e9d6166e86"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash/db/proxy"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_txn_v2/tiflash-proxy.toml"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-18-g0ebf3734b"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
[Fri May 10 12:17:32 CST 2024] <<<<<< START cdc server in kafka_big_txn_v2 case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ [[ no != \n\o ]]
+ GO_FAILPOINTS=
+ (( i = 0 ))
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.31753177.out server --log-file /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc_data --cluster-id default
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 10 May 2024 04:17:35 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/dac86305-4176-41ce-8046-82962c62c827
{"id":"dac86305-4176-41ce-8046-82962c62c827","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60b83eb2f3
dac86305-4176-41ce-8046-82962c62c827
/tidb/cdc/default/default/upstream/7367220278145716041
{"id":7367220278145716041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/dac86305-4176-41ce-8046-82962c62c827
{"id":"dac86305-4176-41ce-8046-82962c62c827","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60b83eb2f3
dac86305-4176-41ce-8046-82962c62c827
/tidb/cdc/default/default/upstream/7367220278145716041
{"id":7367220278145716041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/dac86305-4176-41ce-8046-82962c62c827
{"id":"dac86305-4176-41ce-8046-82962c62c827","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60b83eb2f3
dac86305-4176-41ce-8046-82962c62c827
/tidb/cdc/default/default/upstream/7367220278145716041
{"id":7367220278145716041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.cli.3223.out cli tso query --pd=http://127.0.0.1:2379
+ set +x
+ tso='449659445187969027
PASS
coverage: 1.8% of statements in github.com/pingcap/tiflow/...'
+ echo 449659445187969027 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
[Fri May 10 12:17:37 CST 2024] <<<<<< START cdc server in kafka_big_txn_v2 case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ [[ no != \n\o ]]
+ GO_FAILPOINTS=
+ (( i = 0 ))
+ (( i <= 50 ))
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.32583260.out server --log-file /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_txn_v2/cdc_data --cluster-id default
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 10 May 2024 04:17:37 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/dac86305-4176-41ce-8046-82962c62c827
{"id":"dac86305-4176-41ce-8046-82962c62c827","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60b83eb2f3
dac86305-4176-41ce-8046-82962c62c827
/tidb/cdc/default/default/upstream/7367220278145716041
{"id":7367220278145716041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/dac86305-4176-41ce-8046-82962c62c827
{"id":"dac86305-4176-41ce-8046-82962c62c827","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60b83eb2f3
dac86305-4176-41ce-8046-82962c62c827
/tidb/cdc/default/default/upstream/7367220278145716041
{"id":7367220278145716041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/dac86305-4176-41ce-8046-82962c62c827
{"id":"dac86305-4176-41ce-8046-82962c62c827","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60b83eb2f3
dac86305-4176-41ce-8046-82962c62c827
/tidb/cdc/default/default/upstream/7367220278145716041
{"id":7367220278145716041,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_txn_v2.cli.3285.out cli changefeed create --start-ts=449659445187969027 '--sink-uri=kafka://127.0.0.1:9092/ticdc-big-txn-test-773?protocol=open-protocol&partition-num=4&kafka-version=2.4.1&max-message-bytes=10485760' --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_txn_v2/conf/changefeed.toml
Create changefeed successfully!
ID: 96af48d1-d579-4bc1-aad6-d4182ce9ce6b
Info: {"upstream_id":7367220278145716041,"namespace":"default","id":"96af48d1-d579-4bc1-aad6-d4182ce9ce6b","sink_uri":"kafka://127.0.0.1:9092/ticdc-big-txn-test-773?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-10T12:17:38.163279331+08:00","start_ts":449659445187969027,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-28-gdd60f55b4","resolved_ts":449659445187969027,"checkpoint_ts":449659445187969027,"checkpoint_time":"2024-05-10 12:17:36.021"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
+ set +x
***************** properties *****************
"mysql.host"="127.0.0.1"
"fieldcount"="100"
"workload"="core"
"mysql.db"="big_txn"
"operationcount"="0"
"updateproportion"="0"
"readproportion"="0"
"recordcount"="5000"
"dotransactions"="false"
"scanproportion"="0"
"mysql.port"="4000"
"readallfields"="true"
"requestdistribution"="uniform"
"insertproportion"="0"
"mysql.user"="root"
"threadcount"="1"
**********************************************
INSERT - Takes(s): 10.0, Count: 2371, OPS: 236.9, Avg(us): 3929, Min(us): 1323, Max(us): 94458, 95th(us): 4000, 99th(us): 87000
INSERT - Takes(s): 20.0, Count: 4923, OPS: 246.0, Avg(us): 3751, Min(us): 1271, Max(us): 163567, 95th(us): 4000, 99th(us): 82000
Run finished, takes 20.33580989s
INSERT - Takes(s): 20.3, Count: 5000, OPS: 245.9, Avg(us): 3769, Min(us): 1271, Max(us): 163567, 95th(us): 4000, 99th(us): 82000
[Fri May 10 12:18:00 CST 2024] <<<<<< START kafka consumer in kafka_big_txn_v2 case >>>>>>
table big_txn.usertable not exists for 1-th check, retry later
table big_txn.usertable not exists for 2-th check, retry later
table big_txn.usertable exists
++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 221 100 221 0 0 2682 0 --:--:-- --:--:-- --:--:-- 2662
+ synced_status='{"synced":true,"sink_checkpoint_ts":"2024-05-10 12:18:59.787","puller_resolved_ts":"2024-05-10 12:18:53.787","last_synced_ts":"2024-05-10 12:16:43.987","now_ts":"2024-05-10 12:19:01.000","info":"Data syncing is finished"}'
++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-10' '12:18:59.787","puller_resolved_ts":"2024-05-10' '12:18:53.787","last_synced_ts":"2024-05-10' '12:16:43.987","now_ts":"2024-05-10' '12:19:01.000","info":"Data' syncing is 'finished"}'
++ jq .synced
+ status=true
+ '[' true '!=' true ']'
++ echo '{"synced":true,"sink_checkpoint_ts":"2024-05-10' '12:18:59.787","puller_resolved_ts":"2024-05-10' '12:18:53.787","last_synced_ts":"2024-05-10' '12:16:43.987","now_ts":"2024-05-10' '12:19:01.000","info":"Data' syncing is 'finished"}'
++ jq -r .info
+ info='Data syncing is finished'
+ target_message='Data syncing is finished'
+ '[' 'Data syncing is finished' '!=' 'Data syncing is finished' ']'
+ cleanup_process cdc.test
wait process cdc.test exit for 1-th time...
wait process cdc.test exit for 2-th time...
cdc.test: no process found
wait process cdc.test exit for 3-th time...
process cdc.test already exit
+ stop_tidb_cluster
+ run_case_with_failpoint conf/changefeed-redo.toml
+ rm -rf /tmp/tidb_cdc_test/synced_status
+ mkdir -p /tmp/tidb_cdc_test/synced_status
+ start_tidb_cluster --workdir /tmp/tidb_cdc_test/synced_status
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
The 1 times to try to start tidb cluster...
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
shell-init: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
chdir: error retrieving current directory: getcwd: cannot access parent directories: No such file or directory
start tidb cluster in /tmp/tidb_cdc_test/synced_status
Starting Upstream PD...
Release Version: v7.5.1-6-g78f4254e3
Edition: Community
Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815
Git Branch: release-7.5
UTC Build Time: 2024-04-30 02:49:46
Starting Downstream PD...
Release Version: v7.5.1-6-g78f4254e3
Edition: Community
Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815
Git Branch: release-7.5
UTC Build Time: 2024-04-30 02:49:46
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: a0b1254aafa44634f4c96d339fb95f0180d1353f
Git Commit Branch: release-7.5
UTC Build Time: 2024-05-09 15:32:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: a0b1254aafa44634f4c96d339fb95f0180d1353f
Git Commit Branch: release-7.5
UTC Build Time: 2024-05-09 15:32:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v7.5.1-59-g12c763f3d2
Edition: Community
Git Commit Hash: 12c763f3d25b9cef659215d03bc73da78abe3c66
Git Branch: release-7.5
UTC Build Time: 2024-05-10 03:19:25
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v7.5.1-59-g12c763f3d2
Edition: Community
Git Commit Hash: 12c763f3d25b9cef659215d03bc73da78abe3c66
Git Branch: release-7.5
UTC Build Time: 2024-05-10 03:19:25
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82e8a7540003 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-7xv8g, pid:32674, start at 2024-05-10 12:19:24.249152272 +0800 CST m=+5.962256821 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:21:24.262 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:19:24.245 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:09:24.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82e8a7540003 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-7xv8g, pid:32674, start at 2024-05-10 12:19:24.249152272 +0800 CST m=+5.962256821 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:21:24.262 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:19:24.245 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:09:24.245 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82e8a8a40014 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-7xv8g, pid:32755, start at 2024-05-10 12:19:24.373962764 +0800 CST m=+5.943132306 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:21:24.384 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:19:24.380 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:09:24.380 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-18-g0ebf3734b
Edition: Community
Git Commit Hash: 0ebf3734bec7d0c7c12e7b2d6c1565e9d6166e86
Git Branch: HEAD
UTC Build Time: 2024-05-08 07:04:56
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time: 2024-05-08 07:09:50
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/synced_status/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/synced_status/tiflash/log/error.log
arg matches is ArgMatches { args: {"engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-18-g0ebf3734b"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/db/proxy"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/synced_status/tiflash/log/proxy.log"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["0ebf3734bec7d0c7c12e7b2d6c1565e9d6166e86"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
+ cd /tmp/tidb_cdc_test/synced_status
+ export 'GO_FAILPOINTS=github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)'
+ GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)'
++ run_cdc_cli_tso_query 127.0.0.1 2379
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.33962.out cli tso query --pd=http://127.0.0.1:2379
+ set +x
+ tso='449659474816270337
PASS
coverage: 1.8% of statements in github.com/pingcap/tiflow/...'
+ echo 449659474816270337 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
+ start_ts=449659474816270337
+ run_cdc_server --workdir /tmp/tidb_cdc_test/synced_status --binary cdc.test
[Fri May 10 12:19:30 CST 2024] <<<<<< START cdc server in synced_status case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ GO_FAILPOINTS='github.com/pingcap/tiflow/cdc/owner/ChangefeedOwnerNotUpdateCheckpoint=return(true)'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.3398633988.out server --log-file /tmp/tidb_cdc_test/synced_status/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/synced_status/cdc_data --cluster-id default
+ [[ no != \n\o ]]
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 10 May 2024 04:19:33 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/8a825897-70eb-4ac3-a613-652df314629e
{"id":"8a825897-70eb-4ac3-a613-652df314629e","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60ba0096f8
8a825897-70eb-4ac3-a613-652df314629e
/tidb/cdc/default/default/upstream/7367220779116437898
{"id":7367220779116437898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/8a825897-70eb-4ac3-a613-652df314629e
{"id":"8a825897-70eb-4ac3-a613-652df314629e","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60ba0096f8
8a825897-70eb-4ac3-a613-652df314629e
/tidb/cdc/default/default/upstream/7367220779116437898
{"id":7367220779116437898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/8a825897-70eb-4ac3-a613-652df314629e
{"id":"8a825897-70eb-4ac3-a613-652df314629e","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60ba0096f8
8a825897-70eb-4ac3-a613-652df314629e
/tidb/cdc/default/default/upstream/7367220779116437898
{"id":7367220779116437898,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ config_path=conf/changefeed-redo.toml
+ SINK_URI='mysql://root@127.0.0.1:3306/?max-txn-row=1'
+ run_cdc_cli changefeed create --start-ts=449659474816270337 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.synced_status.cli.34032.out cli changefeed create --start-ts=449659474816270337 '--sink-uri=mysql://root@127.0.0.1:3306/?max-txn-row=1' --changefeed-id=test-1 --config=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/synced_status/conf/changefeed-redo.toml
Create changefeed successfully!
ID: test-1
Info: {"upstream_id":7367220779116437898,"namespace":"default","id":"test-1","sink_uri":"mysql://root@127.0.0.1:3306/?max-txn-row=1","create_time":"2024-05-10T12:19:34.034441403+08:00","start_ts":449659474816270337,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":false,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"eventual","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"storage":"file:///tmp/tidb_cdc_test/synced_status/redo","use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":120,"checkpoint_interval":20}},"state":"normal","creator_version":"v7.5.1-28-gdd60f55b4","resolved_ts":449659474816270337,"checkpoint_ts":449659474816270337,"checkpoint_time":"2024-05-10 12:19:29.044"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
+ set +x
+ sleep 20
++ curl -X GET http://127.0.0.1:8300/api/v2/changefeeds/test-1/synced
% Total % Received % Xferd Average Speed Time Time Time Current
Dload Upload Total Spent Left Speed
0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0
100 723 100 723 0 0 6703 0 --:--:-- --:--:-- --:--:-- 6757
+ synced_status='{"synced":false,"sink_checkpoint_ts":"2024-05-10 12:19:29.044","puller_resolved_ts":"1970-01-01 08:00:00.000","last_synced_ts":"1970-01-01 08:00:00.000","now_ts":"2024-05-10 12:19:55.000","info":"Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' \u003e '\''Resolved-Ts'\'' \u003e '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait"}'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-10' '12:19:29.044","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-10' '12:19:55.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}'
++ jq .synced
+ status=false
+ '[' false '!=' false ']'
++ echo '{"synced":false,"sink_checkpoint_ts":"2024-05-10' '12:19:29.044","puller_resolved_ts":"1970-01-01' '08:00:00.000","last_synced_ts":"1970-01-01' '08:00:00.000","now_ts":"2024-05-10' '12:19:55.000","info":"Please' check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view ''\''TiKV-Details'\''' '\u003e' ''\''Resolved-Ts'\''' '\u003e' ''\''Max' Leader Resolved TS 'gap'\''' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please 'wait"}'
++ jq -r .info
+ info='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait'
+ target_message='Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait'
+ '[' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' '!=' 'Please check whether PD is online and TiKV Regions are all available. If PD is offline or some TiKV regions are not available, it means that the data syncing process is complete. To check whether TiKV regions are all available, you can view '\''TiKV-Details'\'' > '\''Resolved-Ts'\'' > '\''Max Leader Resolved TS gap'\'' on Grafana. If the gap is large, such as a few minutes, it means that some regions in TiKV are unavailable. Otherwise, if the gap is small and PD is online, it means the data syncing is incomplete, so please wait' ']'
+ export GO_FAILPOINTS=
+ GO_FAILPOINTS=
+ cleanup_process cdc.test
wait process cdc.test exit for 1-th time...
wait process cdc.test exit for 2-th time...
cdc.test: no process found
wait process cdc.test exit for 3-th time...
process cdc.test already exit
+ stop_tidb_cluster
+ check_logs /tmp/tidb_cdc_test/synced_status
++ date
+ echo '[Fri May 10 12:20:06 CST 2024] <<<<<< run test case synced_status success! >>>>>>'
[Fri May 10 12:20:06 CST 2024] <<<<<< run test case synced_status success! >>>>>>
+ stop_tidb_cluster
\033[0;36m<<< Run all test success >>>\033[0m
[Pipeline] }
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-578/tiflow-cdc already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
table big_txn.finish_mark exists
check diff successfully
wait process cdc.test exit for 1-th time...
wait process cdc.test exit for 2-th time...
wait process cdc.test exit for 3-th time...
process cdc.test already exit
[Fri May 10 12:20:08 CST 2024] <<<<<< run test case kafka_big_txn_v2 success! >>>>>>
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/kafka_big_messages_v2/run.sh using Sink-Type: kafka... <<=================
The 1 times to try to start tidb cluster...
start tidb cluster in /tmp/tidb_cdc_test/kafka_big_messages_v2
Starting Upstream PD...
Release Version: v7.5.1-6-g78f4254e3
Edition: Community
Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815
Git Branch: release-7.5
UTC Build Time: 2024-04-30 02:49:46
Starting Downstream PD...
Release Version: v7.5.1-6-g78f4254e3
Edition: Community
Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815
Git Branch: release-7.5
UTC Build Time: 2024-04-30 02:49:46
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: a0b1254aafa44634f4c96d339fb95f0180d1353f
Git Commit Branch: release-7.5
UTC Build Time: 2024-05-09 15:32:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: a0b1254aafa44634f4c96d339fb95f0180d1353f
Git Commit Branch: release-7.5
UTC Build Time: 2024-05-09 15:32:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v7.5.1-59-g12c763f3d2
Edition: Community
Git Commit Hash: 12c763f3d25b9cef659215d03bc73da78abe3c66
Git Branch: release-7.5
UTC Build Time: 2024-05-10 03:19:25
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v7.5.1-59-g12c763f3d2
Edition: Community
Git Commit Hash: 12c763f3d25b9cef659215d03bc73da78abe3c66
Git Branch: release-7.5
UTC Build Time: 2024-05-10 03:19:25
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82ed12000013 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:4496, start at 2024-05-10 12:20:36.646945681 +0800 CST m=+5.670885252 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:22:36.654 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:20:36.658 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:10:36.658 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82ed12000013 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:4496, start at 2024-05-10 12:20:36.646945681 +0800 CST m=+5.670885252 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:22:36.654 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:20:36.658 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:10:36.658 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82ed13740013 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:4574, start at 2024-05-10 12:20:36.726508525 +0800 CST m=+5.588060372 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:22:36.735 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:20:36.701 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:10:36.701 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-18-g0ebf3734b
Edition: Community
Git Commit Hash: 0ebf3734bec7d0c7c12e7b2d6c1565e9d6166e86
Git Branch: HEAD
UTC Build Time: 2024-05-08 07:04:56
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time: 2024-05-08 07:09:50
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/error.log
arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["0ebf3734bec7d0c7c12e7b2d6c1565e9d6166e86"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash-proxy.toml"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/kafka_big_messages_v2/tiflash/log/proxy.log"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-18-g0ebf3734b"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
[Fri May 10 12:20:41 CST 2024] <<<<<< START cdc server in kafka_big_messages_v2 case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ GO_FAILPOINTS=
+ [[ no != \n\o ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.kafka_big_messages_v2.58995901.out server --log-file /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/kafka_big_messages_v2/cdc_data --cluster-id default
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 10 May 2024 04:20:44 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/de60a6cf-bbcf-4d3d-a240-64b5e9baaa8c
{"id":"de60a6cf-bbcf-4d3d-a240-64b5e9baaa8c","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60bb1c71f1
de60a6cf-bbcf-4d3d-a240-64b5e9baaa8c
/tidb/cdc/default/default/upstream/7367221087939697852
{"id":7367221087939697852,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/de60a6cf-bbcf-4d3d-a240-64b5e9baaa8c
{"id":"de60a6cf-bbcf-4d3d-a240-64b5e9baaa8c","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60bb1c71f1
de60a6cf-bbcf-4d3d-a240-64b5e9baaa8c
/tidb/cdc/default/default/upstream/7367221087939697852
{"id":7367221087939697852,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/de60a6cf-bbcf-4d3d-a240-64b5e9baaa8c
{"id":"de60a6cf-bbcf-4d3d-a240-64b5e9baaa8c","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60bb1c71f1
de60a6cf-bbcf-4d3d-a240-64b5e9baaa8c
/tidb/cdc/default/default/upstream/7367221087939697852
{"id":7367221087939697852,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
Create changefeed successfully!
ID: ba6a639d-b5d9-4947-86bd-ed75591e37d7
Info: {"upstream_id":7367221087939697852,"namespace":"default","id":"ba6a639d-b5d9-4947-86bd-ed75591e37d7","sink_uri":"kafka://127.0.0.1:9092/big-message-test?protocol=open-protocol\u0026partition-num=1\u0026kafka-version=2.4.1\u0026max-message-bytes=12582912","create_time":"2024-05-10T12:20:44.774009377+08:00","start_ts":449659493799165959,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-28-gdd60f55b4","resolved_ts":449659493799165959,"checkpoint_ts":449659493799165959,"checkpoint_time":"2024-05-10 12:20:41.458"}
[Fri May 10 12:20:44 CST 2024] <<<<<< START kafka consumer in kafka_big_messages_v2 case >>>>>>
Starting generate kafka big messages...
go: downloading github.com/pingcap/errors v0.11.5-0.20240318064555-6bd07397691f
go: downloading go.uber.org/atomic v1.11.0
table kafka_big_messages.test not exists for 1-th check, retry later
table kafka_big_messages.test exists
check diff failed 1-th time, retry later
check diff successfully
wait process cdc.test exit for 1-th time...
wait process cdc.test exit for 2-th time...
cdc.test: no process found
wait process cdc.test exit for 3-th time...
process cdc.test already exit
[Fri May 10 12:20:58 CST 2024] <<<<<< run test case kafka_big_messages_v2 success! >>>>>>
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_tables_ddl_v2/run.sh using Sink-Type: kafka... <<=================
The 1 times to try to start tidb cluster...
start tidb cluster in /tmp/tidb_cdc_test/multi_tables_ddl_v2
Starting Upstream PD...
Release Version: v7.5.1-6-g78f4254e3
Edition: Community
Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815
Git Branch: release-7.5
UTC Build Time: 2024-04-30 02:49:46
Starting Downstream PD...
Release Version: v7.5.1-6-g78f4254e3
Edition: Community
Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815
Git Branch: release-7.5
UTC Build Time: 2024-04-30 02:49:46
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: a0b1254aafa44634f4c96d339fb95f0180d1353f
Git Commit Branch: release-7.5
UTC Build Time: 2024-05-09 15:32:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: a0b1254aafa44634f4c96d339fb95f0180d1353f
Git Commit Branch: release-7.5
UTC Build Time: 2024-05-09 15:32:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v7.5.1-59-g12c763f3d2
Edition: Community
Git Commit Hash: 12c763f3d25b9cef659215d03bc73da78abe3c66
Git Branch: release-7.5
UTC Build Time: 2024-05-10 03:19:25
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v7.5.1-59-g12c763f3d2
Edition: Community
Git Commit Hash: 12c763f3d25b9cef659215d03bc73da78abe3c66
Git Branch: release-7.5
UTC Build Time: 2024-05-10 03:19:25
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82f074e40011 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:7968, start at 2024-05-10 12:21:32.110328231 +0800 CST m=+10.276546424 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:23:32.122 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:21:32.089 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:11:32.089 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82f074e40011 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:7968, start at 2024-05-10 12:21:32.110328231 +0800 CST m=+10.276546424 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:23:32.122 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:21:32.089 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:11:32.089 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82f06b8c0013 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:8024, start at 2024-05-10 12:21:31.509887588 +0800 CST m=+5.483217136 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:23:31.516 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:21:31.491 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:11:31.491 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-18-g0ebf3734b
Edition: Community
Git Commit Hash: 0ebf3734bec7d0c7c12e7b2d6c1565e9d6166e86
Git Branch: HEAD
UTC Build Time: 2024-05-08 07:04:56
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time: 2024-05-08 07:09:50
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/error.log
arg matches is ArgMatches { args: {"engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash-proxy.toml"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/db/proxy"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["0ebf3734bec7d0c7c12e7b2d6c1565e9d6166e86"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-18-g0ebf3734b"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_tables_ddl_v2/tiflash/log/proxy.log"] }, "advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
[Fri May 10 12:21:36 CST 2024] <<<<<< START cdc server in multi_tables_ddl_v2 case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ [[ no != \n\o ]]
+ GO_FAILPOINTS=
+ (( i = 0 ))
+ (( i <= 50 ))
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_tables_ddl_v2.92689270.out server --log-file /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_tables_ddl_v2/cdc_data --cluster-id default
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 10 May 2024 04:21:39 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/943af630-fd8d-4b7b-a403-6af53ae4f991
{"id":"943af630-fd8d-4b7b-a403-6af53ae4f991","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60bbd3c104
943af630-fd8d-4b7b-a403-6af53ae4f991
/tidb/cdc/default/default/upstream/7367221287500698183
{"id":7367221287500698183,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/943af630-fd8d-4b7b-a403-6af53ae4f991
{"id":"943af630-fd8d-4b7b-a403-6af53ae4f991","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60bbd3c104
943af630-fd8d-4b7b-a403-6af53ae4f991
/tidb/cdc/default/default/upstream/7367221287500698183
{"id":7367221287500698183,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/943af630-fd8d-4b7b-a403-6af53ae4f991
{"id":"943af630-fd8d-4b7b-a403-6af53ae4f991","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60bbd3c104
943af630-fd8d-4b7b-a403-6af53ae4f991
/tidb/cdc/default/default/upstream/7367221287500698183
{"id":7367221287500698183,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
Create changefeed successfully!
ID: test-normal
Info: {"upstream_id":7367221287500698183,"namespace":"default","id":"test-normal","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-normal-29164?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-10T12:21:39.458914655+08:00","start_ts":449659508094140417,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t1","multi_tables_ddl_test.t2","multi_tables_ddl_test.t3","multi_tables_ddl_test.t4","multi_tables_ddl_test.t1_7","multi_tables_ddl_test.t2_7","multi_tables_ddl_test.finish_mark"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-28-gdd60f55b4","resolved_ts":449659508094140417,"checkpoint_ts":449659508094140417,"checkpoint_time":"2024-05-10 12:21:35.989"}
Create changefeed successfully!
ID: test-error-1
Info: {"upstream_id":7367221287500698183,"namespace":"default","id":"test-error-1","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-1-18991?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-10T12:21:39.664337645+08:00","start_ts":449659508094140417,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t5","multi_tables_ddl_test.t6","multi_tables_ddl_test.t7","multi_tables_ddl_test.t8"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-28-gdd60f55b4","resolved_ts":449659508094140417,"checkpoint_ts":449659508094140417,"checkpoint_time":"2024-05-10 12:21:35.989"}
Create changefeed successfully!
ID: test-error-2
Info: {"upstream_id":7367221287500698183,"namespace":"default","id":"test-error-2","sink_uri":"kafka://127.0.0.1:9092/ticdc-multi-tables-ddl-test-error-2-1007?protocol=open-protocol\u0026partition-num=4\u0026kafka-version=2.4.1\u0026max-message-bytes=10485760","create_time":"2024-05-10T12:21:39.86453955+08:00","start_ts":449659508094140417,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["multi_tables_ddl_test.t9","multi_tables_ddl_test.t10"]},"mounter":{"worker_num":16},"sink":{"protocol":"open-protocol","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-28-gdd60f55b4","resolved_ts":449659508094140417,"checkpoint_ts":449659508094140417,"checkpoint_time":"2024-05-10 12:21:35.989"}
[Fri May 10 12:21:39 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>>
[Fri May 10 12:21:39 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>>
[Fri May 10 12:21:39 CST 2024] <<<<<< START kafka consumer in multi_tables_ddl_v2 case >>>>>>
table multi_tables_ddl_test.t55 not exists for 1-th check, retry later
table multi_tables_ddl_test.t55 not exists for 2-th check, retry later
table multi_tables_ddl_test.t55 not exists for 3-th check, retry later
table multi_tables_ddl_test.t55 not exists for 4-th check, retry later
table multi_tables_ddl_test.t55 exists
table multi_tables_ddl_test.t66 exists
table multi_tables_ddl_test.t7 exists
table multi_tables_ddl_test.t88 exists
table multi_tables_ddl_test.finish_mark not exists for 1-th check, retry later
table multi_tables_ddl_test.finish_mark not exists for 2-th check, retry later
table multi_tables_ddl_test.finish_mark not exists for 3-th check, retry later
table multi_tables_ddl_test.finish_mark exists
check table exists success
+ endpoints=http://127.0.0.1:2379
+ changefeed_id=test-normal
+ expected_state=normal
+ error_msg=null
+ tls_dir=
+ [[ http://127.0.0.1:2379 =~ https ]]
++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-normal -s
+ info='{
"upstream_id": 7367221287500698183,
"namespace": "default",
"id": "test-normal",
"state": "normal",
"checkpoint_tso": 449659515001896965,
"checkpoint_time": "2024-05-10 12:22:02.340",
"error": null
}'
+ echo '{
"upstream_id": 7367221287500698183,
"namespace": "default",
"id": "test-normal",
"state": "normal",
"checkpoint_tso": 449659515001896965,
"checkpoint_time": "2024-05-10 12:22:02.340",
"error": null
}'
{
"upstream_id": 7367221287500698183,
"namespace": "default",
"id": "test-normal",
"state": "normal",
"checkpoint_tso": 449659515001896965,
"checkpoint_time": "2024-05-10 12:22:02.340",
"error": null
}
++ echo '{' '"upstream_id":' 7367221287500698183, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449659515001896965, '"checkpoint_time":' '"2024-05-10' '12:22:02.340",' '"error":' null '}'
++ jq -r .state
+ state=normal
+ [[ ! normal == \n\o\r\m\a\l ]]
++ echo '{' '"upstream_id":' 7367221287500698183, '"namespace":' '"default",' '"id":' '"test-normal",' '"state":' '"normal",' '"checkpoint_tso":' 449659515001896965, '"checkpoint_time":' '"2024-05-10' '12:22:02.340",' '"error":' null '}'
++ jq -r .error.message
+ message=null
+ [[ ! null =~ null ]]
+ endpoints=http://127.0.0.1:2379
+ changefeed_id=test-error-1
+ expected_state=normal
+ error_msg=null
+ tls_dir=
+ [[ http://127.0.0.1:2379 =~ https ]]
++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-1 -s
+ info='{
"upstream_id": 7367221287500698183,
"namespace": "default",
"id": "test-error-1",
"state": "normal",
"checkpoint_tso": 449659515526184969,
"checkpoint_time": "2024-05-10 12:22:04.340",
"error": null
}'
+ echo '{
"upstream_id": 7367221287500698183,
"namespace": "default",
"id": "test-error-1",
"state": "normal",
"checkpoint_tso": 449659515526184969,
"checkpoint_time": "2024-05-10 12:22:04.340",
"error": null
}'
{
"upstream_id": 7367221287500698183,
"namespace": "default",
"id": "test-error-1",
"state": "normal",
"checkpoint_tso": 449659515526184969,
"checkpoint_time": "2024-05-10 12:22:04.340",
"error": null
}
++ echo '{' '"upstream_id":' 7367221287500698183, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449659515526184969, '"checkpoint_time":' '"2024-05-10' '12:22:04.340",' '"error":' null '}'
++ jq -r .state
+ state=normal
+ [[ ! normal == \n\o\r\m\a\l ]]
++ echo '{' '"upstream_id":' 7367221287500698183, '"namespace":' '"default",' '"id":' '"test-error-1",' '"state":' '"normal",' '"checkpoint_tso":' 449659515526184969, '"checkpoint_time":' '"2024-05-10' '12:22:04.340",' '"error":' null '}'
++ jq -r .error.message
+ message=null
+ [[ ! null =~ null ]]
+ endpoints=http://127.0.0.1:2379
+ changefeed_id=test-error-2
+ expected_state=failed
+ error_msg=ErrSyncRenameTableFailed
+ tls_dir=
+ [[ http://127.0.0.1:2379 =~ https ]]
++ cdc cli changefeed query --pd=http://127.0.0.1:2379 -c test-error-2 -s
+ info='{
"upstream_id": 7367221287500698183,
"namespace": "default",
"id": "test-error-2",
"state": "failed",
"checkpoint_tso": 449659509444182059,
"checkpoint_time": "2024-05-10 12:21:41.139",
"error": {
"time": "2024-05-10T12:21:43.294858258+08:00",
"addr": "127.0.0.1:8300",
"code": "CDC:ErrHandleDDLFailed",
"message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-10 12:21:43.139 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449659509968470024. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449659509968470024]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule."
}
}'
+ echo '{
"upstream_id": 7367221287500698183,
"namespace": "default",
"id": "test-error-2",
"state": "failed",
"checkpoint_tso": 449659509444182059,
"checkpoint_time": "2024-05-10 12:21:41.139",
"error": {
"time": "2024-05-10T12:21:43.294858258+08:00",
"addr": "127.0.0.1:8300",
"code": "CDC:ErrHandleDDLFailed",
"message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-10 12:21:43.139 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449659509968470024. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449659509968470024]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule."
}
}'
{
"upstream_id": 7367221287500698183,
"namespace": "default",
"id": "test-error-2",
"state": "failed",
"checkpoint_tso": 449659509444182059,
"checkpoint_time": "2024-05-10 12:21:41.139",
"error": {
"time": "2024-05-10T12:21:43.294858258+08:00",
"addr": "127.0.0.1:8300",
"code": "CDC:ErrHandleDDLFailed",
"message": "[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-10 12:21:43.139 +0800 CST, Err:\u003cnil\u003e, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449659509968470024. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449659509968470024]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule."
}
}
++ jq -r .state
++ echo '{' '"upstream_id":' 7367221287500698183, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449659509444182059, '"checkpoint_time":' '"2024-05-10' '12:21:41.139",' '"error":' '{' '"time":' '"2024-05-10T12:21:43.294858258+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-10 12:21:43.139 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449659509968470024. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449659509968470024]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}'
+ state=failed
+ [[ ! failed == \f\a\i\l\e\d ]]
++ jq -r .error.message
++ echo '{' '"upstream_id":' 7367221287500698183, '"namespace":' '"default",' '"id":' '"test-error-2",' '"state":' '"failed",' '"checkpoint_tso":' 449659509444182059, '"checkpoint_time":' '"2024-05-10' '12:21:41.139",' '"error":' '{' '"time":' '"2024-05-10T12:21:43.294858258+08:00",' '"addr":' '"127.0.0.1:8300",' '"code":' '"CDC:ErrHandleDDLFailed",' '"message":' '"[CDC:ErrHandleDDLFailed]handle' ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-10 12:21:43.139 +0800 CST, 'Err:\u003cnil\u003e,' ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449659509968470024. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add '`ignore-txn-start-ts=[449659509968470024]`' to the changefeed in the filter configuration.: '[CDC:ErrSyncRenameTableFailed]table'\''s' old name is not in filter rule, and its new name in filter rule table id ''\''128'\'',' ddl query: '[rename' table t11 to 't9],' 'it'\''s' an unexpected behavior, if you want to replicate this table, please add its old name to filter 'rule."' '}' '}'
+ message='[CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-10 12:21:43.139 +0800 CST, Err:<nil>, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449659509968470024. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449659509968470024]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table'\''s old name is not in filter rule, and its new name in filter rule table id '\''128'\'', ddl query: [rename table t11 to t9], it'\''s an unexpected behavior, if you want to replicate this table, please add its old name to filter rule.'
+ [[ ! [CDC:ErrHandleDDLFailed]handle ddl failed, job: ID:140, Type:rename table, State:done, SchemaState:public, SchemaID:102, TableID:128, RowCount:0, ArgLen:0, start time: 2024-05-10 12:21:43.139 +0800 CST, Err:<nil>, ErrCount:0, SnapshotVersion:0, LocalMode: false, query: rename table t11 to t9, startTs: 449659509968470024. If you want to skip this DDL and continue with replication, you can manually execute this DDL downstream. Afterwards, add `ignore-txn-start-ts=[449659509968470024]` to the changefeed in the filter configuration.: [CDC:ErrSyncRenameTableFailed]table's old name is not in filter rule, and its new name in filter rule table id '128', ddl query: [rename table t11 to t9], it's an unexpected behavior, if you want to replicate this table, please add its old name to filter rule. =~ ErrSyncRenameTableFailed ]]
check diff successfully
wait process cdc.test exit for 1-th time...
wait process cdc.test exit for 2-th time...
wait process cdc.test exit for 3-th time...
cdc.test: no process found
wait process cdc.test exit for 4-th time...
process cdc.test already exit
[Fri May 10 12:22:07 CST 2024] <<<<<< run test case multi_tables_ddl_v2 success! >>>>>>
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/run.sh using Sink-Type: kafka... <<=================
The 1 times to try to start tidb cluster...
ERROR: Failed to launch ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-cxz9g
io.fabric8.kubernetes.client.KubernetesClientTimeoutException: Timed out waiting for [1000000] milliseconds for [Pod] with name:[ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-cxz9g] in namespace [jenkins-tiflow].
at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilCondition(BaseOperation.java:939)
at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:921)
at io.fabric8.kubernetes.client.dsl.internal.BaseOperation.waitUntilReady(BaseOperation.java:97)
at org.csanchez.jenkins.plugins.kubernetes.KubernetesLauncher.launch(KubernetesLauncher.java:185)
at hudson.slaves.SlaveComputer.lambda$_connect$0(SlaveComputer.java:297)
at jenkins.util.ContextResettingExecutorService$2.call(ContextResettingExecutorService.java:46)
at jenkins.security.ImpersonatingExecutorService$2.call(ImpersonatingExecutorService.java:80)
at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
at java.base/java.lang.Thread.run(Thread.java:829)
start tidb cluster in /tmp/tidb_cdc_test/multi_topics_v2
Starting Upstream PD...
Release Version: v7.5.1-6-g78f4254e3
Edition: Community
Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815
Git Branch: release-7.5
UTC Build Time: 2024-04-30 02:49:46
Starting Downstream PD...
Release Version: v7.5.1-6-g78f4254e3
Edition: Community
Git Commit Hash: 78f4254e3f5adb48e3e1e2489065f5ccf6cf1815
Git Branch: release-7.5
UTC Build Time: 2024-04-30 02:49:46
Verifying upstream PD is started...
Verifying downstream PD is started...
Starting Upstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: a0b1254aafa44634f4c96d339fb95f0180d1353f
Git Commit Branch: release-7.5
UTC Build Time: 2024-05-09 15:32:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Downstream TiKV...
TiKV
Release Version: 7.5.2
Edition: Community
Git Commit Hash: a0b1254aafa44634f4c96d339fb95f0180d1353f
Git Commit Branch: release-7.5
UTC Build Time: 2024-05-09 15:32:46
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Enable Features: pprof-fp jemalloc mem-profiling portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Profile: dist_release
Starting Upstream TiDB...
Release Version: v7.5.1-59-g12c763f3d2
Edition: Community
Git Commit Hash: 12c763f3d25b9cef659215d03bc73da78abe3c66
Git Branch: release-7.5
UTC Build Time: 2024-05-10 03:19:25
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Starting Downstream TiDB...
Release Version: v7.5.1-59-g12c763f3d2
Edition: Community
Git Commit Hash: 12c763f3d25b9cef659215d03bc73da78abe3c66
Git Branch: release-7.5
UTC Build Time: 2024-05-10 03:19:25
GoVersion: go1.21.6
Race Enabled: false
Check Table Before Drop: false
Store: unistore
Verifying Upstream TiDB is started...
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82f477300013 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:10597, start at 2024-05-10 12:22:37.813850789 +0800 CST m=+7.089039064 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:24:37.820 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:22:37.822 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:12:37.822 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
ERROR 2003 (HY000): Can't connect to MySQL server on '127.0.0.1' (111)
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82f477300013 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:10597, start at 2024-05-10 12:22:37.813850789 +0800 CST m=+7.089039064 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:24:37.820 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:22:37.822 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:12:37.822 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Verifying Downstream TiDB is started...
VARIABLE_NAME VARIABLE_VALUE COMMENT
bootstrapped True Bootstrap flag. Do not delete.
tidb_server_version 179 Bootstrap version. Do not delete.
system_tz Asia/Shanghai TiDB Global System Timezone.
new_collation_enabled True If the new collations are enabled. Do not edit it.
ddl_table_version 3 DDL Table Version. Do not delete.
tikv_gc_leader_uuid 63d82f478fc0002 Current GC worker leader UUID. (DO NOT EDIT)
tikv_gc_leader_desc host:ap-tiflow-release-7-5-pull-cdc-integration-kafka-test-578-gbh70, pid:10681, start at 2024-05-10 12:22:37.88931875 +0800 CST m=+5.455962591 Host name and pid of current GC leader. (DO NOT EDIT)
tikv_gc_leader_lease 20240510-12:24:37.896 +0800 Current GC worker leader lease. (DO NOT EDIT)
tikv_gc_auto_concurrency true Let TiDB pick the concurrency automatically. If set false, tikv_gc_concurrency will be used
tikv_gc_enable true Current GC enable status
tikv_gc_run_interval 10m0s GC run interval, at least 10m, in Go format.
tikv_gc_life_time 10m0s All versions within life time will not be collected by GC, at least 10m, in Go format.
tikv_gc_last_run_time 20240510-12:22:37.887 +0800 The time when last GC starts. (DO NOT EDIT)
tikv_gc_safe_point 20240510-12:12:37.887 +0800 All versions after safe point can be accessed. (DO NOT EDIT)
Starting Upstream TiFlash...
TiFlash
Release Version: v7.5.1-18-g0ebf3734b
Edition: Community
Git Commit Hash: 0ebf3734bec7d0c7c12e7b2d6c1565e9d6166e86
Git Branch: HEAD
UTC Build Time: 2024-05-08 07:04:56
Enable Features: jemalloc sm4(GmSSL) avx2 avx512 unwind thinlto
Profile: RELWITHDEBINFO
Raft Proxy
Git Commit Hash: 521fd9dbc55e58646045d88f91c3c35db50b5981
Git Commit Branch: HEAD
UTC Build Time: 2024-05-08 07:09:50
Rust Version: rustc 1.67.0-nightly (96ddd32c4 2022-11-14)
Storage Engine: tiflash
Prometheus Prefix: tiflash_proxy_
Profile: release
Enable Features: portable sse test-engine-kv-rocksdb test-engine-raft-raft-engine cloud-aws cloud-gcp cloud-azure
Verifying Upstream TiFlash is started...
Logging trace to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/server.log
Logging errors to /tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/error.log
arg matches is ArgMatches { args: {"advertise-addr": MatchedArg { occurs: 1, indices: [4], vals: ["127.0.0.1:9000"] }, "log-file": MatchedArg { occurs: 1, indices: [18], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/log/proxy.log"] }, "pd-endpoints": MatchedArg { occurs: 1, indices: [16], vals: ["127.0.0.1:2379"] }, "addr": MatchedArg { occurs: 1, indices: [20], vals: ["127.0.0.1:9000"] }, "data-dir": MatchedArg { occurs: 1, indices: [6], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash/db/proxy"] }, "config": MatchedArg { occurs: 1, indices: [8], vals: ["/tmp/tidb_cdc_test/multi_topics_v2/tiflash-proxy.toml"] }, "engine-addr": MatchedArg { occurs: 1, indices: [2], vals: ["127.0.0.1:9500"] }, "engine-version": MatchedArg { occurs: 1, indices: [12], vals: ["v7.5.1-18-g0ebf3734b"] }, "engine-label": MatchedArg { occurs: 1, indices: [14], vals: ["tiflash"] }, "engine-git-hash": MatchedArg { occurs: 1, indices: [10], vals: ["0ebf3734bec7d0c7c12e7b2d6c1565e9d6166e86"] }}, subcommand: None, usage: Some("USAGE:\n TiFlash Proxy [FLAGS] [OPTIONS] --engine-git-hash <engine-git-hash> --engine-label <engine-label> --engine-version <engine-version>") }
+ pd_host=127.0.0.1
+ pd_port=2379
++ run_cdc_cli tso query --pd=http://127.0.0.1:2379
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.11910.out cli tso query --pd=http://127.0.0.1:2379
+ set +x
+ tso='449659525666439169
PASS
coverage: 1.8% of statements in github.com/pingcap/tiflow/...'
+ echo 449659525666439169 PASS coverage: 1.8% of statements in github.com/pingcap/tiflow/...
+ awk -F ' ' '{print $1}'
+ set +x
[Fri May 10 12:22:44 CST 2024] <<<<<< START cdc server in multi_topics_v2 case >>>>>>
+ [[ '' == \t\r\u\e ]]
+ set +e
+ get_info_fail_msg='failed to get info:'
+ etcd_info_msg='etcd info'
+ '[' -z '' ']'
+ curl_status_cmd='curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info'
+ GO_FAILPOINTS=
+ [[ no != \n\o ]]
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.1194611948.out server --log-file /tmp/tidb_cdc_test/multi_topics_v2/cdc.log --log-level debug --data-dir /tmp/tidb_cdc_test/multi_topics_v2/cdc_data --cluster-id default
+ (( i = 0 ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connection refused
* Failed connect to 127.0.0.1:8300; Connection refused
* Closing connection 0
+ res=
+ echo ''
+ grep -q 'failed to get info:'
+ echo ''
+ grep -q 'etcd info'
+ '[' 0 -eq 50 ']'
+ sleep 3
+ (( i++ ))
+ (( i <= 50 ))
++ curl -vsL --max-time 20 http://127.0.0.1:8300/debug/info
* About to connect() to 127.0.0.1 port 8300 (#0)
* Trying 127.0.0.1...
* Connected to 127.0.0.1 (127.0.0.1) port 8300 (#0)
> GET /debug/info HTTP/1.1
> User-Agent: curl/7.29.0
> Host: 127.0.0.1:8300
> Accept: */*
>
< HTTP/1.1 200 OK
< Date: Fri, 10 May 2024 04:22:47 GMT
< Content-Length: 613
< Content-Type: text/plain; charset=utf-8
<
{ [data not shown]
* Connection #0 to host 127.0.0.1 left intact
+ res='
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/a7c34c5a-0f2a-4561-bb69-de57541911f0
{"id":"a7c34c5a-0f2a-4561-bb69-de57541911f0","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60bce74af9
a7c34c5a-0f2a-4561-bb69-de57541911f0
/tidb/cdc/default/default/upstream/7367221587711019178
{"id":7367221587711019178,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/a7c34c5a-0f2a-4561-bb69-de57541911f0
{"id":"a7c34c5a-0f2a-4561-bb69-de57541911f0","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60bce74af9
a7c34c5a-0f2a-4561-bb69-de57541911f0
/tidb/cdc/default/default/upstream/7367221587711019178
{"id":7367221587711019178,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'failed to get info:'
+ echo '
*** owner info ***:
*** processors info ***:
*** etcd info ***:
/tidb/cdc/default/__cdc_meta__/capture/a7c34c5a-0f2a-4561-bb69-de57541911f0
{"id":"a7c34c5a-0f2a-4561-bb69-de57541911f0","address":"127.0.0.1:8300","version":"v7.5.1-28-gdd60f55b4"}
/tidb/cdc/default/__cdc_meta__/meta/meta-version
1
/tidb/cdc/default/__cdc_meta__/owner/22318f60bce74af9
a7c34c5a-0f2a-4561-bb69-de57541911f0
/tidb/cdc/default/default/upstream/7367221587711019178
{"id":7367221587711019178,"pd-endpoints":"http://127.0.0.1:2379,http://127.0.0.1:2379","key-path":"","cert-path":"","ca-path":"","cert-allowed-cn":null}'
+ grep -q 'etcd info'
+ break
+ set +x
+ cdc.test -test.coverprofile=/tmp/tidb_cdc_test/cov.multi_topics_v2.cli.11993.out cli changefeed create --start-ts=449659525666439169 '--sink-uri=kafka://127.0.0.1:9092/multi_topics?protocol=canal-json&enable-tidb-extension=true&kafka-version=2.4.1' --config /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/multi_topics_v2/conf/changefeed.toml
Create changefeed successfully!
ID: f107ba91-ceb3-465d-8844-9a682c659486
Info: {"upstream_id":7367221587711019178,"namespace":"default","id":"f107ba91-ceb3-465d-8844-9a682c659486","sink_uri":"kafka://127.0.0.1:9092/multi_topics?protocol=canal-json\u0026enable-tidb-extension=true\u0026kafka-version=2.4.1","create_time":"2024-05-10T12:22:47.970958888+08:00","start_ts":449659525666439169,"config":{"memory_quota":1073741824,"case_sensitive":false,"force_replicate":false,"ignore_ineligible_table":false,"check_gc_safe_point":true,"enable_sync_point":false,"bdr_mode":false,"sync_point_interval":600000000000,"sync_point_retention":86400000000000,"filter":{"rules":["*.*"]},"mounter":{"worker_num":16},"sink":{"protocol":"canal-json","csv":{"delimiter":",","quote":"\"","null":"\\N","include_commit_ts":false,"binary_encoding_method":"base64"},"dispatchers":[{"matcher":["test.*"],"topic":"{schema}_{table}"}],"encoder_concurrency":32,"terminator":"\r\n","date_separator":"day","enable_partition_separator":true,"enable_kafka_sink_v2":true,"only_output_updated_columns":false,"delete_only_output_handle_key_columns":false,"advance_timeout":150,"send_bootstrap_interval_in_sec":120,"send_bootstrap_in_msg_count":10000,"send_bootstrap_to_all_partition":true,"open":{"output_old_value":true}},"consistent":{"level":"none","max_log_size":64,"flush_interval":2000,"meta_flush_interval":200,"encoding_worker_num":16,"flush_worker_num":8,"use_file_backend":false,"memory_usage":{"memory_quota_percentage":50,"event_cache_percentage":0}},"scheduler":{"enable_table_across_nodes":false,"region_threshold":100000,"write_key_threshold":0},"integrity":{"integrity_check_level":"none","corruption_handle_level":"warn"},"changefeed_error_stuck_duration":1800000000000,"sql_mode":"ONLY_FULL_GROUP_BY,STRICT_TRANS_TABLES,NO_ZERO_IN_DATE,NO_ZERO_DATE,ERROR_FOR_DIVISION_BY_ZERO,NO_AUTO_CREATE_USER,NO_ENGINE_SUBSTITUTION","synced_status":{"synced_check_interval":300,"checkpoint_interval":15}},"state":"normal","creator_version":"v7.5.1-28-gdd60f55b4","resolved_ts":449659525666439169,"checkpoint_ts":449659525666439169,"checkpoint_time":"2024-05-10 12:22:43.022"}
PASS
coverage: 2.5% of statements in github.com/pingcap/tiflow/...
+ set +x
[Fri May 10 12:24:49 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>>
schema registry uri found: 1
[Fri May 10 12:24:49 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>>
schema registry uri found: 2
[Fri May 10 12:24:49 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>>
schema registry uri found: 3
table test.table1 not exists for 1-th check, retry later
table test.table1 exists
table test.table2 exists
table test.table3 exists
check diff successfully
table test.table10 not exists for 1-th check, retry later
table test.table10 exists
table test.table20 exists
check diff successfully
[Fri May 10 12:25:25 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>>
schema registry uri found: 10
[Fri May 10 12:25:25 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>>
schema registry uri found: 20
[Fri May 10 12:25:25 CST 2024] <<<<<< START kafka consumer in multi_topics_v2 case >>>>>>
schema registry uri found: finish
table test.finish not exists for 1-th check, retry later
table test.finish not exists for 2-th check, retry later
table test.finish exists
check diff successfully
wait process cdc.test exit for 1-th time...
wait process cdc.test exit for 2-th time...
cdc.test: no process found
wait process cdc.test exit for 3-th time...
process cdc.test already exit
[Fri May 10 12:25:31 CST 2024] <<<<<< run test case multi_topics_v2 success! >>>>>>
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh using Sink-Type: kafka... <<=================
+++ dirname /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/run.sh
++ cd /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup
++ pwd
+ CUR=/home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup
+ source /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/storage_cleanup/../_utils/test_prepare
++ UP_TIDB_HOST=127.0.0.1
++ UP_TIDB_PORT=4000
++ UP_TIDB_OTHER_PORT=4001
++ UP_TIDB_STATUS=10080
++ UP_TIDB_OTHER_STATUS=10081
++ DOWN_TIDB_HOST=127.0.0.1
++ DOWN_TIDB_PORT=3306
++ DOWN_TIDB_STATUS=20080
++ TLS_TIDB_HOST=127.0.0.1
++ TLS_TIDB_PORT=3307
++ TLS_TIDB_STATUS=30080
++ UP_PD_HOST_1=127.0.0.1
++ UP_PD_PORT_1=2379
++ UP_PD_PEER_PORT_1=2380
++ UP_PD_HOST_2=127.0.0.1
++ UP_PD_PORT_2=2679
++ UP_PD_PEER_PORT_2=2680
++ UP_PD_HOST_3=127.0.0.1
++ UP_PD_PORT_3=2779
++ UP_PD_PEER_PORT_3=2780
++ DOWN_PD_HOST=127.0.0.1
++ DOWN_PD_PORT=2479
++ DOWN_PD_PEER_PORT=2480
++ TLS_PD_HOST=127.0.0.1
++ TLS_PD_PORT=2579
++ TLS_PD_PEER_PORT=2580
++ UP_TIKV_HOST_1=127.0.0.1
++ UP_TIKV_PORT_1=20160
++ UP_TIKV_STATUS_PORT_1=20181
++ UP_TIKV_HOST_2=127.0.0.1
++ UP_TIKV_PORT_2=20161
++ UP_TIKV_STATUS_PORT_2=20182
++ UP_TIKV_HOST_3=127.0.0.1
++ UP_TIKV_PORT_3=20162
++ UP_TIKV_STATUS_PORT_3=20183
++ DOWN_TIKV_HOST=127.0.0.1
++ DOWN_TIKV_PORT=21160
++ DOWN_TIKV_STATUS_PORT=21180
++ TLS_TIKV_HOST=127.0.0.1
++ TLS_TIKV_PORT=22160
++ TLS_TIKV_STATUS_PORT=22180
+++ cat /tmp/tidb_cdc_test/KAFKA_VERSION
+++ echo 2.4.1
++ KAFKA_VERSION=2.4.1
+ WORK_DIR=/tmp/tidb_cdc_test/storage_cleanup
+ CDC_BINARY=cdc.test
+ SINK_TYPE=kafka
+ EXIST_FILES=()
+ CLEANED_FILES=()
+ trap stop_tidb_cluster EXIT
+ run kafka
+ '[' kafka '!=' storage ']'
+ return
+ check_logs /tmp/tidb_cdc_test/storage_cleanup
++ date
+ echo '[Fri May 10 12:25:45 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>'
[Fri May 10 12:25:45 CST 2024] <<<<<< run test case storage_cleanup success! >>>>>>
+ stop_tidb_cluster
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_basic/run.sh using Sink-Type: kafka... <<=================
[Fri May 10 12:25:48 CST 2024] <<<<<< run test case csv_storage_basic success! >>>>>>
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_multi_tables_ddl/run.sh using Sink-Type: kafka... <<=================
[Fri May 10 12:25:51 CST 2024] <<<<<< run test case csv_storage_multi_tables_ddl success! >>>>>>
=================>> Running test /home/jenkins/agent/workspace/pingcap/tiflow/release-7.5/pull_cdc_integration_kafka_test/tiflow/tests/integration_tests/csv_storage_partition_table/run.sh using Sink-Type: kafka... <<=================
[Fri May 10 12:25:57 CST 2024] <<<<<< run test case csv_storage_partition_table success! >>>>>>
\033[0;36m<<< Run all test success >>>\033[0m
[Pipeline] }
Cache not saved (ws/jenkins-pingcap-tiflow-release-7.5-pull_cdc_integration_kafka_test-578/tiflow-cdc already exists)
[Pipeline] // cache
[Pipeline] }
[Pipeline] // dir
[Pipeline] }
[Pipeline] // withCredentials
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // parallel
[Pipeline] }
[Pipeline] // stage
[Pipeline] }
[Pipeline] // timeout
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // container
[Pipeline] }
[Pipeline] // withEnv
[Pipeline] }
[Pipeline] // node
[Pipeline] }
[Pipeline] // podTemplate
[Pipeline] End of Pipeline
Finished: SUCCESS
![[Jenkins]](/jenkins/static/62b7ca6c/images/svgs/logo.svg){kind=logo}
